From 6a6c72801e6fb021851b4f2591b03ed136a27363 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Fri, 17 Jun 2016 12:54:26 +0200 Subject: beam_makeops: Separate static information from counters The counters are only used in the special 'icount' emulator. We will save some memory by including the counters in the OpEntry. It will also make it possible to make opc 'const'. --- erts/emulator/beam/erl_bif_info.c | 2 +- erts/emulator/beam/erl_vm.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 3fb866733c..e5d3f38ce4 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -2394,7 +2394,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) ERTS_ATOM_ENC_LATIN1, 1), erts_bld_uint(hpp, hszp, - opc[i].count)), + erts_instr_count[i])), res); } diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index f97716d030..22c8e8ee12 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -146,12 +146,13 @@ typedef struct op_entry { int sz; /* Number of loaded words. */ char* pack; /* Instructions for packing engine. */ char* sign; /* Signature string. */ - unsigned count; /* Number of times executed. */ } OpEntry; extern OpEntry opc[]; /* Description of all instructions. */ extern int num_instructions; /* Number of instruction in opc[]. */ +extern Uint erts_instr_count[]; + /* some constants for various table sizes etc */ #define ATOM_TEXT_SIZE 32768 /* Increment for allocating atom text space */ -- cgit v1.2.3 From 644cc778e9770b847700dca41a0c3ebe20da64df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Fri, 17 Jun 2016 05:23:08 +0200 Subject: beam_makeops: Save some memory by making loader tables 'const' Before: $ size bin/x86_64-unknown-linux-gnu/beam.smp text data bss dec hex filename 3080982 188369 158472 3427823 344def bin/x86_64-unknown-linux-gnu/beam.smp After: $ size bin/x86_64-unknown-linux-gnu/beam.smp text data bss dec hex filename 3164694 104657 158472 3427823 344def bin/x86_64-unknown-linux-gnu/beam.smp --- erts/emulator/beam/beam_load.c | 4 ++-- erts/emulator/beam/beam_load.h | 2 +- erts/emulator/beam/erl_vm.h | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 0c2743beb2..4ca2c6a13f 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -4860,7 +4860,7 @@ transform_engine(LoaderState* st) { Uint op; int ap; /* Current argument. */ - Uint* restart; /* Where to restart if current match fails. */ + const Uint* restart; /* Where to restart if current match fails. */ GenOpArg var[TE_MAX_VARS]; /* Buffer for variables. */ GenOpArg* rest_args = NULL; int num_rest_args = 0; @@ -4869,7 +4869,7 @@ transform_engine(LoaderState* st) GenOp* instr; GenOp* first = st->genop; GenOp* keep = NULL; - Uint* pc; + const Uint* pc; static Uint restart_fail[1] = {TOP_fail}; ASSERT(gen_opc[first->op].transform != -1); diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index fd2dd97fee..92a9abddbf 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -35,7 +35,7 @@ typedef struct gen_op_entry { int transform; } GenOpEntry; -extern GenOpEntry gen_opc[]; +extern const GenOpEntry gen_opc[]; #ifdef NO_JUMP_TABLE #define BeamOp(Op) (Op) diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index 22c8e8ee12..60c2349f36 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -148,8 +148,8 @@ typedef struct op_entry { char* sign; /* Signature string. */ } OpEntry; -extern OpEntry opc[]; /* Description of all instructions. */ -extern int num_instructions; /* Number of instruction in opc[]. */ +extern const OpEntry opc[]; /* Description of all instructions. */ +extern const int num_instructions; /* Number of instruction in opc[]. */ extern Uint erts_instr_count[]; -- cgit v1.2.3 From f0f4e72c8ec5c08993ff84d4eac5c48897a09657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Mon, 20 Jun 2016 13:02:59 +0200 Subject: Simplify creation of new GC BIFs Add the BIF type "gcbif" in bif.tab for defining GC BIFs. That will eliminate some of the hand-written administrative code for handling GC BIFs, saving the developer's time. --- erts/emulator/beam/beam_emu.c | 31 +++--------- erts/emulator/beam/beam_load.c | 111 +++++++++++++---------------------------- erts/emulator/beam/bif.tab | 38 +++++++------- erts/emulator/beam/global.h | 12 ----- 4 files changed, 61 insertions(+), 131 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 4716460a6b..c9fb5a3022 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -5428,31 +5428,14 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) static BifFunction translate_gc_bif(void* gcf) { - if (gcf == erts_gc_length_1) { - return length_1; - } else if (gcf == erts_gc_size_1) { - return size_1; - } else if (gcf == erts_gc_bit_size_1) { - return bit_size_1; - } else if (gcf == erts_gc_byte_size_1) { - return byte_size_1; - } else if (gcf == erts_gc_map_size_1) { - return map_size_1; - } else if (gcf == erts_gc_abs_1) { - return abs_1; - } else if (gcf == erts_gc_float_1) { - return float_1; - } else if (gcf == erts_gc_round_1) { - return round_1; - } else if (gcf == erts_gc_trunc_1) { - return round_1; - } else if (gcf == erts_gc_binary_part_2) { - return binary_part_2; - } else if (gcf == erts_gc_binary_part_3) { - return binary_part_3; - } else { - erts_exit(ERTS_ERROR_EXIT, "bad gc bif"); + const ErtsGcBif* p; + + for (p = erts_gc_bifs; p->bif != 0; p++) { + if (p->gc_bif == gcf) { + return p->bif; + } } + erts_exit(ERTS_ERROR_EXIT, "bad gc bif"); } /* diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 0c2743beb2..897b0f7e6a 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -4013,60 +4013,52 @@ gen_make_fun2(LoaderState* stp, GenOpArg idx) op->next = NULL; return op; } + +static GenOp* +translate_gc_bif(LoaderState* stp, GenOp* op, GenOpArg Bif) +{ + const ErtsGcBif* p; + BifFunction bf; + + bf = stp->import[Bif.val].bf; + for (p = erts_gc_bifs; p->bif != 0; p++) { + if (p->bif == bf) { + op->a[1].type = TAG_u; + op->a[1].val = (BeamInstr) p->gc_bif; + return op; + } + } + + op->op = genop_unsupported_guard_bif_3; + op->arity = 3; + op->a[0].type = TAG_a; + op->a[0].val = stp->import[Bif.val].module; + op->a[1].type = TAG_a; + op->a[1].val = stp->import[Bif.val].function; + op->a[2].type = TAG_u; + op->a[2].val = stp->import[Bif.val].arity; + return op; +} + /* - * Rewrite gc_bifs with one parameter (the common case). Utilized - * in ops.tab to rewrite instructions calling bif's in guards - * to use a garbage collecting implementation. + * Rewrite gc_bifs with one parameter (the common case). */ static GenOp* gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, GenOpArg Src, GenOpArg Dst) { GenOp* op; - BifFunction bf; NEW_GENOP(stp, op); op->next = NULL; - bf = stp->import[Bif.val].bf; - /* The translations here need to have a reverse counterpart in - beam_emu.c:translate_gc_bif for error handling to work properly. */ - if (bf == length_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_length_1; - } else if (bf == size_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_size_1; - } else if (bf == bit_size_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_bit_size_1; - } else if (bf == byte_size_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_byte_size_1; - } else if (bf == map_size_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_map_size_1; - } else if (bf == abs_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_abs_1; - } else if (bf == float_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_float_1; - } else if (bf == round_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_round_1; - } else if (bf == trunc_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_trunc_1; - } else { - op->op = genop_unsupported_guard_bif_3; - op->arity = 3; - op->a[0].type = TAG_a; - op->a[0].val = stp->import[Bif.val].module; - op->a[1].type = TAG_a; - op->a[1].val = stp->import[Bif.val].function; - op->a[2].type = TAG_u; - op->a[2].val = stp->import[Bif.val].arity; - return op; - } op->op = genop_i_gc_bif1_5; op->arity = 5; op->a[0] = Fail; - op->a[1].type = TAG_u; + /* op->a[1] is set by translate_gc_bif() */ op->a[2] = Src; op->a[3] = Live; op->a[4] = Dst; - return op; + return translate_gc_bif(stp, op, Bif); } /* @@ -4077,35 +4069,18 @@ gen_guard_bif2(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, GenOpArg S1, GenOpArg S2, GenOpArg Dst) { GenOp* op; - BifFunction bf; NEW_GENOP(stp, op); op->next = NULL; - bf = stp->import[Bif.val].bf; - /* The translations here need to have a reverse counterpart in - beam_emu.c:translate_gc_bif for error handling to work properly. */ - if (bf == binary_part_2) { - op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_2; - } else { - op->op = genop_unsupported_guard_bif_3; - op->arity = 3; - op->a[0].type = TAG_a; - op->a[0].val = stp->import[Bif.val].module; - op->a[1].type = TAG_a; - op->a[1].val = stp->import[Bif.val].function; - op->a[2].type = TAG_u; - op->a[2].val = stp->import[Bif.val].arity; - return op; - } op->op = genop_i_gc_bif2_6; op->arity = 6; op->a[0] = Fail; - op->a[1].type = TAG_u; + /* op->a[1] is set by translate_gc_bif() */ op->a[2] = Live; op->a[3] = S1; op->a[4] = S2; op->a[5] = Dst; - return op; + return translate_gc_bif(stp, op, Bif); } /* @@ -4116,37 +4091,19 @@ gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, GenOpArg S1, GenOpArg S2, GenOpArg S3, GenOpArg Dst) { GenOp* op; - BifFunction bf; NEW_GENOP(stp, op); op->next = NULL; - bf = stp->import[Bif.val].bf; - /* The translations here need to have a reverse counterpart in - beam_emu.c:translate_gc_bif for error handling to work properly. */ - if (bf == binary_part_3) { - op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_3; - } else { - op->op = genop_unsupported_guard_bif_3; - op->arity = 3; - op->a[0].type = TAG_a; - op->a[0].val = stp->import[Bif.val].module; - op->a[1].type = TAG_a; - op->a[1].val = stp->import[Bif.val].function; - op->a[2].type = TAG_u; - op->a[2].val = stp->import[Bif.val].arity; - return op; - } op->op = genop_ii_gc_bif3_7; op->arity = 7; op->a[0] = Fail; - op->a[1].type = TAG_u; + /* op->a[1] is set by translate_gc_bif() */ op->a[2] = Live; op->a[3] = S1; op->a[4] = S2; op->a[5] = S3; op->a[6] = Dst; - op->next = NULL; - return op; + return translate_gc_bif(stp, op, Bif); } static GenOp* diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 065018514a..978bcd4bba 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -23,22 +23,24 @@ # # Lines starting with '#' are ignored. # -# ::= "bif" * | "ubif" * +# ::= "bif" * | +# "ubif" * | +# "gcbif" * # ::= ":" "/" # -# "ubif" is an unwrapped bif, i.e. a bif without a trace wrapper, -# or rather; the trace entry point in the export entry is the same -# as the normal entry point, and no trace wrapper is generated. +# ubif: Use for operators and guard BIFs that never build anything +# on the heap (such as tuple_size/1) and operators. # -# Important: Use "ubif" for guard BIFs and operators; use "bif" for ordinary BIFs. +# gcbif: Use for guard BIFs that may build on the heap (such as abs/1). +# +# bif: Use for all other BIFs. # # Add new BIFs to the end of the file. # -# Note: Guards BIFs require special support in the compiler (to be able to actually -# call them from within a guard). +# Note: Guards BIFs usually require special support in the compiler. # -ubif erlang:abs/1 +gcbif erlang:abs/1 bif erlang:adler32/1 bif erlang:adler32/2 bif erlang:adler32_combine/3 @@ -62,7 +64,7 @@ bif erlang:exit/1 bif erlang:exit/2 bif erlang:external_size/1 bif erlang:external_size/2 -ubif erlang:float/1 +gcbif erlang:float/1 bif erlang:float_to_list/1 bif erlang:float_to_list/2 bif erlang:fun_info/2 @@ -79,7 +81,7 @@ bif erlang:phash2/2 ubif erlang:hd/1 bif erlang:integer_to_list/1 bif erlang:is_alive/0 -ubif erlang:length/1 +gcbif erlang:length/1 bif erlang:link/1 bif erlang:list_to_atom/1 bif erlang:list_to_binary/1 @@ -126,10 +128,10 @@ bif erlang:processes/0 bif erlang:put/2 bif erlang:register/2 bif erlang:registered/0 -ubif erlang:round/1 +gcbif erlang:round/1 ubif erlang:self/0 bif erlang:setelement/3 -ubif erlang:size/1 +gcbif erlang:size/1 bif erlang:spawn/3 bif erlang:spawn_link/3 bif erlang:split_binary/2 @@ -139,7 +141,7 @@ bif erlang:term_to_binary/2 bif erlang:throw/1 bif erlang:time/0 ubif erlang:tl/1 -ubif erlang:trunc/1 +gcbif erlang:trunc/1 bif erlang:tuple_to_list/1 bif erlang:universaltime/0 bif erlang:universaltime_to_localtime/1 @@ -461,8 +463,8 @@ bif erlang:list_to_existing_atom/1 # ubif erlang:is_bitstring/1 ubif erlang:tuple_size/1 -ubif erlang:byte_size/1 -ubif erlang:bit_size/1 +gcbif erlang:byte_size/1 +gcbif erlang:bit_size/1 bif erlang:list_to_bitstring/1 bif erlang:bitstring_to_list/1 @@ -514,8 +516,8 @@ bif erlang:binary_to_term/2 # # The searching/splitting/substituting thingies # -ubif erlang:binary_part/2 -ubif erlang:binary_part/3 +gcbif erlang:binary_part/2 +gcbif erlang:binary_part/3 bif binary:compile_pattern/1 bif binary:match/2 @@ -607,7 +609,7 @@ bif os:unsetenv/1 bif re:inspect/2 ubif erlang:is_map/1 -ubif erlang:map_size/1 +gcbif erlang:map_size/1 bif maps:to_list/1 bif maps:find/2 bif maps:get/2 diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index f3d4ac56cd..961260041f 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1427,18 +1427,6 @@ Eterm erts_gc_bor(Process* p, Eterm* reg, Uint live); Eterm erts_gc_bxor(Process* p, Eterm* reg, Uint live); Eterm erts_gc_bnot(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_length_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_size_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_bit_size_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_byte_size_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_map_size_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_float_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_round_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_trunc_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_binary_part_3(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_binary_part_2(Process* p, Eterm* reg, Uint live); - Uint erts_current_reductions(Process* current, Process *p); int erts_print_system_version(int to, void *arg, Process *c_p); -- cgit v1.2.3 From eb9766ca5d507d8cb8e1448780f96ba1cb45ff36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Wed, 22 Jun 2016 12:57:56 +0200 Subject: beam_debug: Improve the disassembly of gc_bif instructions Using the translation table in erts_gc_bifs[], we can now print out the name of GC BIFs, instead of just the pointer value. --- erts/emulator/beam/beam_debug.c | 55 ++++++++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 14 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index a4ad3e7886..8f47ed4d9d 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -51,6 +51,7 @@ void dbg_bt(Process* p, Eterm* sp); void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg); static int print_op(int to, void *to_arg, int op, int size, BeamInstr* addr); +static void print_bif_name(int to, void* to_arg, BifFunction bif); BIF_RETTYPE erts_debug_same_2(BIF_ALIST_2) @@ -520,7 +521,27 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) break; case 'I': /* Untagged integer. */ case 't': - erts_print(to, to_arg, "%d", *ap); + switch (op) { + case op_i_gc_bif1_jIsId: + case op_i_gc_bif2_jIIssd: + case op_i_gc_bif3_jIIssd: + { + const ErtsGcBif* p; + BifFunction gcf = (BifFunction) *ap; + for (p = erts_gc_bifs; p->bif != 0; p++) { + if (p->gc_bif == gcf) { + print_bif_name(to, to_arg, p->bif); + break; + } + } + if (p->bif == 0) { + erts_print(to, to_arg, "%d", (Uint)gcf); + } + break; + } + default: + erts_print(to, to_arg, "%d", *ap); + } ap++; break; case 'f': /* Destination label */ @@ -560,19 +581,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) case 'F': /* Function definition */ break; case 'b': - for (i = 0; i < BIF_SIZE; i++) { - BifFunction bif = (BifFunction) *ap; - if (bif == bif_table[i].f) { - break; - } - } - if (i == BIF_SIZE) { - erts_print(to, to_arg, "b(%d)", (Uint) *ap); - } else { - Eterm name = bif_table[i].name; - unsigned arity = bif_table[i].arity; - erts_print(to, to_arg, "%T/%u", name, arity); - } + print_bif_name(to, to_arg, (BifFunction) *ap); ap++; break; case 'P': /* Byte offset into tuple (see beam_load.c) */ @@ -731,3 +740,21 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) return size; } + +static void print_bif_name(int to, void* to_arg, BifFunction bif) +{ + int i; + + for (i = 0; i < BIF_SIZE; i++) { + if (bif == bif_table[i].f) { + break; + } + } + if (i == BIF_SIZE) { + erts_print(to, to_arg, "b(%d)", (Uint) bif); + } else { + Eterm name = bif_table[i].name; + unsigned arity = bif_table[i].arity; + erts_print(to, to_arg, "%T/%u", name, arity); + } +} -- cgit v1.2.3 From dfb64c6be9363a0f0f16bc0eb37da11ec4ec10fa Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 20 May 2016 17:19:40 +0200 Subject: erts: Improve system_info(dist_ctrl) to not block multi threading. Do the same as erlang:nodes/1, just grab dist table read lock. --- erts/emulator/beam/erl_bif_info.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 3fb866733c..6cedf3c8a5 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -2284,9 +2284,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } else if (ERTS_IS_ATOM_STR("dist_ctrl", BIF_ARG_1)) { DistEntry *dep; i = 0; - /* Need to be the only thread running... */ - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); for (dep = erts_visible_dist_entries; dep; dep = dep->next) ++i; for (dep = erts_hidden_dist_entries; dep; dep = dep->next) @@ -2309,8 +2307,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) res = CONS(hp, tpl, res); hp += 2; } - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); BIF_RET(res); } else if (BIF_ARG_1 == am_system_version) { erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0); -- cgit v1.2.3 From 6e9af06b17476cb680fbc33ae47dff173ea24204 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 29 Jun 2016 17:14:18 +0200 Subject: Fix group_leader/2 --- erts/emulator/beam/bif.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index fc14061a44..d9048065c8 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -4305,8 +4305,9 @@ BIF_RETTYPE group_leader_2(BIF_ALIST_2) else { locks &= ~ERTS_PROC_LOCK_STATUS; erts_smp_proc_unlock(new_member, ERTS_PROC_LOCK_STATUS); - if (erts_smp_atomic32_read_nob(&new_member->state) - & !(ERTS_PSFLG_DIRTY_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + if (new_member == BIF_P + || !(erts_smp_atomic32_read_nob(&new_member->state) + & (ERTS_PSFLG_DIRTY_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS))) { new_member->group_leader = STORE_NC_IN_PROC(new_member, BIF_ARG_1); } @@ -4326,6 +4327,7 @@ BIF_RETTYPE group_leader_2(BIF_ALIST_2) BIF_ARG_1); bp->next = new_member->mbuf; new_member->mbuf = bp; + new_member->mbuf_sz += bp->used_size; } } } -- cgit v1.2.3 From f0131c58c42a286c8b3f611b47106393a37197b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Magnus=20L=C3=A5ng?= Date: Mon, 23 May 2016 15:41:31 +0200 Subject: check_process_code: Sweep HiPE stack for literals Because check_process_code neglected checking the HiPE stack for references to the literal area, such references would survive the purge and subsequent deletion of a module and its literal area. These dangling references would then cause incorrect behaviour or even hard crashes of the VM. By simply adding a scan of the HiPE stack to check_process_code and erts_garbage_collect_literals, this problem is fixed. In order to support full stack walks without deleting the graylimit trap, a new stack walking interface function, nstack_walk_init_sdesc_ignore_trap() was introduced. --- erts/emulator/beam/beam_bif_load.c | 8 +++ erts/emulator/beam/erl_gc.c | 104 +++++++++++++++++++------------------ 2 files changed, 61 insertions(+), 51 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 15e878ba65..92815b2bcd 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -37,6 +37,10 @@ #include "erl_bits.h" #include "erl_thr_progress.h" +#ifdef HIPE +# include "hipe_stack.h" +#endif + static void set_default_trace_pattern(Eterm module); static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls); static void delete_code(Module* modp); @@ -916,6 +920,10 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls } if (any_heap_ref_ptrs(rp->stop, rp->hend, literals, lit_bsize)) goto try_literal_gc; +#ifdef HIPE + if (nstack_any_heap_ref_ptrs(rp, literals, lit_bsize)) + goto try_literal_gc; +#endif if (any_heap_refs(rp->heap, rp->htop, literals, lit_bsize)) goto try_literal_gc; if (any_heap_refs(rp->old_heap, rp->old_htop, literals, lit_bsize)) diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index d0d74bbf44..29646719d0 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -882,6 +882,58 @@ erts_garbage_collect_hibernate(Process* p) } +/* + * HiPE native code stack scanning procedures: + * - fullsweep_nstack() + * - gensweep_nstack() + * - offset_nstack() + * - sweep_literals_nstack() + */ +#if defined(HIPE) + +#define GENSWEEP_NSTACK(p,old_htop,n_htop) \ + do { \ + Eterm *tmp_old_htop = old_htop; \ + Eterm *tmp_n_htop = n_htop; \ + gensweep_nstack((p), &tmp_old_htop, &tmp_n_htop); \ + old_htop = tmp_old_htop; \ + n_htop = tmp_n_htop; \ + } while(0) + +/* + * offset_nstack() can ignore the descriptor-based traversal the other + * nstack procedures use and simply call offset_heap_ptr() instead. + * This relies on two facts: + * 1. The only live non-Erlang terms on an nstack are return addresses, + * and they will be skipped thanks to the low/high range check. + * 2. Dead values, even if mistaken for pointers into the low/high area, + * can be offset safely since they won't be dereferenced. + * + * XXX: WARNING: If HiPE starts storing other non-Erlang values on the + * nstack, such as floats, then this will have to be changed. + */ +static ERTS_INLINE void offset_nstack(Process* p, Sint offs, + char* area, Uint area_size) +{ + if (p->hipe.nstack) { + ASSERT(p->hipe.nsp && p->hipe.nstend); + offset_heap_ptr(hipe_nstack_start(p), hipe_nstack_used(p), + offs, area, area_size); + } + else { + ASSERT(!p->hipe.nsp && !p->hipe.nstend); + } +} + +#else /* !HIPE */ + +#define fullsweep_nstack(p,n_htop) (n_htop) +#define GENSWEEP_NSTACK(p,old_htop,n_htop) do{}while(0) +#define offset_nstack(p,offs,area,area_size) do{}while(0) +#define sweep_literals_nstack(p,old_htop,area,area_size) (old_htop) + +#endif /* HIPE */ + void erts_garbage_collect_literals(Process* p, Eterm* literals, Uint byte_lit_size, @@ -944,7 +996,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, area_size = byte_lit_size; n = setup_rootset(p, p->arg_reg, p->arity, &rootset); roots = rootset.roots; - old_htop = p->old_htop; + old_htop = sweep_literals_nstack(p, p->old_htop, area, area_size); while (n--) { Eterm* g_ptr = roots->v; Uint g_sz = roots->sz; @@ -1211,56 +1263,6 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end, return -1; } -/* - * HiPE native code stack scanning procedures: - * - fullsweep_nstack() - * - gensweep_nstack() - * - offset_nstack() - */ -#if defined(HIPE) - -#define GENSWEEP_NSTACK(p,old_htop,n_htop) \ - do { \ - Eterm *tmp_old_htop = old_htop; \ - Eterm *tmp_n_htop = n_htop; \ - gensweep_nstack((p), &tmp_old_htop, &tmp_n_htop); \ - old_htop = tmp_old_htop; \ - n_htop = tmp_n_htop; \ - } while(0) - -/* - * offset_nstack() can ignore the descriptor-based traversal the other - * nstack procedures use and simply call offset_heap_ptr() instead. - * This relies on two facts: - * 1. The only live non-Erlang terms on an nstack are return addresses, - * and they will be skipped thanks to the low/high range check. - * 2. Dead values, even if mistaken for pointers into the low/high area, - * can be offset safely since they won't be dereferenced. - * - * XXX: WARNING: If HiPE starts storing other non-Erlang values on the - * nstack, such as floats, then this will have to be changed. - */ -static ERTS_INLINE void offset_nstack(Process* p, Sint offs, - char* area, Uint area_size) -{ - if (p->hipe.nstack) { - ASSERT(p->hipe.nsp && p->hipe.nstend); - offset_heap_ptr(hipe_nstack_start(p), hipe_nstack_used(p), - offs, area, area_size); - } - else { - ASSERT(!p->hipe.nsp && !p->hipe.nstend); - } -} - -#else /* !HIPE */ - -#define fullsweep_nstack(p,n_htop) (n_htop) -#define GENSWEEP_NSTACK(p,old_htop,n_htop) do{}while(0) -#define offset_nstack(p,offs,area,area_size) do{}while(0) - -#endif /* HIPE */ - static void do_minor(Process *p, ErlHeapFragment *live_hf_end, char *mature, Uint mature_size, -- cgit v1.2.3 From 53bf2a9593a2f34151016f7d84fad6d213d76ee9 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Thu, 7 Jul 2016 20:59:28 +0200 Subject: erts: Check if return_to trace is enabled for bif We have to do an extra check if the return_to trace is enabled here as it may have been cleared by the bif after the flags where created by the call_trace. --- erts/emulator/beam/beam_bp.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 8489897d3a..920c8b1ed0 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -858,7 +858,8 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) if (flags & MATCH_SET_RX_TRACE) { erts_trace_return(p, ep->code, result, &ERTS_TRACER(p)); } - if (flags & MATCH_SET_RETURN_TO_TRACE) { + if (flags & MATCH_SET_RETURN_TO_TRACE && + IS_TRACED_FL(p, F_TRACE_RETURN_TO)) { /* can only happen if(local)*/ if (applying) { /* Apply of BIF, cp is in calling function */ -- cgit v1.2.3 From 75fc0b498602e6d11961567a737a9e4163d498b7 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 8 Jul 2016 15:37:05 +0200 Subject: erts: Optimize db_finalize_dbterm_hash Always free term after WUNLOCK_HASH --- erts/emulator/beam/erl_db_hash.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index fa925c94a5..ccffc2767a 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -2916,6 +2916,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) HashDbTerm **bp = (HashDbTerm **) handle->bp; HashDbTerm *b = *bp; erts_smp_rwmtx_t* lck = (erts_smp_rwmtx_t*) handle->lck; + HashDbTerm* free_me = NULL; ERTS_SMP_LC_ASSERT(IS_HASH_WLOCKED(tb, lck)); /* locked by db_lookup_dbterm_hash */ @@ -2927,7 +2928,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) b->hvalue = INVALID_HASH; } else { *bp = b->next; - free_term(tb, b); + free_me = b; } WUNLOCK_HASH(lck); @@ -2936,12 +2937,15 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) } else if (handle->flags & DB_MUST_RESIZE) { db_finalize_resize(handle, offsetof(HashDbTerm,dbterm)); WUNLOCK_HASH(lck); - - free_term(tb, b); + free_me = b; } else { WUNLOCK_HASH(lck); } + + if (free_me) + free_term(tb, free_me); + #ifdef DEBUG handle->dbterm = 0; #endif -- cgit v1.2.3 From 028455fe63ebc8f816c7d63029e5b562e2da5a86 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 8 Jul 2016 15:13:35 +0200 Subject: erts: Fix deadlock in ets:update_counter/4 in 'set' with 'write_concurrency' when inserting default object causes table to grow and the bucket to split is protected by same lock as the key. --- erts/emulator/beam/erl_db_hash.c | 34 ++++++++++++++++++---------------- erts/emulator/beam/erl_db_util.h | 1 + 2 files changed, 19 insertions(+), 16 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index ccffc2767a..0b31eb3bcd 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -2869,15 +2869,7 @@ db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj, q->hvalue = hval; q->next = NULL; *bp = b = q; - - { - int nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems); - int nactive = NACTIVE(tb); - - if (nitems > nactive * (CHAIN_LEN + 1) && !IS_FIXED(tb)) { - grow(tb, nactive); - } - } + flags |= DB_INC_TRY_GROW; } else { HashDbTerm *q, *next = b->next; @@ -2934,13 +2926,23 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) WUNLOCK_HASH(lck); erts_smp_atomic_dec_nob(&tb->common.nitems); try_shrink(tb); - } else if (handle->flags & DB_MUST_RESIZE) { - db_finalize_resize(handle, offsetof(HashDbTerm,dbterm)); - WUNLOCK_HASH(lck); - free_me = b; - } - else { - WUNLOCK_HASH(lck); + } else { + if (handle->flags & DB_MUST_RESIZE) { + db_finalize_resize(handle, offsetof(HashDbTerm,dbterm)); + free_me = b; + } + if (handle->flags & DB_INC_TRY_GROW) { + int nactive; + int nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems); + WUNLOCK_HASH(lck); + nactive = NACTIVE(tb); + + if (nitems > nactive * (CHAIN_LEN + 1) && !IS_FIXED(tb)) { + grow(tb, nactive); + } + } else { + WUNLOCK_HASH(lck); + } } if (free_me) diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 0903a40460..bfd81f5d86 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -79,6 +79,7 @@ typedef union db_table DbTable; #define DB_MUST_RESIZE 1 #define DB_NEW_OBJECT 2 +#define DB_INC_TRY_GROW 4 /* Info about a database entry while it's being updated * (by update_counter or update_element) -- cgit v1.2.3 From 3585c1e6832083acbfc130c0aafc8f2db2f3f706 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 8 Jul 2016 16:41:22 +0200 Subject: erts: Fix GC overrun bug in 'bsl' op with small Op1 Symptom: VM abort "Overrun stack and heap" Problem: The temporary bignum created in buffer tmp_big[] will be part of the GC initiated by TestHeapPreserve, but its size is not included which can cause the GC to overflow if very unlucky. Solution: Do not include tmp_big in the GC. --- erts/emulator/beam/beam_emu.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 4716460a6b..c6258287c4 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -3032,6 +3032,7 @@ do { \ if (i == 0) { StoreBifResult(4, Op1); } + ires = big_size(Op1); goto big_shift; } } else if (is_big(Op2)) { @@ -3047,7 +3048,6 @@ do { \ OpCase(i_bsl_jIssd): GetArg2(2, Op1, Op2); - do_bsl: if (is_small(Op2)) { i = signed_val(Op2); @@ -3073,16 +3073,12 @@ do { \ StoreBifResult(4, Op1); } } - Op1 = small_to_big(ires, tmp_big); -#ifdef TAG_LITERAL_PTR - Op1 |= TAG_LITERAL_PTR; -#endif + ires = 1; /* big_size(small_to_big(Op1)) */ big_shift: if (i > 0) { /* Left shift. */ - ires = big_size(Op1) + (i / D_EXP); + ires += (i / D_EXP); } else { /* Right shift. */ - ires = big_size(Op1); if (ires <= (-i / D_EXP)) ires = 3; /* ??? */ else @@ -3101,6 +3097,9 @@ do { \ goto lb_Cl_error; } TestHeapPreserve(ires+1, Arg(1), Op1); + if (is_small(Op1)) { + Op1 = small_to_big(signed_val(Op1), tmp_big); + } bigp = HTOP; Op1 = big_lshift(Op1, i, bigp); if (is_big(Op1)) { @@ -3123,6 +3122,7 @@ do { \ if (i == 0) { StoreBifResult(4, Op1); } + ires = big_size(Op1); goto big_shift; } } else if (is_big(Op2)) { -- cgit v1.2.3 From 382adc9e5d7c7dbd4e1f12adc483bc6eb7ed6c3d Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 8 Jul 2016 15:37:05 +0200 Subject: erts: Optimize db_finalize_dbterm_hash Always free term after WUNLOCK_HASH --- erts/emulator/beam/erl_db_hash.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 74979f984a..6b23be1adf 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -2910,6 +2910,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) HashDbTerm **bp = (HashDbTerm **) handle->bp; HashDbTerm *b = *bp; erts_smp_rwmtx_t* lck = (erts_smp_rwmtx_t*) handle->lck; + HashDbTerm* free_me = NULL; ERTS_SMP_LC_ASSERT(IS_HASH_WLOCKED(tb, lck)); /* locked by db_lookup_dbterm_hash */ @@ -2921,7 +2922,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) b->hvalue = INVALID_HASH; } else { *bp = b->next; - free_term(tb, b); + free_me = b; } WUNLOCK_HASH(lck); @@ -2930,12 +2931,15 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) } else if (handle->flags & DB_MUST_RESIZE) { db_finalize_resize(handle, offsetof(HashDbTerm,dbterm)); WUNLOCK_HASH(lck); - - free_term(tb, b); + free_me = b; } else { WUNLOCK_HASH(lck); } + + if (free_me) + free_term(tb, free_me); + #ifdef DEBUG handle->dbterm = 0; #endif -- cgit v1.2.3 From a6d87854c3ae65dba8dba6571b4d31d901316a39 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 8 Jul 2016 15:13:35 +0200 Subject: erts: Fix deadlock in ets:update_counter/4 in 'set' with 'write_concurrency' when inserting default object causes table to grow and the bucket to split is protected by same lock as the key. --- erts/emulator/beam/erl_db_hash.c | 34 ++++++++++++++++++---------------- erts/emulator/beam/erl_db_util.h | 1 + 2 files changed, 19 insertions(+), 16 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 6b23be1adf..12ae086b31 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -2866,15 +2866,7 @@ db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj, q->hvalue = hval; q->next = NULL; *bp = b = q; - - { - int nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems); - int nactive = NACTIVE(tb); - - if (nitems > nactive * (CHAIN_LEN + 1) && !IS_FIXED(tb)) { - grow(tb, nactive); - } - } + flags |= DB_INC_TRY_GROW; } else { HashDbTerm *q, *next = b->next; @@ -2928,13 +2920,23 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) WUNLOCK_HASH(lck); erts_smp_atomic_dec_nob(&tb->common.nitems); try_shrink(tb); - } else if (handle->flags & DB_MUST_RESIZE) { - db_finalize_resize(handle, offsetof(HashDbTerm,dbterm)); - WUNLOCK_HASH(lck); - free_me = b; - } - else { - WUNLOCK_HASH(lck); + } else { + if (handle->flags & DB_MUST_RESIZE) { + db_finalize_resize(handle, offsetof(HashDbTerm,dbterm)); + free_me = b; + } + if (handle->flags & DB_INC_TRY_GROW) { + int nactive; + int nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems); + WUNLOCK_HASH(lck); + nactive = NACTIVE(tb); + + if (nitems > nactive * (CHAIN_LEN + 1) && !IS_FIXED(tb)) { + grow(tb, nactive); + } + } else { + WUNLOCK_HASH(lck); + } } if (free_me) diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 60f7067d70..4acedbfed0 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -79,6 +79,7 @@ typedef union db_table DbTable; #define DB_MUST_RESIZE 1 #define DB_NEW_OBJECT 2 +#define DB_INC_TRY_GROW 4 /* Info about a database entry while it's being updated * (by update_counter or update_element) -- cgit v1.2.3 From 3b186417f6641a3aa13a46e00712d7dfcb29fc0f Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Thu, 7 Jul 2016 15:50:03 +0200 Subject: erts: After a call to a tracer nif, reset htop This is needed in order for the heap to never contain any pointers to invalid memory. This is good for performance and debugging reasons. --- erts/emulator/beam/erl_nif.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 23931f0e54..c6127a4967 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -3502,6 +3502,7 @@ Eterm erts_nif_call_function(Process *p, Process *tracee, struct enif_environment_t env; ErlHeapFragment *orig_hf = MBUF(p); ErlOffHeap orig_oh = MSO(p); + Eterm *orig_htop = HEAP_TOP(p); ASSERT(is_internal_pid(p->common.id)); MBUF(p) = NULL; clear_offheap(&MSO(p)); @@ -3523,6 +3524,7 @@ Eterm erts_nif_call_function(Process *p, Process *tracee, /* restore original heap fragment list */ MBUF(p) = orig_hf; MSO(p) = orig_oh; + HEAP_TOP(p) = orig_htop; } else { /* Nif call was done without a process context, so we create a phony one. */ -- cgit v1.2.3 From 8e1e877265b312c95978e9b70e8c383fb18f7508 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Wed, 13 Jul 2016 10:38:33 +0200 Subject: erts: Make sure to de-allocate the old seq tracer --- erts/emulator/beam/erl_trace.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 4cf38bf894..d14d237ca6 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -493,8 +493,8 @@ erts_get_system_seq_tracer(void) if (st != erts_tracer_nil && call_enabled_tracer(st, NULL, TRACE_FUN_ENABLED, am_trace_status, am_undefined) == am_remove) { - erts_set_system_seq_tracer(NULL, 0, erts_tracer_nil); - st = erts_tracer_nil; + st = erts_set_system_seq_tracer(NULL, 0, erts_tracer_nil); + ERTS_TRACER_CLEAR(&st); } return st; -- cgit v1.2.3 From 8d64850d816957f9decf26148eb4ced8275ea1f9 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Fri, 17 Jun 2016 16:20:53 +0200 Subject: hipe: Remove performance profiling code This type of statistics is now available through the microstate accounting API. --- erts/emulator/beam/benchmark.c | 301 --------------------------------------- erts/emulator/beam/benchmark.h | 295 -------------------------------------- erts/emulator/beam/erl_init.c | 4 - erts/emulator/beam/erl_message.c | 19 --- erts/emulator/beam/erl_process.c | 20 --- erts/emulator/beam/global.h | 1 - 6 files changed, 640 deletions(-) delete mode 100644 erts/emulator/beam/benchmark.c delete mode 100644 erts/emulator/beam/benchmark.h (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/benchmark.c b/erts/emulator/beam/benchmark.c deleted file mode 100644 index c8409784ef..0000000000 --- a/erts/emulator/beam/benchmark.c +++ /dev/null @@ -1,301 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2002-2016. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "sys.h" -#include "global.h" -#include "benchmark.h" - -#ifdef BM_COUNTERS -unsigned long long processes_busy; -unsigned long long processes_spawned; -unsigned long long messages_sent; -unsigned long long messages_copied; -unsigned long long messages_ego; -unsigned long long minor_gc; -unsigned long long major_gc; -#endif /* BM_COUNTERS */ - -#ifdef BM_TIMERS - -/* assuming Solaris */ -#include -BM_TIMER_T system_clock; - -unsigned long local_pause_times[MAX_PAUSE_TIME]; -unsigned long pause_times[MAX_PAUSE_TIME]; -unsigned long pause_times_old[MAX_PAUSE_TIME]; - -BM_TIMER_T mmu; -BM_TIMER_T mmu_counter; - -BM_NEW_TIMER(timer); -BM_NEW_TIMER(system); -BM_NEW_TIMER(gc); -BM_NEW_TIMER(minor_gc); -BM_NEW_TIMER(major_gc); -BM_NEW_TIMER(minor_global_gc); -BM_NEW_TIMER(major_global_gc); -BM_NEW_TIMER(send); -BM_NEW_TIMER(copy); -BM_NEW_TIMER(size); -BM_NEW_TIMER(max_minor); -BM_NEW_TIMER(max_major); -BM_NEW_TIMER(max_global_minor); -BM_NEW_TIMER(max_global_major); -BM_NEW_TIMER(misc0); -BM_NEW_TIMER(misc1); -BM_NEW_TIMER(misc2); -#endif /* BM_TIMERS */ - -#ifdef BM_HEAP_SIZES -unsigned long long max_used_heap; -unsigned long long max_allocated_heap; -unsigned long long max_used_global_heap; -unsigned long long max_allocated_global_heap; -#endif /* BM_HEAP_SIZES */ - -#ifdef BM_MESSAGE_SIZES -unsigned long long words_sent; -unsigned long long words_copied; -unsigned long long words_prealloc; -unsigned long long message_sizes[1000]; -#endif /* BM_MESSAGE_SIZES */ - -/***** - * The following functions have to be defined, but they only have contents - * if certain keywords are defined. - */ - -void init_benchmarking() -{ -#ifdef BM_TIMERS - int i; - for (i = 0; i < 1000; i++) - { - BM_START_TIMER(system); - BM_STOP_TIMER(system); - } - timer_time = system_time / 1000; - - for (i = 0; i < MAX_PAUSE_TIME; i++) { - local_pause_times[i] = 0; - pause_times[i] = 0; - pause_times_old[i] = 0; - } - - mmu = 0; - mmu_counter = 0; - - BM_MMU_INIT(); -#endif /* BM_TIMERS */ - -#ifdef BM_COUNTERS - processes_busy = 0; - processes_spawned = 0; - messages_sent = 0; - messages_copied = 0; - messages_ego = 0; - minor_gc = 0; - major_gc = 0; -#endif /* BM_COUNTERS */ - -#ifdef BM_HEAP_SIZES - max_used_heap = 0; - max_allocated_heap = 0; - max_used_global_heap = 0; - max_allocated_global_heap = 0; -#endif /* BM_HEAP_SIZES */ - -#ifdef BM_MESSAGE_SIZES - words_sent = 0; - words_copied = 0; - words_prealloc = 0; - { - int i; - for (i = 0; i < 1000; i++) - message_sizes[i] = 0; - } -#endif /* BM_MESSAGE_SIZES */ -} - -void save_statistics() -{ -#ifdef BM_STATISTICS - FILE *file = fopen(BM_STATISTICS_FILE,"a"); - long i = 0; - - if (file) - { - erts_fprintf(file,"-------------------------------------------------------------------------\n"); - erts_fprintf(file,"The counters are reset at system start and are sums over the entire node.\n"); - erts_fprintf(file,"You may reset them manually using the BIFs in the module hipe_bifs.\n"); - erts_fprintf(file,"All times are given in milliseconds.\n"); - erts_fprintf(file,"-------------------------------------------------------------------------\n"); - - erts_fprintf(file,"Node: %T\n",erts_this_node->sysname); - -#ifdef BM_COUNTERS - erts_fprintf(file,"Number of processes spawned: %lld\n",processes_spawned); - erts_fprintf(file,"Number of local minor GCs: %lld\n",minor_gc); - erts_fprintf(file,"Number of local major GCs: %lld\n",major_gc); - erts_fprintf(file,"Number of messages sent: %lld\n",messages_sent); - erts_fprintf(file,"Number of messages copied: %lld\n",messages_copied); - erts_fprintf(file,"Number of messages sent to self: %lld\n",messages_ego); -#endif /* BM_COUNTERS */ - -#ifdef BM_MESSAGE_SIZES - erts_fprintf(file,"Number of words sent: %lld\n",words_sent); - erts_fprintf(file,"Number of words copied: %lld\n",words_copied); - erts_fprintf(file,"Number of words preallocated: %lld\n",words_prealloc); -#endif /* BM_MESSAGE_SIZES */ - -#ifdef BM_HEAP_SIZES - erts_fprintf(file,"Biggest local heap used (in words): %lld\n",max_used_heap); - erts_fprintf(file,"Biggest local heap allocated (in words): %lld\n",max_allocated_heap); - erts_fprintf(file,"Biggest global heap used (in words): %lld\n",max_used_global_heap); - erts_fprintf(file,"Biggest global heap allocated (in words): %lld\n",max_allocated_global_heap); -#endif /* BM_HEAP_SIZES */ - -#ifdef BM_TIMERS - erts_fprintf(file,"--- The total active system time is the sum of all times below ---\n"); - BM_TIME_PRINTER("Mutator time",system_time); - BM_TIME_PRINTER("Time spent in send (excluding size & copy)",send_time); - BM_TIME_PRINTER("Time spent in size",size_time); - BM_TIME_PRINTER("Time spent in copy",copy_time); - BM_TIME_PRINTER("Time spent in local minor GC",minor_gc_time); - BM_TIME_PRINTER("Time spent in local major GC",major_gc_time); - BM_TIME_PRINTER("Time spent in global minor GC",minor_global_gc_time); - BM_TIME_PRINTER("Time spent in global major GC",major_global_gc_time); - erts_fprintf(file,"---\n"); - BM_TIME_PRINTER("Maximum time spent in one separate local minor GC",max_minor_time); - BM_TIME_PRINTER("Maximum time spent in one separate local major GC",max_major_time); - BM_TIME_PRINTER("Maximum time spent in one separate global minor GC",max_global_minor_time); - BM_TIME_PRINTER("Maximum time spent in one separate global major GC",max_global_major_time); -#endif /* BM_TIMERS */ - -#if 0 - /* Save a log file for import into excel */ - - long long total_time, n; - long left, right, mid; - -#ifdef BM_COUNTERS - erts_fprintf(file,"Spawns\tLocalGC\tMAGC\tMessages\tMutator_t\tLocalGC_t\tMAGC_t\tLocMaxP\tLocMeanP\tLocGeoMP\tMAMaxP\tMAMeanP\tMAGeoMP\t\tCMAGC\tCMAGC_t\n"); - erts_fprintf(file,"%lld\t%lld\t%lld\t%lld\t", - processes_spawned, - minor_garbage_cols + major_garbage_cols, - minor_global_garbage_cols + major_global_garbage_cols, - messages_sent); -#endif /* BM_COUNTERS */ - -#ifdef BM_TIMERS - erts_fprintf(file,"%lld\t%lld\t%lld\t", - (long long)(system_time + send_time + size_time + copy_time), - (long long)(minor_gc_time + major_gc_time), - (long long)(minor_global_gc_time + major_global_gc_time)); - - total_time = 0; n = 0; - left = 0; right = 0; mid = 0; - for (i = 0; i < MAX_PAUSE_TIME; i++) { - total_time += local_pause_times[i] * i; - n += local_pause_times[i]; - if (i > mid) - right += local_pause_times[i]; - while(right > left) { - left += local_pause_times[mid++]; - right -= local_pause_times[mid]; - } - } - erts_fprintf(file,"%lld\t%lld\t%ld\t", - (long long)((max_minor_time > max_major_time ? - max_minor_time : - max_major_time)*1000), - total_time / n, - mid); - - total_time = 0; n = 0; - left = 0; right = 0; mid = 0; - for (i = 0; i < MAX_PAUSE_TIME; i++) { - if (pause_times[i] > 0) { - total_time += pause_times[i] * i; - n += pause_times[i]; - if (i > mid) - right += pause_times[i]; - while(right > left) { - left += pause_times[mid++]; - right -= pause_times[mid]; - } - } - } - erts_fprintf(file,"%lld\t%lld\t%ld\t", - (long long)((max_global_minor_time > max_global_major_time ? - max_global_minor_time : - max_global_major_time)*1000), - (n > 0 ? total_time / n : 0), - mid); - - erts_fprintf(file,"\t%lld\t%lld\n",n,total_time); - - erts_fprintf(file,"\nMinor:\n"); - for (i = 0; i < MAX_PAUSE_TIME; i++) { - if (i < 1000 || pause_times[i] > 0) { - erts_fprintf(file,"%d\t%ld\n",i,pause_times[i]); - } - } - - fprintf(file,"Major:\n"); - for (i = 0; i < MAX_PAUSE_TIME; i++) { - if (pause_times_old[i] > 0) { - fprintf(file,"%d\t%ld\n",i,pause_times_old[i]); - } - } -#endif /* BM_TIMERS */ - -#ifdef BM_TIMERS - total_time = 0; n = 0; - left = 0; right = 0; mid = 0; - fprintf(file,"\nLocal:\n"); - for (i = 0; i < MAX_PAUSE_TIME; i++) { - if (local_pause_times[i] > 0) { - erts_fprintf(file,"%d\t%ld\n",i,local_pause_times[i]); - total_time += local_pause_times[i] * i; - n += local_pause_times[i]; - if (i > mid) - right += local_pause_times[i]; - while(right > left) { - left += local_pause_times[mid++]; - right -= local_pause_times[mid]; - } - } - } - erts_fprintf(file,"Mid: %ld Mean: %ld\n",(long)mid, - (long)(n > 0 ? total_time / n : 0)); -#endif -#endif /* 0 */ - fclose(file); - } - else - fprintf(stderr,"Sorry... Can not write to %s!\n\r",BM_STATISTICS_FILE); -#endif /* BM_STATISTICS */ -} diff --git a/erts/emulator/beam/benchmark.h b/erts/emulator/beam/benchmark.h deleted file mode 100644 index 0272896f4f..0000000000 --- a/erts/emulator/beam/benchmark.h +++ /dev/null @@ -1,295 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2002-2016. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -#ifndef __BENCHMARK_H__ -#define __BENCHMARK_H__ - -/* The define __BENCHMARK__ is the master switch to turn on and off - * benchmarking. This will enable the benchmark-BIFs in hipe_bif1.c. - * Documentation for the BIFs is in hipe_bif1.c, and that is where you - * will find the information about how to accually get some data out - * from these timers and counters. - */ -/* #define __BENCHMARK__ */ - -#ifdef __BENCHMARK__ -/* - * The defines below enables different parts of the benchmaring. - * Counters and timers that are disabled, always report zero in - * the BIFs. - */ - -/* BM_TIMERS keeps track of the time spent in diferent parts of the - * system. It only measures accual active time, not time spent in idle - * mode. Currently, the Solaris hrtime_t will be used. - * To add new timers look below. - */ -#define BM_TIMERS - -/* BM_COUNTERS count all kinds of events that occurs in the system. - * Among other things it counts the number of messages, then number of - * garbage collections, the number of processes spawned etc. - * To add new counters look below. - */ -#define BM_COUNTERS - -/* BM_MESSAGE_SIZES keeps a log of the size of all messages sent in - * the system. This introduce an overhead in time for the shared heap - * system since all message sizes have to be calculated at send. - */ -/* #define BM_MESSAGE_SIZES */ - -/* BM_HEAP_SIZES goes through all processes at garbage collection time - * to sum their allocated and used heap sizes. In anything else than a - * shared heap system, this will cost. - */ -/* #define BM_HEAP_SIZES */ - -/* BM_STATISTICS saves an entry in the file BM_STATISTICS_FILE. This - * is done for each erlang node at exit time. - */ -/* #define BM_STATISTICS */ - -#endif /* __BENCHMARK__ */ - - -#ifdef BM_STATISTICS -# define BM_STATISTICS_FILE "/tmp/erlang_statistics.joppe.log" -#endif /* BM_STATISTICS */ - - -/************ There are no more settings below this line *************/ - -/* - * Maintenance and how to add new stuff is documented by the code - * below ;-) - */ - -#ifdef BM_COUNTERS -/********************************************************************* - * To add new counters: - * - * Add the variable here AND in benchmark.c. Use the macro - * BM_COUNT(var) in the code where you want to increase it. - * - */ -extern unsigned long long processes_busy; -extern unsigned long long processes_spawned; -extern unsigned long long messages_sent; -extern unsigned long long messages_copied; -extern unsigned long long messages_ego; -extern unsigned long long minor_gc; -extern unsigned long long major_gc; - -#define BM_COUNT(var) (var)++; - -#define BM_EGO_COUNT(send,rec) { \ - if ((send) == (rec)) \ - BM_COUNT(messages_ego); } - -#define BM_LAZY_COPY_START long long gcs = minor_global_gc + major_global_gc; -#define BM_LAZY_COPY_STOP { gcs = (minor_global_gc + major_global_gc) - gcs; \ - if (gcs > gc_in_copy) gc_in_copy = gcs; } - -#else /* !BM_COUNTERS */ -# define BM_COUNT(var) -# define BM_EGO_COUNT(send,rec) -# define BM_LAZY_COPY_START -# define BM_LAZY_COPY_STOP -#endif /* BM_COUNTERS */ - - -#ifdef BM_TIMERS -/********************************************************************* - * To add new timers: - * - * Add the variable below using the form extern BM_TIMER_T blah_time. - * Also add them in benchmark.c using the macro NEW_TIMER(blah). Use - * the macro BM_SWAP_TIMER(from,blah) ... BM_SWAP_TIMER(blah,to) to - * start and stop the new timer. Note, that you have to know what - * timer is running at the place where you want to insert your new - * timer to be able to stop and start (from,to) it. - * - * You can use the macros BM_STOP_TIMER(blah) and BM_START_TIMER(blah) - * around code that should not be timed at all. As above, you have to - * know what timer to start and stop. The system timer is running at - * most places in the emulator. Only the garbage collector and the - * message sending has its own timers at the moment. - * - * The timer_time used when stopping timers is the time it takes to - * start and stop the timers, calculated in init_benchmarking(). If it - * is not there, the time it takes to do this will accually be - * substantial compared to some small times in the system we want to - * meassure (send time in shared heap for instance). - */ - -/* (Assuming Solaris) */ - -#define BM_TIMER_T ErtsMonotonicTime -#define BM_START_TIMER(t) system_clock = ERTS_MONOTONIC_TO_NSEC(erts_os_monotonic_time()) -#define BM_STOP_TIMER(t) do { \ - BM_TIMER_T tmp = (ERTS_MONOTONIC_TO_NSEC(erts_os_monotonic_time()) - system_clock) - timer_time; \ - t##_time += (tmp > 0 ? tmp : 0); \ -} while(0) - -#define BM_TIME_PRINTER(str,time) do { \ - int min,sec,milli,micro; \ - BM_TIMER_T tmp; \ - tmp = (time) / 1000; \ - micro = tmp % 1000; \ - tmp /= 1000; \ - milli = tmp % 1000; \ - tmp /= 1000; \ - sec = tmp % 60; \ - min = tmp / 60; \ - erts_fprintf(file,str": %d:%02d.%03d %03d\n",min,sec,milli,micro); \ -} while(0) - -extern BM_TIMER_T system_clock; - -extern BM_TIMER_T timer_time; -extern BM_TIMER_T system_time; -extern BM_TIMER_T gc_time; -extern BM_TIMER_T minor_gc_time; -extern BM_TIMER_T major_gc_time; -extern BM_TIMER_T minor_global_gc_time; -extern BM_TIMER_T major_global_gc_time; -extern BM_TIMER_T send_time; -extern BM_TIMER_T copy_time; -extern BM_TIMER_T size_time; -extern BM_TIMER_T max_minor_time; -extern BM_TIMER_T max_major_time; -extern BM_TIMER_T max_global_minor_time; -extern BM_TIMER_T max_global_major_time; -extern BM_TIMER_T misc0_time; -extern BM_TIMER_T misc1_time; -extern BM_TIMER_T misc2_time; - -#define MAX_PAUSE_TIME 500000 -extern unsigned long local_pause_times[MAX_PAUSE_TIME]; -extern unsigned long pause_times[MAX_PAUSE_TIME]; -extern unsigned long pause_times_old[MAX_PAUSE_TIME]; - -#define MMU_INTERVAL 5 /* milli seconds */ -extern BM_TIMER_T mmu_counter; -extern BM_TIMER_T mmu; - -#define BM_NEW_TIMER(t) BM_TIMER_T t##_time = 0; -#define BM_RESET_TIMER(t) t##_time = 0; -#define BM_SWAP_TIMER(t1,t2) do { BM_STOP_TIMER(t1); BM_START_TIMER(t2); } while(0) -#define BM_MMU_INIT() do { \ - BM_TIMER_T gc = gc_time; \ - while (gc > 0) { \ - if (gc > MMU_INTERVAL) { \ - gc -= MMU_INTERVAL - mmu_counter; \ - erts_printf("%d\n",(int)((mmu / MMU_INTERVAL) * 100)); \ - mmu_counter = 0; mmu = 0; \ - } else { \ - mmu_counter += gc; \ - if (mmu_counter >= MMU_INTERVAL) { \ - mmu_counter -= MMU_INTERVAL; \ - erts_printf("%d\n",(int)((mmu / MMU_INTERVAL) * 100)); \ - mmu = 0; \ - } \ - gc = 0; \ - } \ - } \ - BM_RESET_TIMER(system); \ - BM_RESET_TIMER(send); \ - BM_RESET_TIMER(copy); \ - BM_RESET_TIMER(size); \ -} while(0) - -#define BM_MMU_READ() do { \ - BM_TIMER_T mut = system_time + send_time + copy_time + size_time; \ - while (mut > 0) { \ - if (mut > MMU_INTERVAL) { \ - BM_TIMER_T tmp = MMU_INTERVAL - mmu_counter; \ - mmu += tmp; mut -= tmp; \ - erts_printf("%d\n",(int)((mmu / MMU_INTERVAL) * 100)); \ - mmu_counter = 0; mmu = 0; \ - } else { \ - mmu_counter += mut; mmu += mut; \ - if (mmu_counter >= MMU_INTERVAL) { \ - mmu_counter -= MMU_INTERVAL; \ - mmu -= mmu_counter; \ - erts_printf("%d\n",(int)((mmu / MMU_INTERVAL) * 100)); \ - mmu = mmu_counter; \ - } \ - mut = 0; \ - } \ - } \ -} while(0) - -#else /* !BM_TIMERS */ -# define BM_NEW_TIMER(t) -# define BM_START_TIMER(t) -# define BM_STOP_TIMER(t) -# define BM_RESET_TIMER(t) -# define BM_SWAP_TIMER(t1,t2) -# define BM_TIME_PRINTER(str,time) -# define BM_MMU_INIT() -# define BM_MMU_READ() -#endif /* BM_TIMERS */ - -#ifdef BM_HEAP_SIZES -extern unsigned long long max_used_heap; -extern unsigned long long max_allocated_heap; -extern unsigned long long max_used_global_heap; -extern unsigned long long max_allocated_global_heap; -#endif /* BM_HEAP_SIZES */ - -#ifdef BM_MESSAGE_SIZES -extern unsigned long long words_sent; -extern unsigned long long words_copied; -extern unsigned long long words_prealloc; -extern unsigned long long message_sizes[1000]; - -#define BM_MESSAGE_COPIED(size) { \ - words_copied += size; \ - BM_COUNT(messages_copied); } - -#define BM_PREALLOC_DATA(size) { \ - words_prealloc += size; } - -#define BM_MESSAGE(mess,send,rec) { \ - Uint msize = size_object(mess); \ - words_sent += msize; \ - if (msize < 1000) \ - message_sizes[msize]++; \ - else \ - message_sizes[999]++; \ - BM_EGO_COUNT(send,rec); \ - BM_COUNT(messages_sent); } - -#else /* !BM_MESSAGE_SIZES */ - -#define BM_MESSAGE_COPIED(size) BM_COUNT(messages_copied); -#define BM_PREALLOC_DATA(size) -#define BM_MESSAGE(mess,send,rec) { \ - BM_EGO_COUNT(send,rec); \ - BM_COUNT(messages_sent); } - -#endif /* BM_MESSAGE_SIZES */ - -void init_benchmarking(void); -void save_statistics(void); - -#endif /* _BENCHMARK_H_ */ diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index fbdafec4ef..2239b6f045 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -332,8 +332,6 @@ erl_init(int ncpu, int node_tab_delete_delay, ErtsDbSpinCount db_spin_count) { - init_benchmarking(); - erts_bif_unique_init(); erts_init_monitors(); erts_init_time(time_correction, time_warp_mode); @@ -2345,8 +2343,6 @@ erts_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) { system_cleanup(flush_async); - save_statistics(); - if (erts_mtrace_enabled) erts_mtrace_exit((Uint32) n); diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 71ab92937d..91e06cde2d 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -697,9 +697,6 @@ erts_send_message(Process* sender, #ifdef SHCOPY_SEND erts_shcopy_t info; #endif - BM_STOP_TIMER(system); - BM_MESSAGE(message,sender,receiver); - BM_START_TIMER(send); #ifdef USE_VM_PROBES *sender_name = *receiver_name = '\0'; @@ -720,7 +717,6 @@ erts_send_message(Process* sender, #ifdef USE_VM_PROBES Uint dt_utag_size = 0; #endif - BM_SWAP_TIMER(send,size); /* SHCOPY corrupts the heap between * copy_shared_calculate, and @@ -747,8 +743,6 @@ erts_send_message(Process* sender, #else msize = size_object(message); #endif - BM_SWAP_TIMER(size,send); - mp = erts_alloc_message_heap_state(receiver, &receiver_state, receiver_locks, @@ -760,8 +754,6 @@ erts_send_message(Process* sender, &hp, &ohp); - BM_SWAP_TIMER(send,copy); - #ifdef SHCOPY_SEND if (is_not_immed(message)) message = copy_shared_perform(message, msize, &info, &hp, ohp); @@ -792,9 +784,6 @@ erts_send_message(Process* sender, msize, tok_label, tok_lastcnt, tok_serial); } #endif - BM_MESSAGE_COPIED(msize); - BM_SWAP_TIMER(copy,send); - } else { Eterm *hp; @@ -803,22 +792,18 @@ erts_send_message(Process* sender, msize = 0; } else { - BM_SWAP_TIMER(send,size); #ifdef SHCOPY_SEND INITIALIZE_SHCOPY(info); msize = copy_shared_calculate(message, &info); #else msize = size_object(message); #endif - BM_SWAP_TIMER(size,send); - mp = erts_alloc_message_heap_state(receiver, &receiver_state, receiver_locks, msize, &hp, &ohp); - BM_SWAP_TIMER(send,copy); #ifdef SHCOPY_SEND if (is_not_immed(message)) message = copy_shared_perform(message, msize, &info, &hp, ohp); @@ -827,8 +812,6 @@ erts_send_message(Process* sender, if (is_not_immed(message)) message = copy_struct(message, msize, &hp, ohp); #endif - BM_MESSAGE_COPIED(msz); - BM_SWAP_TIMER(copy,send); } #ifdef USE_VM_PROBES DTRACE6(message_send, sender_name, receiver_name, @@ -846,8 +829,6 @@ erts_send_message(Process* sender, mp, message, sender->common.id); - BM_SWAP_TIMER(send,system); - return res; } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 66f22979ad..f0f5b13f9d 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -9588,8 +9588,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) erts_smp_runq_lock(rq); } } - BM_STOP_TIMER(system); - } ERTS_SMP_LC_ASSERT(!is_normal_sched || !erts_thr_progress_is_blocking()); @@ -9827,8 +9825,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_EMULATOR); - BM_START_TIMER(system); - /* * Take the chosen process out of the queue. */ @@ -11154,18 +11150,11 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). || (erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ)); -#ifdef BM_COUNTERS - processes_busy++; -#endif - BM_COUNT(processes_spawned); - - BM_SWAP_TIMER(system,size); #ifdef SHCOPY_SPAWN arg_size = copy_shared_calculate(args, &info); #else arg_size = size_object(args); #endif - BM_SWAP_TIMER(size,system); heap_need = arg_size; p->flags = flags; @@ -11242,18 +11231,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->max_arg_reg = sizeof(p->def_arg_reg)/sizeof(p->def_arg_reg[0]); p->arg_reg[0] = mod; p->arg_reg[1] = func; - BM_STOP_TIMER(system); - BM_MESSAGE(args,p,parent); - BM_START_TIMER(system); - BM_SWAP_TIMER(system,copy); #ifdef SHCOPY_SPAWN p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap); DESTROY_SHCOPY(info); #else p->arg_reg[2] = copy_struct(args, arg_size, &p->htop, &p->off_heap); #endif - BM_MESSAGE_COPIED(arg_size); - BM_SWAP_TIMER(copy,system); p->arity = 3; p->fvalue = NIL; @@ -12826,9 +12809,6 @@ erts_continue_exit_process(Process *p) dep = (p->flags & F_DISTRIBUTION) ? erts_this_dist_entry : NULL; erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); -#ifdef BM_COUNTERS - processes_busy--; -#endif if (dep) { erts_do_net_exits(dep, reason); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index f3d4ac56cd..8933bcad16 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -35,7 +35,6 @@ #include "register.h" #include "erl_fun.h" #include "erl_node_tables.h" -#include "benchmark.h" #include "erl_process.h" #include "erl_sys_driver.h" #include "erl_debug.h" -- cgit v1.2.3 From cb5fca03189deb691b9bc2f1f8bd65033bd99320 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Fri, 17 Jun 2016 16:03:58 +0200 Subject: erts: Fix msacc for dirty scheduler and heap_limit --- erts/emulator/beam/erl_gc.c | 5 ++++- erts/emulator/beam/erl_process.c | 8 ++++++-- erts/emulator/beam/erl_process.h | 3 +-- 3 files changed, 11 insertions(+), 5 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index d0d74bbf44..a224383493 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -672,6 +672,7 @@ do_major_collection: killed before a GC could be done. */ if (reds == -2) { ErtsProcLocks locks = ERTS_PROC_LOCKS_ALL; + int res; erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); erts_send_exit_signal(p, p->common.id, p, &locks, @@ -683,7 +684,9 @@ do_major_collection: erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); /* We have to make sure that we have space for need on the heap */ - return delay_garbage_collection(p, live_hf_end, need, fcalls); + res = delay_garbage_collection(p, live_hf_end, need, fcalls); + ERTS_MSACC_POP_STATE_M(); + return res; } erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index f0f5b13f9d..b5d8c5bc75 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -8177,6 +8177,8 @@ sched_dirty_cpu_thread_func(void *vesdp) esdp->thr_id += erts_no_schedulers; + erts_msacc_init_thread("dirty_cpu_scheduler", no, 0); + erts_thr_progress_register_unmanaged_thread(&callbacks); #ifdef ERTS_ENABLE_LOCK_CHECK { @@ -8222,6 +8224,8 @@ sched_dirty_io_thread_func(void *vesdp) esdp->thr_id += erts_no_schedulers + erts_no_dirty_cpu_schedulers; + erts_msacc_init_thread("dirty_io_scheduler", no, 0); + erts_thr_progress_register_unmanaged_thread(&callbacks); #ifdef ERTS_ENABLE_LOCK_CHECK { @@ -9823,8 +9827,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) goto check_activities_to_run; } - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_EMULATOR); - /* * Take the chosen process out of the queue. */ @@ -9930,6 +9932,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) } + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_EMULATOR); + #ifdef ERTS_SMP if (flags & ERTS_RUNQ_FLG_PROTECTED) diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 7c98b60647..714e63e986 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -989,8 +989,7 @@ struct process { Uint32 rcount; /* suspend count */ int schedule_count; /* Times left to reschedule a low prio process */ Uint reds; /* No of reductions for this process */ - Eterm group_leader; /* Pid in charge - (can be boxed) */ + Eterm group_leader; /* Pid in charge (can be boxed) */ Uint flags; /* Trap exit, etc (no trace flags anymore) */ Eterm fvalue; /* Exit & Throw value (failure reason) */ Uint freason; /* Reason for detected failure */ -- cgit v1.2.3 From 80b628bb624101e1a4aaee0ddeb0274ee2aac455 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Sun, 19 Jun 2016 22:52:30 +0200 Subject: erts: Add erts_map_from_ks_and_vs --- erts/emulator/beam/beam_emu.c | 6 +++++- erts/emulator/beam/erl_map.c | 48 +++++++++++++++++++++++++++++++++++-------- erts/emulator/beam/erl_map.h | 8 +++++--- 3 files changed, 50 insertions(+), 12 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 4716460a6b..797880d452 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -7017,7 +7017,11 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) /* The expensive case, need to build a hashmap */ if (n > MAP_SMALL_MAP_LIMIT) { - res = erts_hashmap_from_ks_and_vs(p,flatmap_get_keys(mp),flatmap_get_values(mp),n); + ErtsHeapFactory factory; + erts_factory_proc_init(&factory, p); + res = erts_hashmap_from_ks_and_vs(&factory,flatmap_get_keys(mp), + flatmap_get_values(mp),n); + erts_factory_close(&factory); } return res; } diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index 8efc983f04..979a0040b0 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -497,18 +497,50 @@ Eterm erts_hashmap_from_array(ErtsHeapFactory* factory, Eterm *leafs, Uint n, return res; } +Eterm erts_map_from_ks_and_vs(ErtsHeapFactory *factory, Eterm *ks0, Eterm *vs0, Uint n) +{ + if (n < MAP_SMALL_MAP_LIMIT) { + Eterm *ks, *vs, *hp; + flatmap_t *mp; + Eterm keys; -Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n, + hp = erts_produce_heap(factory, 3 + 1 + (2 * n), 0); + keys = make_tuple(hp); + *hp++ = make_arityval(n); + ks = hp; + hp += n; + mp = (flatmap_t*)hp; + hp += MAP_HEADER_FLATMAP_SZ; + vs = hp; + + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = n; + mp->keys = keys; + + sys_memcpy(ks, ks0, n * sizeof(Eterm)); + sys_memcpy(vs, vs0, n * sizeof(Eterm)); + + erts_validate_and_sort_flatmap(mp); + + return make_flatmap(mp); + } else { + return erts_hashmap_from_ks_and_vs(factory, ks0, vs0, n); + } + return THE_NON_VALUE; +} + + +Eterm erts_hashmap_from_ks_and_vs_extra(ErtsHeapFactory *factory, + Eterm *ks, Eterm *vs, Uint n, Eterm key, Eterm value) { Uint32 sw, hx; Uint i,sz; hxnode_t *hxns; - ErtsHeapFactory factory; Eterm *hp, res; sz = (key == THE_NON_VALUE) ? n : (n + 1); ASSERT(sz > MAP_SMALL_MAP_LIMIT); - hp = HAlloc(p, 2 * sz); + hp = erts_produce_heap(factory, 2 * sz, 0); /* create tmp hx values and leaf ptrs */ hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, sz * sizeof(hxnode_t)); @@ -531,12 +563,9 @@ Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n hxns[i].i = i; } - erts_factory_proc_init(&factory, p); - res = hashmap_from_unsorted_array(&factory, hxns, sz, 0); - erts_factory_close(&factory); + res = hashmap_from_unsorted_array(factory, hxns, sz, 0); erts_free(ERTS_ALC_T_TMP, (void *) hxns); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); return res; } @@ -1780,11 +1809,14 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) { /* the map will grow */ if (n >= MAP_SMALL_MAP_LIMIT) { + ErtsHeapFactory factory; HRelease(p, shp + MAP_HEADER_FLATMAP_SZ + n, shp); ks = flatmap_get_keys(mp); vs = flatmap_get_values(mp); - res = erts_hashmap_from_ks_and_vs_extra(p,ks,vs,n,key,value); + erts_factory_proc_init(&factory, p); + res = erts_hashmap_from_ks_and_vs_extra(&factory,ks,vs,n,key,value); + erts_factory_close(&factory); return res; } diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h index 8b5c9582ba..61a841f7f0 100644 --- a/erts/emulator/beam/erl_map.h +++ b/erts/emulator/beam/erl_map.h @@ -98,10 +98,12 @@ Eterm* hashmap_iterator_prev(struct ErtsWStack_* s); int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp); Eterm erts_hashmap_from_array(ErtsHeapFactory*, Eterm *leafs, Uint n, int reject_dupkeys); -#define erts_hashmap_from_ks_and_vs(P, KS, VS, N) \ - erts_hashmap_from_ks_and_vs_extra((P), (KS), (VS), (N), THE_NON_VALUE, THE_NON_VALUE); +#define erts_hashmap_from_ks_and_vs(F, KS, VS, N) \ + erts_hashmap_from_ks_and_vs_extra((F), (KS), (VS), (N), THE_NON_VALUE, THE_NON_VALUE); -Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n, +Eterm erts_map_from_ks_and_vs(ErtsHeapFactory *factory, Eterm *ks, Eterm *vs, Uint n); +Eterm erts_hashmap_from_ks_and_vs_extra(ErtsHeapFactory *factory, + Eterm *ks, Eterm *vs, Uint n, Eterm k, Eterm v); const Eterm *erts_maps_get(Eterm key, Eterm map); -- cgit v1.2.3 From f5d6f414691313a60468c9aa2f6fd862f94e69c2 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Sun, 19 Jun 2016 22:54:39 +0200 Subject: erts: Add extra bif msacc states --- erts/emulator/beam/beam_emu.c | 16 +--- erts/emulator/beam/erl_msacc.c | 190 +++++++++++++++++++++-------------------- erts/emulator/beam/erl_msacc.h | 51 ++++++++--- 3 files changed, 137 insertions(+), 120 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 797880d452..7419a62058 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -2826,13 +2826,7 @@ do { \ goto context_switch3; } - if (ERTS_MSACC_IS_ENABLED_CACHED_X()) { - if (GET_BIF_MODULE(Arg(0)) == am_ets) { - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS); - } else { - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_BIF); - } - } + ERTS_MSACC_SET_BIF_STATE_CACHED_X(GET_BIF_MODULE(Arg(0)), GET_BIF_ADDRESS(Arg(0))); bf = GET_BIF_ADDRESS(Arg(0)); @@ -3593,13 +3587,7 @@ do { \ goto context_switch; } - if (ERTS_MSACC_IS_ENABLED_CACHED_X()) { - if ((Eterm)I[-3] == am_ets) { - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS); - } else { - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_BIF); - } - } + ERTS_MSACC_SET_BIF_STATE_CACHED_X((Eterm)I[-3], (BifFunction)Arg(0)); c_p->current = I-3; /* In case we apply process_info/1,2 or load_nif/1 */ c_p->i = I; /* In case we apply check_process_code/2. */ diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c index 544bc8b983..421445fbad 100644 --- a/erts/emulator/beam/erl_msacc.c +++ b/erts/emulator/beam/erl_msacc.c @@ -40,10 +40,11 @@ #include "erl_bif_unique.h" #include "erl_map.h" #include "erl_msacc.h" +#include "erl_bif_table.h" #if ERTS_ENABLE_MSACC -static Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, Eterm **hpp, Uint *szp); +static Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, ErtsHeapFactory *factory); static void erts_msacc_reset(ErtsMsAcc *msacc); static ErtsMsAcc* get_msacc(void); @@ -52,7 +53,9 @@ erts_tsd_key_t ERTS_WRITE_UNLIKELY(erts_msacc_key); #else ErtsMsAcc *ERTS_WRITE_UNLIKELY(erts_msacc) = NULL; #endif +#ifndef ERTS_MSACC_ALWAYS_ON int ERTS_WRITE_UNLIKELY(erts_msacc_enabled); +#endif static Eterm *erts_msacc_state_atoms = NULL; static erts_rwmtx_t msacc_mutex; @@ -62,6 +65,12 @@ static ErtsMsAcc *msacc_unmanaged = NULL; static Uint msacc_unmanaged_count = 0; #endif +#if ERTS_MSACC_STATE_COUNT < MAP_SMALL_MAP_LIMIT +#define DEFAULT_MSACC_MSG_SIZE (3 + 1 + ERTS_MSACC_STATE_COUNT * 2 + 3 + REF_THING_SIZE) +#else +#define DEFAULT_MSACC_MSG_SIZE (3 + ERTS_MSACC_STATE_COUNT * 3 + 3 + REF_THING_SIZE) +#endif + /* we have to split initiation as atoms are not inited in early init */ void erts_msacc_early_init(void) { #ifndef ERTS_MSACC_ALWAYS_ON @@ -88,7 +97,8 @@ void erts_msacc_init(void) { void erts_msacc_init_thread(char *type, int id, int managed) { ErtsMsAcc *msacc; - msacc = erts_alloc(ERTS_ALC_T_MSACC, sizeof(ErtsMsAcc)); + msacc = erts_alloc(ERTS_ALC_T_MSACC, sizeof(ErtsMsAcc) + + sizeof(ErtsMsAccPerfCntr) * ERTS_MSACC_STATE_COUNT); msacc->type = strdup(type); msacc->id = make_small(id); @@ -122,79 +132,80 @@ void erts_msacc_init_thread(char *type, int id, int managed) { #endif } +#ifdef ERTS_MSACC_EXTENDED_STATES + +void erts_msacc_set_bif_state(ErtsMsAcc *__erts_msacc_cache, Eterm mod, void *fn) { + +#ifdef ERTS_MSACC_EXTENDED_BIFS +#define BIF_LIST(Mod,Func,Arity,FuncAddr,Num) \ + if (fn == &FuncAddr) { \ + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATIC_STATE_COUNT + Num); \ + } else +#include "erl_bif_list.h" +#undef BIF_LIST + { /* The last else in the macro expansion, + this happens for internal bifs, i.e. traps etc */ + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_BIF); + } +#else + if (mod == am_ets) { + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS); + } else { + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_BIF); + } +#endif +} + +#endif + /* * Creates a structure looking like this * #{ type => scheduler, id => 1, counters => #{ State1 => Counter1 ... StateN => CounterN}} */ static -Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, Eterm **hpp, Uint *szp) { +Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, ErtsHeapFactory *factory) { + Uint sz = 0; + Eterm *hp, cvs[ERTS_MSACC_STATE_COUNT]; + Eterm key, state_map; int i; - Eterm *hp; - Eterm key, state_key, state_map; - Eterm res = THE_NON_VALUE; - flatmap_t *map; - - if (szp) { - *szp += MAP_HEADER_FLATMAP_SZ + 1 + 2*(3); - *szp += MAP_HEADER_FLATMAP_SZ + 1 + 2*(ERTS_MSACC_STATE_COUNT); - for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) { - (void)erts_bld_sint64(NULL,szp,(Sint64)msacc->perf_counters[i]); + flatmap_t *map; + + hp = erts_produce_heap(factory, 4, 0); + key = TUPLE3(hp,am_counters,am_id,am_type); + + for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) { + cvs[i] = erts_bld_sint64(NULL, &sz,(Sint64)msacc->counters[i].pc); #ifdef ERTS_MSACC_STATE_COUNTERS - (void)erts_bld_uint64(NULL,szp,msacc->state_counters[i]); - *szp += 3; /* tuple to put state+perf counter in */ + erts_bld_uint64(NULL,&sz,msacc->counters[i].sc); + sz += 3; #endif - } } - if (hpp) { - Eterm counters[ERTS_MSACC_STATE_COUNT]; - hp = *hpp; - for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) { - Eterm counter = erts_bld_sint64(&hp,NULL,(Sint64)msacc->perf_counters[i]); + hp = erts_produce_heap(factory, sz, 0); + + for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) { + cvs[i] = erts_bld_sint64(&hp,NULL,(Sint64)msacc->counters[i].pc); #ifdef ERTS_MSACC_STATE_COUNTERS - Eterm counter__ = erts_bld_uint64(&hp,NULL,msacc->state_counters[i]); - counters[i] = TUPLE2(hp,counter,counter__); - hp += 3; -#else - counters[i] = counter; + Eterm counter__ = erts_bld_uint64(&hp,NULL,msacc->counters[i].sc); + cvs[i] = TUPLE2(hp,cvs[i],counter__); + hp += 3; #endif - } - - key = TUPLE3(hp,am_counters,am_id,am_type); - hp += 4; - - state_key = make_tuple(hp); - hp[0] = make_arityval(ERTS_MSACC_STATE_COUNT); - - for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) - hp[1+i] = erts_msacc_state_atoms[i]; - hp += 1 + ERTS_MSACC_STATE_COUNT; - - map = (flatmap_t*)hp; - hp += MAP_HEADER_FLATMAP_SZ; - map->thing_word = MAP_HEADER_FLATMAP; - map->size = ERTS_MSACC_STATE_COUNT; - map->keys = state_key; - for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) - hp[i] = counters[i]; - hp += ERTS_MSACC_STATE_COUNT; - state_map = make_flatmap(map); - - map = (flatmap_t*)hp; - hp += MAP_HEADER_FLATMAP_SZ; - map->thing_word = MAP_HEADER_FLATMAP; - map->size = 3; - map->keys = key; - hp[0] = state_map; - hp[1] = msacc->id; - hp[2] = am_atom_put(msacc->type,strlen(msacc->type)); - hp += 3; - - *hpp = hp; - res = make_flatmap(map); } - return res; + state_map = erts_map_from_ks_and_vs(factory, erts_msacc_state_atoms, cvs, + ERTS_MSACC_STATE_COUNT); + + hp = erts_produce_heap(factory, MAP_HEADER_FLATMAP_SZ + 3, 0); + map = (flatmap_t*)hp; + hp += MAP_HEADER_FLATMAP_SZ; + map->thing_word = MAP_HEADER_FLATMAP; + map->size = 3; + map->keys = key; + hp[0] = state_map; + hp[1] = msacc->id; + hp[2] = am_atom_put(msacc->type,strlen(msacc->type)); + + return make_flatmap(map); } typedef struct { @@ -222,40 +233,31 @@ static void send_reply(ErtsMsAcc *msacc, ErtsMSAccReq *msaccrp) { ErtsSchedulerData *esdp = erts_get_scheduler_data(); Process *rp = msaccrp->proc; ErtsMessage *msgp = NULL; - Eterm **hpp, *hp; + Eterm *hp; Eterm ref_copy = NIL, msg; - Uint sz, *szp; - ErlOffHeap *ohp = NULL; ErtsProcLocks rp_locks = (esdp && msaccrp->req_sched == esdp->no ? ERTS_PROC_LOCK_MAIN : 0); + ErtsHeapFactory factory; - sz = 0; - hpp = NULL; - szp = &sz; - - if (msacc->unmanaged) erts_mtx_lock(&msacc->mtx); - - while (1) { - if (hpp) - ref_copy = STORE_NC(hpp, ohp, msaccrp->ref); - else - *szp += REF_THING_SIZE; - - if (msaccrp->action != ERTS_MSACC_GATHER) - msg = ref_copy; - else { - msg = erts_msacc_gather_stats(msacc, hpp, szp); - msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg); - } - if (hpp) - break; - - msgp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); - hpp = &hp; - szp = NULL; - } + if (msaccrp->action == ERTS_MSACC_GATHER) { + + msgp = erts_factory_message_create(&factory, rp, &rp_locks, DEFAULT_MSACC_MSG_SIZE); + + if (msacc->unmanaged) erts_mtx_lock(&msacc->mtx); - if (msacc->unmanaged) erts_mtx_unlock(&msacc->mtx); + hp = erts_produce_heap(&factory, REF_THING_SIZE + 3 /* tuple */, 0); + ref_copy = STORE_NC(&hp, &msgp->hfrag.off_heap, msaccrp->ref); + msg = erts_msacc_gather_stats(msacc, &factory); + msg = TUPLE2(hp, ref_copy, msg); + + if (msacc->unmanaged) erts_mtx_unlock(&msacc->mtx); + + erts_factory_close(&factory); + } else { + ErlOffHeap *ohp = NULL; + msgp = erts_alloc_message_heap(rp, &rp_locks, REF_THING_SIZE, &hp, &ohp); + msg = STORE_NC(&hp, &msgp->hfrag.off_heap, msaccrp->ref); + } erts_queue_message(rp, rp_locks, msgp, msg, am_system); @@ -308,9 +310,9 @@ static void erts_msacc_reset(ErtsMsAcc *msacc) { if (msacc->unmanaged) erts_mtx_lock(&msacc->mtx); for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) { - msacc->perf_counters[i] = 0; + msacc->counters[i].pc = 0; #ifdef ERTS_MSACC_STATE_COUNTERS - msacc->state_counters[i] = 0; + msacc->counters[i].sc = 0; #endif } @@ -415,7 +417,7 @@ erts_msacc_request(Process *c_p, int action, Eterm *threads) ErtsSysPerfCounter perf_counter; /* if enabled update stats */ perf_counter = erts_sys_perf_counter(); - unmanaged[i]->perf_counters[unmanaged[i]->state] += + unmanaged[i]->counters[unmanaged[i]->state].pc += perf_counter - unmanaged[i]->perf_counter; unmanaged[i]->perf_counter = perf_counter; } @@ -454,7 +456,7 @@ erts_msacc_request(Process *c_p, int action, Eterm *threads) for (msacc = msacc_unmanaged; msacc != NULL; msacc = msacc->next) { erts_mtx_lock(&msacc->mtx); perf_counter = erts_sys_perf_counter(); - msacc->perf_counters[msacc->state] += perf_counter - msacc->perf_counter; + msacc->counters[msacc->state].pc += perf_counter - msacc->perf_counter; msacc->perf_counter = 0; erts_mtx_unlock(&msacc->mtx); } diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h index 284388f7aa..96d7d227bc 100644 --- a/erts/emulator/beam/erl_msacc.h +++ b/erts/emulator/beam/erl_msacc.h @@ -35,6 +35,10 @@ this reduces overhead a little bit when profiling */ /* #define ERTS_MSACC_ALWAYS_ON 1 */ +/* Uncomment this to keep individual stats for all + of the bifs when extended states is enabled */ +/* #define ERTS_MSACC_EXTENDED_BIFS 1 */ + #define ERTS_MSACC_DISABLE 0 #define ERTS_MSACC_ENABLE 1 #define ERTS_MSACC_RESET 2 @@ -92,7 +96,13 @@ static char *erts_msacc_states[] = { #define ERTS_MSACC_STATE_SLEEP 13 #define ERTS_MSACC_STATE_TIMERS 14 -#define ERTS_MSACC_STATE_COUNT 15 +#define ERTS_MSACC_STATIC_STATE_COUNT 15 + +#ifdef ERTS_MSACC_EXTENDED_BIFS +#define ERTS_MSACC_STATE_COUNT (ERTS_MSACC_STATIC_STATE_COUNT + BIF_SIZE) +#else +#define ERTS_MSACC_STATE_COUNT ERTS_MSACC_STATIC_STATE_COUNT +#endif #if ERTS_MSACC_STATE_STRINGS static char *erts_msacc_states[] = { @@ -111,22 +121,26 @@ static char *erts_msacc_states[] = { "send", "sleep", "timers" +#ifdef ERTS_MSACC_EXTENDED_BIFS +#define BIF_LIST(Mod,Func,Arity,FuncAddr,Num) \ + ,"bif_" #Mod "_" #Func "_" #Arity +#include "erl_bif_list.h" +#undef BIF_LIST +#endif }; #endif #endif typedef struct erl_msacc_t_ ErtsMsAcc; - -struct erl_msacc_t_ { - - /* the the values below are protected by mtx iff unmanaged = 1 */ - ErtsSysPerfCounter perf_counter; - ErtsSysPerfCounter perf_counters[ERTS_MSACC_STATE_COUNT]; +typedef struct erl_msacc_p_cnt_t_ { + ErtsSysPerfCounter pc; #ifdef ERTS_MSACC_STATE_COUNTERS - Uint64 state_counters[ERTS_MSACC_STATE_COUNT]; + Uint64 sc; #endif - Uint state; +} ErtsMsAccPerfCntr; + +struct erl_msacc_t_ { /* protected by msacc_mutex in erl_msacc.c, and should be constant */ int unmanaged; @@ -135,6 +149,12 @@ struct erl_msacc_t_ { erts_tid_t tid; Eterm id; char *type; + + /* the the values below are protected by mtx iff unmanaged = 1 */ + ErtsSysPerfCounter perf_counter; + Uint state; + ErtsMsAccPerfCntr counters[]; + }; #if ERTS_ENABLE_MSACC @@ -334,9 +354,9 @@ void erts_msacc_set_state_m__(ErtsMsAcc *msacc, Uint new_state, int increment) { msacc->perf_counter = erts_sys_perf_counter(); diff = msacc->perf_counter - prev_perf_counter; ASSERT(diff >= 0); - msacc->perf_counters[msacc->state] += diff; + msacc->counters[msacc->state].pc += diff; #ifdef ERTS_MSACC_STATE_COUNTERS - msacc->state_counters[new_state] += increment; + msacc->counters[new_state].sc += increment; #endif msacc->state = new_state; } @@ -364,7 +384,7 @@ void erts_msacc_set_state_m__(ErtsMsAcc *msacc, Uint new_state, int increment) { #define ERTS_MSACC_SET_STATE_CACHED_M(state) #define ERTS_MSACC_POP_STATE_M() #define ERTS_MSACC_PUSH_AND_SET_STATE_M(state) - +#define ERTS_MSACC_SET_BIF_STATE_CACHED_X(Mod,Addr) #endif /* ERTS_ENABLE_MSACC */ @@ -385,9 +405,13 @@ void erts_msacc_set_state_m__(ErtsMsAcc *msacc, Uint new_state, int increment) { #define ERTS_MSACC_SET_STATE_CACHED_M_X(state) #define ERTS_MSACC_POP_STATE_M_X() #define ERTS_MSACC_PUSH_AND_SET_STATE_M_X(state) +#define ERTS_MSACC_PUSH_AND_SET_STATE_CACHED_M_X(state) +#define ERTS_MSACC_SET_BIF_STATE_CACHED_X(Mod,Addr) #else +void erts_msacc_set_bif_state(ErtsMsAcc *msacc, Eterm mod, void *addr); + #define ERTS_MSACC_PUSH_STATE_X() ERTS_MSACC_PUSH_STATE() #define ERTS_MSACC_POP_STATE_X() ERTS_MSACC_POP_STATE() #define ERTS_MSACC_SET_STATE_X(state) ERTS_MSACC_SET_STATE(state) @@ -403,6 +427,9 @@ void erts_msacc_set_state_m__(ErtsMsAcc *msacc, Uint new_state, int increment) { #define ERTS_MSACC_SET_STATE_CACHED_M_X(state) ERTS_MSACC_SET_STATE_CACHED_M(state) #define ERTS_MSACC_POP_STATE_M_X() ERTS_MSACC_POP_STATE_M() #define ERTS_MSACC_PUSH_AND_SET_STATE_M_X(state) ERTS_MSACC_PUSH_AND_SET_STATE_M(state) +#define ERTS_MSACC_SET_BIF_STATE_CACHED_X(Mod,Addr) \ + if (ERTS_MSACC_IS_ENABLED_CACHED_X()) \ + erts_msacc_set_bif_state(__erts_msacc_cache, Mod, Addr) #endif /* !ERTS_MSACC_EXTENDED_STATES */ -- cgit v1.2.3 From 336f9be03b7d38223db3866ecbde948fbf393a41 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Tue, 21 Jun 2016 17:52:43 +0200 Subject: erts: Fix some msacc inline directives --- erts/emulator/beam/erl_msacc.h | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h index 96d7d227bc..ad7c8c5eee 100644 --- a/erts/emulator/beam/erl_msacc.h +++ b/erts/emulator/beam/erl_msacc.h @@ -159,8 +159,6 @@ struct erl_msacc_t_ { #if ERTS_ENABLE_MSACC -#define ERTS_MSACC_INLINE ERTS_GLB_INLINE - #ifdef USE_THREADS extern erts_tsd_key_t erts_msacc_key; #else @@ -296,20 +294,20 @@ void erts_msacc_init_thread(char *type, int id, int liberty); #define ERTS_MSACC_PUSH_AND_SET_STATE_M(state) \ ERTS_MSACC_PUSH_STATE_M(); ERTS_MSACC_SET_STATE_CACHED_M(state) -ERTS_MSACC_INLINE +ERTS_GLB_INLINE void erts_msacc_set_state_um__(ErtsMsAcc *msacc,Uint state,int increment); -ERTS_MSACC_INLINE +ERTS_GLB_INLINE void erts_msacc_set_state_m__(ErtsMsAcc *msacc,Uint state,int increment); -ERTS_MSACC_INLINE +ERTS_GLB_INLINE Uint erts_msacc_get_state_um__(ErtsMsAcc *msacc); -ERTS_MSACC_INLINE +ERTS_GLB_INLINE Uint erts_msacc_get_state_m__(ErtsMsAcc *msacc); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_MSACC_INLINE +ERTS_GLB_INLINE Uint erts_msacc_get_state_um__(ErtsMsAcc *msacc) { Uint state; if (msacc->unmanaged) @@ -320,12 +318,12 @@ Uint erts_msacc_get_state_um__(ErtsMsAcc *msacc) { return state; } -ERTS_MSACC_INLINE +ERTS_GLB_INLINE Uint erts_msacc_get_state_m__(ErtsMsAcc *msacc) { return msacc->state; } -ERTS_MSACC_INLINE +ERTS_GLB_INLINE void erts_msacc_set_state_um__(ErtsMsAcc *msacc, Uint new_state, int increment) { if (ERTS_UNLIKELY(msacc->unmanaged)) { erts_mtx_lock(&msacc->mtx); @@ -342,7 +340,7 @@ void erts_msacc_set_state_um__(ErtsMsAcc *msacc, Uint new_state, int increment) erts_mtx_unlock(&msacc->mtx); } -ERTS_MSACC_INLINE +ERTS_GLB_INLINE void erts_msacc_set_state_m__(ErtsMsAcc *msacc, Uint new_state, int increment) { ErtsSysPerfCounter prev_perf_counter; Sint64 diff; -- cgit v1.2.3 From ee03460053a40df8358679d21adaa33c40b243fb Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Tue, 21 Jun 2016 18:04:34 +0200 Subject: erts: Cleanup a bunch of un-neccesary #ifndefs --- erts/emulator/beam/atom.h | 3 --- erts/emulator/beam/big.h | 9 --------- erts/emulator/beam/export.h | 6 ------ erts/emulator/beam/hash.h | 2 -- erts/emulator/beam/index.h | 5 ----- erts/emulator/beam/module.h | 2 -- erts/emulator/beam/safe_hash.h | 5 ----- 7 files changed, 32 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h index fbd0528009..ae60904785 100644 --- a/erts/emulator/beam/atom.h +++ b/erts/emulator/beam/atom.h @@ -21,10 +21,7 @@ #ifndef __ATOM_H__ #define __ATOM_H__ -#ifndef __INDEX_H__ #include "index.h" -#endif - #include "erl_atom_table.h" #define MAX_ATOM_CHARACTERS 255 diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index 464acd67f6..4a96d971c3 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -21,17 +21,8 @@ #ifndef __BIG_H__ #define __BIG_H__ -#ifndef __SYS_H__ #include "sys.h" -#endif - -#ifndef __CONFIG_H__ -#include "erl_vm.h" -#endif - -#ifndef __GLOBAL_H__ #include "global.h" -#endif typedef Uint ErtsDigit; diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h index 8c81cbd410..1e7bb8514b 100644 --- a/erts/emulator/beam/export.h +++ b/erts/emulator/beam/export.h @@ -21,14 +21,8 @@ #ifndef __EXPORT_H__ #define __EXPORT_H__ -#ifndef __SYS_H__ #include "sys.h" -#endif - -#ifndef __INDEX_H__ #include "index.h" -#endif - #include "code_ix.h" /* diff --git a/erts/emulator/beam/hash.h b/erts/emulator/beam/hash.h index 9f773d8faa..4e769c0119 100644 --- a/erts/emulator/beam/hash.h +++ b/erts/emulator/beam/hash.h @@ -25,9 +25,7 @@ #ifndef __HASH_H__ #define __HASH_H__ -#ifndef __SYS_H__ #include "sys.h" -#endif typedef unsigned long HashValue; typedef struct hash Hash; diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h index 218779c33b..0a109d8699 100644 --- a/erts/emulator/beam/index.h +++ b/erts/emulator/beam/index.h @@ -26,13 +26,8 @@ #ifndef __INDEX_H__ #define __INDEX_H__ -#ifndef __HASH_H__ #include "hash.h" -#endif - -#ifndef ERL_ALLOC_H__ #include "erl_alloc.h" -#endif typedef struct index_slot { diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 4e12731d85..5a60bc90d9 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -21,9 +21,7 @@ #ifndef __MODULE_H__ #define __MODULE_H__ -#ifndef __INDEX_H__ #include "index.h" -#endif struct erl_module_instance { BeamCodeHeader* code_hdr; diff --git a/erts/emulator/beam/safe_hash.h b/erts/emulator/beam/safe_hash.h index 6910b33004..285103cb17 100644 --- a/erts/emulator/beam/safe_hash.h +++ b/erts/emulator/beam/safe_hash.h @@ -26,14 +26,9 @@ #ifndef __SAFE_HASH_H__ #define __SAFE_HASH_H__ - -#ifndef __SYS_H__ #include "sys.h" -#endif - #include "erl_alloc.h" - typedef unsigned long SafeHashValue; typedef int (*SHCMP_FUN)(void*, void*); -- cgit v1.2.3 From 2c8a13e5d3aabeedeb290624d6acc02d25db1bd5 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 20 Jul 2016 15:48:27 +0200 Subject: Fix dirty schedulers build on windows --- erts/emulator/beam/beam_emu.c | 19 +++++++------------ erts/emulator/beam/erl_process.c | 8 ++++++++ erts/emulator/beam/erl_process.h | 29 +++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 12 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index c6258287c4..626d99f233 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -5261,19 +5261,14 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) else { /* * Dirty CPU scheduler: - * Currently two reductions consumed per - * micro second spent in the dirty NIF. + * Reductions based on time consumed by + * the dirty NIF. */ - ErtsMonotonicTime time; - time = erts_get_monotonic_time(esdp); - time -= start_time; - time = ERTS_MONOTONIC_TO_USEC(time); - time *= (CONTEXT_REDS-1)/1000 + 1; - ASSERT(time >= 0); - if (time == 0) - time = 1; /* At least one reduction */ - time += esdp->virtual_reds; - reds_used = time > INT_MAX ? INT_MAX : (int) time; + Sint64 treds; + treds = erts_time2reds(start_time, + erts_get_monotonic_time(esdp)); + treds += esdp->virtual_reds; + reds_used = treds > INT_MAX ? INT_MAX : (int) treds; } PROCESS_MAIN_CHK_LOCKS(c_p); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 66f22979ad..1ee0cbbcf8 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -9364,6 +9364,14 @@ erts_set_process_priority(Process *p, Eterm value) } } +#ifdef __WIN32__ +Sint64 +erts_time2reds(ErtsMonotonicTime start, ErtsMonotonicTime end) +{ + return ERTS_TIME2REDS_IMPL__(start, end); +} +#endif + static int scheduler_gc_proc(Process *c_p, int reds_left) { diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 7c98b60647..d134fe913f 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -2474,6 +2474,35 @@ erts_get_atom_cache_map(Process *c_p) } #endif +#ifdef __WIN32__ +/* + * Don't want erts_time2reds() inlined in beam_emu.c on windows since + * it is compiled with gcc which fails on it. Implementation is in + * erl_process.c on windows. + */ +# define ERTS_TIME2REDS_IMPL__ erts_time2reds__ +#else +# define ERTS_TIME2REDS_IMPL__ erts_time2reds +#endif + +ERTS_GLB_INLINE Sint64 ERTS_TIME2REDS_IMPL__(ErtsMonotonicTime start, + ErtsMonotonicTime end); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE Sint64 +ERTS_TIME2REDS_IMPL__(ErtsMonotonicTime start, ErtsMonotonicTime end) +{ + ErtsMonotonicTime time = end - start; + ASSERT(time >= 0); + time = ERTS_MONOTONIC_TO_USEC(time); + if (time == 0) + return (Sint64) 1; /* At least one reduction */ + /* Currently two reductions per micro second */ + time *= (CONTEXT_REDS-1)/1000 + 1; + return (Sint64) time; +} +#endif + Process *erts_pid2proc_suspend(Process *, ErtsProcLocks, Eterm, -- cgit v1.2.3 From d51dea1f10128537112ebecbecbf2435b05cdc2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Thu, 21 Jul 2016 10:41:28 +0200 Subject: erts: Fix whitespaces --- erts/emulator/beam/hash.c | 21 ++++--- erts/emulator/beam/utils.c | 144 ++++++++++++++++++++++----------------------- 2 files changed, 81 insertions(+), 84 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c index e255b961f1..cd038d100b 100644 --- a/erts/emulator/beam/hash.c +++ b/erts/emulator/beam/hash.c @@ -35,9 +35,9 @@ static const int h_size_table[] = { 2, 5, 11, 23, 47, 97, 197, 397, 797, /* double upto here */ 1201, 1597, - 2411, 3203, + 2411, 3203, 4813, 6421, - 9643, 12853, + 9643, 12853, 19289, 25717, 51437, 102877, @@ -49,8 +49,8 @@ static const int h_size_table[] = { 6584983, 13169977, 26339969, - 52679969, - -1 + 52679969, + -1 }; /* @@ -69,7 +69,7 @@ void hash_get_info(HashInfo *hi, Hash *h) for (i = 0; i < size; i++) { int depth = 0; HashBucket* b = h->bucket[i]; - + while (b != (HashBucket*) 0) { objects++; depth++; @@ -112,7 +112,7 @@ void hash_info(int to, void *arg, Hash* h) /* * Returns size of table in bytes. Stored objects not included. */ -int +int hash_table_sz(Hash *h) { int i; @@ -190,7 +190,7 @@ void hash_delete(Hash* h) HashBucket* b = h->bucket[i]; while (b != (HashBucket*) 0) { HashBucket* b_next = b->next; - + h->fun.free((void*) b); b = b_next; } @@ -250,7 +250,7 @@ void* hash_get(Hash* h, void* tmpl) HashValue hval = h->fun.hash(tmpl); int ix = hval % h->size; HashBucket* b = h->bucket[ix]; - + while(b != (HashBucket*) 0) { if ((b->hvalue == hval) && (h->fun.cmp(tmpl, (void*)b) == 0)) return (void*) b; @@ -294,7 +294,7 @@ void* hash_erase(Hash* h, void* tmpl) int ix = hval % h->size; HashBucket* b = h->bucket[ix]; HashBucket* prev = 0; - + while(b != 0) { if ((b->hvalue == hval) && (h->fun.cmp(tmpl, (void*)b) == 0)) { if (prev != 0) @@ -326,7 +326,7 @@ hash_remove(Hash *h, void *tmpl) int ix = hval % h->size; HashBucket *b = h->bucket[ix]; HashBucket *prev = NULL; - + while (b) { if ((b->hvalue == hval) && (h->fun.cmp(tmpl, (void*)b) == 0)) { if (prev) @@ -355,4 +355,3 @@ void hash_foreach(Hash* h, void (*func)(void *, void *), void *func_arg2) } } } - diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 675fafa726..4dc724605b 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -85,7 +85,7 @@ erts_heap_alloc(Process* p, Uint need, Uint xtra) && HEAP_TOP(p) >= p->space_verified_from && HEAP_TOP(p) + need <= p->space_verified_from + p->space_verified && HEAP_LIMIT(p) - HEAP_TOP(p) >= need) { - + Uint consumed = need + (HEAP_TOP(p) - p->space_verified_from); ASSERT(consumed <= p->space_verified); p->space_verified -= consumed; @@ -638,7 +638,7 @@ erts_bld_atom_uword_2tup_list(Uint **hpp, Uint *szp, ui = uint_to_big(uints[i], *hpp); *hpp += BIG_UINT_HEAP_SIZE; } - + res = CONS(*hpp+3, TUPLE2(*hpp, atoms[i], ui), res); *hpp += 5; } @@ -676,14 +676,14 @@ erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length, ui1 = uint_to_big(uints1[i], *hpp); *hpp += BIG_UINT_HEAP_SIZE; } - + if (IS_USMALL(0, uints2[i])) ui2 = make_small(uints2[i]); else { ui2 = uint_to_big(uints2[i], *hpp); *hpp += BIG_UINT_HEAP_SIZE; } - + res = CONS(*hpp+4, TUPLE3(*hpp, atoms[i], ui1, ui2), res); *hpp += 6; } @@ -794,7 +794,7 @@ hash_binary_bytes(Eterm bin, Uint sz, Uint32 hash) Uint b; Uint lshift = bitoffs; Uint rshift = 8 - lshift; - + while (sz--) { b = (previous << lshift) & 0xFF; previous = *ptr++; @@ -805,7 +805,7 @@ hash_binary_bytes(Eterm bin, Uint sz, Uint32 hash) b = (previous << lshift) & 0xFF; previous = *ptr++; b |= previous >> rshift; - + b >>= 8 - bitsize; hash = (hash*FUNNY_NUMBER1 + b) * FUNNY_NUMBER12 + bitsize; } @@ -835,21 +835,21 @@ Uint32 make_hash(Eterm term_arg) do { \ Uint32 x = (Uint32) (Expr); \ hash = \ - (((((hash)*(Prime1) + (x & 0xFF)) * (Prime1) + \ - ((x >> 8) & 0xFF)) * (Prime1) + \ - ((x >> 16) & 0xFF)) * (Prime1) + \ + (((((hash)*(Prime1) + (x & 0xFF)) * (Prime1) + \ + ((x >> 8) & 0xFF)) * (Prime1) + \ + ((x >> 16) & 0xFF)) * (Prime1) + \ (x >> 24)); \ } while(0) -#define UINT32_HASH_RET(Expr, Prime1, Prime2) \ +#define UINT32_HASH_RET(Expr, Prime1, Prime2) \ UINT32_HASH_STEP(Expr, Prime1); \ hash = hash * (Prime2); \ - break - - + break + + /* * Significant additions needed for real 64 bit port with larger fixnums. - */ + */ /* * Note, for the simple 64bit port, not utilizing the @@ -864,7 +864,7 @@ tail_recur: hash = hash*FUNNY_NUMBER3 + 1; break; case ATOM_DEF: - hash = hash*FUNNY_NUMBER1 + + hash = hash*FUNNY_NUMBER1 + (atom_tab(atom_val(term))->slot.bucket.hvalue); break; case SMALL_DEF: @@ -893,9 +893,9 @@ tail_recur: Export* ep = *((Export **) (export_val(term) + 1)); hash = hash * FUNNY_NUMBER11 + ep->code[2]; - hash = hash*FUNNY_NUMBER1 + + hash = hash*FUNNY_NUMBER1 + (atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue); - hash = hash*FUNNY_NUMBER1 + + hash = hash*FUNNY_NUMBER1 + (atom_tab(atom_val(ep->code[1]))->slot.bucket.hvalue); break; } @@ -906,7 +906,7 @@ tail_recur: Uint num_free = funp->num_free; hash = hash * FUNNY_NUMBER10 + num_free; - hash = hash*FUNNY_NUMBER1 + + hash = hash*FUNNY_NUMBER1 + (atom_tab(atom_val(funp->fe->module))->slot.bucket.hvalue); hash = hash*FUNNY_NUMBER2 + funp->fe->old_index; hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq; @@ -931,7 +931,7 @@ tail_recur: UINT32_HASH_RET(internal_ref_numbers(term)[0],FUNNY_NUMBER9,FUNNY_NUMBER10); case EXTERNAL_REF_DEF: UINT32_HASH_RET(external_ref_numbers(term)[0],FUNNY_NUMBER9,FUNNY_NUMBER10); - case FLOAT_DEF: + case FLOAT_DEF: { FloatDef ff; GET_DOUBLE(term, ff); @@ -958,12 +958,12 @@ tail_recur: ** as multiplications on a Sparc is so slow. */ hash = hash*FUNNY_NUMBER2 + unsigned_val(*list); - + if (is_not_list(CDR(list))) { WSTACK_PUSH(stack, MAKE_HASH_CDR_POST_OP); term = CDR(list); goto tail_recur; - } + } list = list_val(CDR(list)); } WSTACK_PUSH2(stack, CDR(list), MAKE_HASH_CDR_PRE_OP); @@ -1004,17 +1004,17 @@ tail_recur: } hash *= is_neg ? FUNNY_NUMBER4 : FUNNY_NUMBER3; break; - } + } case MAP_DEF: hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term); break; - case TUPLE_DEF: + case TUPLE_DEF: { Eterm* ptr = tuple_val(term); Uint arity = arityval(*ptr); WSTACK_PUSH3(stack, (UWord) arity, (UWord)(ptr+1), (UWord) arity); - op = MAKE_HASH_TUPLE_OP; + op = MAKE_HASH_TUPLE_OP; }/*fall through*/ case MAKE_HASH_TUPLE_OP: case MAKE_HASH_TERM_ARRAY_OP: @@ -1031,8 +1031,8 @@ tail_recur: hash = hash*FUNNY_NUMBER9 + arity; } break; - } - + } + default: erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op); return 0; @@ -1159,8 +1159,8 @@ make_hash2(Eterm term) if (y < 0) { \ UINT32_HASH(-y, AConst); \ /* Negative numbers are unnecessarily mixed twice. */ \ - } \ - UINT32_HASH(y, AConst); \ + } \ + UINT32_HASH(y, AConst); \ } while(0) #define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2) @@ -1242,7 +1242,7 @@ make_hash2(Eterm term) int arity = header_arity(hdr); Eterm* elem = tuple_val(term); UINT32_HASH(arity, HCONST_9); - if (arity == 0) /* Empty tuple */ + if (arity == 0) /* Empty tuple */ goto hash2_common; for (i = arity; ; i--) { term = elem[i]; @@ -1329,7 +1329,7 @@ make_hash2(Eterm term) { Export* ep = *((Export **) (export_val(term) + 1)); UINT32_HASH_2 - (ep->code[2], + (ep->code[2], atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue, HCONST); UINT32_HASH @@ -1343,7 +1343,7 @@ make_hash2(Eterm term) ErlFunThing* funp = (ErlFunThing *) fun_val(term); Uint num_free = funp->num_free; UINT32_HASH_2 - (num_free, + (num_free, atom_tab(atom_val(funp->fe->module))->slot.bucket.hvalue, HCONST); UINT32_HASH_2 @@ -1468,7 +1468,7 @@ make_hash2(Eterm term) goto hash2_common; } break; - + default: erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); } @@ -1541,7 +1541,7 @@ make_hash2(Eterm term) } case HASH_MAP_PAIR: hash_xor_pairs ^= hash; - hash = 0; + hash = 0; goto hash2_common; default: break; @@ -1906,6 +1906,7 @@ make_internal_hash(Eterm term) pop_next: if (ESTACK_ISEMPTY(s)) { DESTROY_ESTACK(s); + return hash; } @@ -1953,8 +1954,8 @@ Uint32 make_broken_hash(Eterm term) DECLARE_WSTACK(stack); unsigned op; tail_recur: - op = tag_val_def(term); - for (;;) { + op = tag_val_def(term); + for (;;) { switch (op) { case NIL_DEF: hash = hash*FUNNY_NUMBER3 + 1; @@ -1976,8 +1977,7 @@ tail_recur: { /* like a bignum */ Uint32 y4 = (Uint32) y2; hash = hash*FUNNY_NUMBER2 + ((y4 << 16) | (y4 >> 16)); - if (y3) - { + if (y3) { hash = hash*FUNNY_NUMBER2 + ((y3 << 16) | (y3 >> 16)); arity++; } @@ -2020,9 +2020,9 @@ tail_recur: Export* ep = *((Export **) (export_val(term) + 1)); hash = hash * FUNNY_NUMBER11 + ep->code[2]; - hash = hash*FUNNY_NUMBER1 + + hash = hash*FUNNY_NUMBER1 + (atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue); - hash = hash*FUNNY_NUMBER1 + + hash = hash*FUNNY_NUMBER1 + (atom_tab(atom_val(ep->code[1]))->slot.bucket.hvalue); break; } @@ -2033,7 +2033,7 @@ tail_recur: Uint num_free = funp->num_free; hash = hash * FUNNY_NUMBER10 + num_free; - hash = hash*FUNNY_NUMBER1 + + hash = hash*FUNNY_NUMBER1 + (atom_tab(atom_val(funp->fe->module))->slot.bucket.hvalue); hash = hash*FUNNY_NUMBER2 + funp->fe->old_index; hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq; @@ -2065,7 +2065,7 @@ tail_recur: case EXTERNAL_REF_DEF: hash = hash*FUNNY_NUMBER9 + external_ref_numbers(term)[0]; break; - case FLOAT_DEF: + case FLOAT_DEF: { FloatDef ff; GET_DOUBLE(term, ff); @@ -2149,7 +2149,7 @@ tail_recur: } #else -#error "unsupported D_EXP size" +#error "unsupported D_EXP size" #endif hash = hash * (is_neg ? FUNNY_NUMBER3 : FUNNY_NUMBER2) + arity; } @@ -2158,14 +2158,14 @@ tail_recur: case MAP_DEF: hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term); break; - case TUPLE_DEF: + case TUPLE_DEF: { Eterm* ptr = tuple_val(term); Uint arity = arityval(*ptr); WSTACK_PUSH3(stack, (UWord) arity, (UWord) (ptr+1), (UWord) arity); op = MAKE_HASH_TUPLE_OP; - }/*fall through*/ + }/*fall through*/ case MAKE_HASH_TUPLE_OP: case MAKE_HASH_TERM_ARRAY_OP: { @@ -2193,7 +2193,7 @@ tail_recur: DESTROY_WSTACK(stack); return hash; - + #undef MAKE_HASH_TUPLE_OP #undef MAKE_HASH_TERM_ARRAY_OP #undef MAKE_HASH_CDR_PRE_OP @@ -2353,13 +2353,13 @@ static int do_send_term_to_logger(Eterm tag, Eterm gleader, } static ERTS_INLINE int -send_info_to_logger(Eterm gleader, char *buf, int len) +send_info_to_logger(Eterm gleader, char *buf, int len) { return do_send_to_logger(am_info_msg, gleader, buf, len); } static ERTS_INLINE int -send_warning_to_logger(Eterm gleader, char *buf, int len) +send_warning_to_logger(Eterm gleader, char *buf, int len) { Eterm tag; switch (erts_error_logger_warnings) { @@ -2371,7 +2371,7 @@ send_warning_to_logger(Eterm gleader, char *buf, int len) } static ERTS_INLINE int -send_error_to_logger(Eterm gleader, char *buf, int len) +send_error_to_logger(Eterm gleader, char *buf, int len) { return do_send_to_logger(am_error, gleader, buf, len); } @@ -2613,7 +2613,7 @@ tailrecur_ne: break; /* not equal */ case TAG_PRIMARY_BOXED: - { + { Eterm hdr = *boxed_val(a); switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: @@ -2639,7 +2639,7 @@ tailrecur_ne: Uint b_bitsize; Uint a_bitoffs; Uint b_bitoffs; - + if (!is_binary(b)) { goto not_equal; } @@ -2671,7 +2671,7 @@ tailrecur_ne: { ErlFunThing* f1; ErlFunThing* f2; - + if (!is_fun(b)) goto not_equal; f1 = (ErlFunThing *) fun_val(a); @@ -2702,7 +2702,7 @@ tailrecur_ne: if(ap->header == bp->header && ap->node == bp->node) { ASSERT(1 == external_data_words(a)); ASSERT(1 == external_data_words(b)); - + if (ap->data.ui[0] == bp->data.ui[0]) goto pop_next; } break; /* not equal */ @@ -2759,7 +2759,7 @@ tailrecur_ne: if (alen == 3 && blen == 3) { /* Most refs are of length 3 */ if (anum[1] == bnum[1] && anum[2] == bnum[2]) { - goto pop_next; + goto pop_next; } else { goto not_equal; } @@ -2784,7 +2784,7 @@ tailrecur_ne: for (i = common_len; i < blen; i++) if (bnum[i] != 0) goto not_equal; - } + } } goto pop_next; } @@ -2792,7 +2792,7 @@ tailrecur_ne: case NEG_BIG_SUBTAG: { int i; - + if (!is_big(b)) goto not_equal; aa = big_val(a); @@ -2810,7 +2810,7 @@ tailrecur_ne: { FloatDef af; FloatDef bf; - + if (is_float(b)) { GET_DOUBLE(a, af); GET_DOUBLE(b, bf); @@ -2889,7 +2889,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'sz' */ } goto tailrecur_ne; } - + pop_next: if (!WSTACK_ISEMPTY(stack)) { UWord something = WSTACK_POP(stack); @@ -3093,7 +3093,7 @@ tailrecur_ne: } anode = erts_this_node; adata = internal_port_data(a); - + port_common: CMP_NODES(anode, bnode); ON_CMP_GOTO((Sint)(adata - bdata)); @@ -3111,7 +3111,7 @@ tailrecur_ne: } anode = erts_this_node; adata = internal_pid_data(a); - + pid_common: if (adata != bdata) { RETURN_NEQ(adata < bdata ? -1 : 1); @@ -3336,7 +3336,7 @@ tailrecur_ne: diff = f1->num_free - f2->num_free; if (diff != 0) { RETURN_NEQ(diff); - } + } i = f1->num_free; if (i == 0) goto pop_next; aa = f1->env; @@ -3397,10 +3397,10 @@ tailrecur_ne: anum = internal_thing_ref_numbers(athing); alen = internal_thing_ref_no_of_numbers(athing); } - + ref_common: CMP_NODES(anode, bnode); - + ASSERT(alen > 0 && blen > 0); if (alen != blen) { if (alen > blen) { @@ -3418,7 +3418,7 @@ tailrecur_ne: } while (alen < blen); } } - + ASSERT(alen == blen); for (i = (Sint) alen - 1; i >= 0; i--) if (anum[i] != bnum[i]) @@ -3633,8 +3633,8 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */ } a = *aa; b = *bb; - goto tailrecur; - + goto tailrecur; + pop_next: if (!WSTACK_ISEMPTY(stack)) { UWord something = WSTACK_POP(stack); @@ -3897,19 +3897,19 @@ intlist_to_buf(Eterm list, char *buf, Sint len) Eterm* listptr; Sint sz = 0; - if (is_nil(list)) + if (is_nil(list)) return 0; if (is_not_list(list)) return -1; listptr = list_val(list); while (sz < len) { - if (!is_byte(*listptr)) + if (!is_byte(*listptr)) return -1; buf[sz++] = unsigned_val(*listptr); if (is_nil(*(listptr + 1))) return(sz); - if (is_not_list(*(listptr + 1))) + if (is_not_list(*(listptr + 1))) return -1; listptr = list_val(*(listptr + 1)); } @@ -4157,10 +4157,10 @@ do { \ } else if (yield_support && --yield_count <= 0) goto L_yield; } - + res = len; - L_return: + L_return: DESTROY_ESTACK(s); @@ -5053,7 +5053,7 @@ Process *p; if(p) print_process_info(ERTS_PRINT_STDERR, NULL, p); } - + void ppi(Eterm pid) { pp(erts_proc_lookup(pid)); @@ -5079,5 +5079,3 @@ ps(Process* p, Eterm* stop) } } #endif - - -- cgit v1.2.3 From 249e3ec86de802232d17666a93665ae638cbc319 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Thu, 21 Jul 2016 15:57:15 +0200 Subject: erts: Fix internal hashing entropy for maps We need to use an initial hash seed for each map pair, i.e. the hash value. The hash value is reset to the seed for each pair instead of setting the seed to zero. If we don't, no additional entropy is given to the system and the hash collision resolution in maps:from_list/1 would fail. --- erts/emulator/beam/utils.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 4dc724605b..85647b8500 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1678,17 +1678,22 @@ make_internal_hash(Eterm term) * the order in which keys and values are encountered. * We therefore calculate context independent hashes for all . * key-value pairs and then xor them together. + * + * We *do* need to use an initial seed for each pair, i.e. the + * hash value, so the hash value is reset for each pair with 'hash'. + * If we don't, no additional entropy is given to the system and the + * hash collision resolution in maps:from_list/1 would fail. */ ESTACK_PUSH(s, hash_xor_pairs); ESTACK_PUSH(s, hash); ESTACK_PUSH(s, HASH_MAP_TAIL); - hash = 0; - hash_xor_pairs = 0; for (i = size - 1; i >= 0; i--) { + ESTACK_PUSH(s, hash); /* initial seed for all pairs */ ESTACK_PUSH(s, HASH_MAP_PAIR); ESTACK_PUSH(s, vs[i]); ESTACK_PUSH(s, ks[i]); } + hash_xor_pairs = 0; goto pop_next; } case HAMT_SUBTAG_HEAD_ARRAY: @@ -1700,7 +1705,6 @@ make_internal_hash(Eterm term) ESTACK_PUSH(s, hash_xor_pairs); ESTACK_PUSH(s, hash); ESTACK_PUSH(s, HASH_MAP_TAIL); - hash = 0; hash_xor_pairs = 0; } switch (hdr & _HEADER_MAP_SUBTAG_MASK) { @@ -1717,6 +1721,7 @@ make_internal_hash(Eterm term) while (i) { if (is_list(*ptr)) { Eterm* cons = list_val(*ptr); + ESTACK_PUSH(s, hash); /* initial seed for all pairs */ ESTACK_PUSH(s, HASH_MAP_PAIR); ESTACK_PUSH(s, CDR(cons)); ESTACK_PUSH(s, CAR(cons)); @@ -1921,7 +1926,7 @@ make_internal_hash(Eterm term) } case HASH_MAP_PAIR: hash_xor_pairs ^= hash; - hash = 0; + hash = (Uint32) ESTACK_POP(s); /* initial seed for all pairs */ goto pop_next; case HASH_CDR: -- cgit v1.2.3 From a2359927ca4e7c315e9849deec3375b1f2ae4170 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Tue, 26 Jul 2016 10:19:31 +0200 Subject: erts: Remove the need for copying of literals * Literals are not copied between processes for messages or spawn Increases performance of message sent and processes spawned when literals are involved in messages or arguments. --- erts/emulator/beam/copy.c | 80 ++++++++++++++++++++++++++++------------ erts/emulator/beam/erl_gc.c | 8 ---- erts/emulator/beam/erl_message.c | 19 ++++++---- erts/emulator/beam/erl_process.c | 10 +++-- erts/emulator/beam/global.h | 20 ++++++---- 5 files changed, 88 insertions(+), 49 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index ccc4cbad43..7d3777f37e 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -40,7 +40,8 @@ static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*, int); /* * Copy object "obj" to process p. */ -Eterm copy_object_x(Eterm obj, Process* to, Uint extra) { +Eterm copy_object_x(Eterm obj, Process* to, Uint extra) +{ if (!is_immed(obj)) { Uint size = size_object(obj); Eterm* hp = HAllocX(to, size, extra); @@ -70,7 +71,12 @@ Eterm copy_object_x(Eterm obj, Process* to, Uint extra) { * Return the "flat" size of the object. */ -Uint size_object(Eterm obj) +#define in_literal_purge_area(PTR) \ + (lit_purge_ptr && ( \ + (lit_purge_ptr <= (PTR) && \ + (PTR) < (lit_purge_ptr + lit_purge_sz)))) + +Uint size_object_x(Eterm obj, Eterm *lit_purge_ptr, Uint lit_purge_sz, Uint litopt) { Uint sum = 0; Eterm* ptr; @@ -78,7 +84,6 @@ Uint size_object(Eterm obj) #ifdef DEBUG Eterm mypid = erts_get_current_pid(); #endif - DECLARE_ESTACK(s); VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size_object %p\n", mypid, obj)); @@ -86,17 +91,25 @@ Uint size_object(Eterm obj) for (;;) { switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: - sum += 2; ptr = list_val(obj); + if (litopt && erts_is_literal(obj,ptr) && !in_literal_purge_area(ptr)) { + goto pop_next; + } + sum += 2; obj = *ptr++; if (!IS_CONST(obj)) { ESTACK_PUSH(s, obj); - } + } obj = *ptr; break; case TAG_PRIMARY_BOXED: { - Eterm hdr = *boxed_val(obj); + Eterm hdr; + ptr = boxed_val(obj); + if (litopt && erts_is_literal(obj,ptr) && !in_literal_purge_area(ptr)) { + goto pop_next; + } + hdr = *ptr; ASSERT(is_header(hdr)); switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: @@ -279,10 +292,6 @@ do { \ #define COUNT_OFF_HEAP (0) -#define IN_LITERAL_PURGE_AREA(info, ptr) \ - ((info)->range_ptr && ( \ - (info)->range_ptr <= (ptr) && \ - (ptr) < ((info)->range_ptr + (info)->range_sz))) /* * Return the real size of an object and find sharing information * This currently returns the same as erts_debug:size/1. @@ -599,7 +608,8 @@ cleanup: /* * Copy a structure to a heap. */ -Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz) +Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz, + Eterm *lit_purge_ptr, Uint lit_purge_sz, Uint litopt) { char* hstart; Uint hsize; @@ -651,7 +661,6 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint L_copy: while (hp != htop) { obj = *hp; - switch (primary_tag(obj)) { case TAG_PRIMARY_IMMED1: hp++; @@ -667,6 +676,10 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint L_copy_list: tailp = argp; + if (litopt && erts_is_literal(obj,objp) && !in_literal_purge_area(objp)) { + *tailp = obj; + goto L_copy; + } for (;;) { tp = tailp; elem = CAR(objp); @@ -674,18 +687,23 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint hbot -= 2; CAR(hbot) = elem; tailp = &CDR(hbot); - } - else { + } else { CAR(htop) = elem; tailp = &CDR(htop); htop += 2; } *tp = make_list(tailp - 1); obj = CDR(objp); + if (!is_list(obj)) { break; } objp = list_val(obj); + + if (litopt && erts_is_literal(obj,objp) && !in_literal_purge_area(objp)) { + *tailp = obj; + goto L_copy; + } } switch (primary_tag(obj)) { case TAG_PRIMARY_IMMED1: *tailp = obj; goto L_copy; @@ -695,7 +713,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint "%s, line %d: Internal error in copy_struct: 0x%08x\n", __FILE__, __LINE__,obj); } - + case TAG_PRIMARY_BOXED: if (ErtsInArea(boxed_val(obj),hstart,hsize)) { hp++; @@ -705,6 +723,10 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint L_copy_boxed: objp = boxed_val(obj); + if (litopt && erts_is_literal(obj,objp) && !in_literal_purge_area(objp)) { + *argp = obj; + break; + } hdr = *objp; switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: @@ -765,7 +787,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint extra_bytes = 1; } else { extra_bytes = 0; - } + } real_size = size+extra_bytes; objp = binary_val(real_bin); if (thing_subtag(*objp) == HEAP_BINARY_SUBTAG) { @@ -780,7 +802,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint } else { ProcBin* from = (ProcBin *) objp; ProcBin* to; - + ASSERT(thing_subtag(*objp) == REFC_BINARY_SUBTAG); if (from->flags) { erts_emasculate_writable_binary(from); @@ -900,6 +922,12 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz = hend - hbot; } else { #ifdef DEBUG + if (!eq(org_obj, res)) { + erts_exit(ERTS_ABORT_EXIT, + "Internal error in copy_struct() when copying %T:" + " not equal to copy %T\n", + org_obj, res); + } if (htop != hbot) erts_exit(ERTS_ABORT_EXIT, "Internal error in copy_struct() when copying %T:" @@ -1036,6 +1064,8 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) Uint e; unsigned sz; Eterm* ptr; + Eterm *lit_purge_ptr = info->lit_purge_ptr; + Uint lit_purge_sz = info->lit_purge_sz; #ifdef DEBUG Eterm mypid = erts_get_current_pid(); #endif @@ -1081,7 +1111,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) /* off heap list pointers are copied verbatim */ if (erts_is_literal(obj,ptr)) { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj)); - if (IN_LITERAL_PURGE_AREA(info,ptr)) + if (in_literal_purge_area(ptr)) info->literal_size += size_object(obj); goto pop_next; } @@ -1132,7 +1162,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) /* off heap pointers to boxes are copied verbatim */ if (erts_is_literal(obj,ptr)) { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj)); - if (IN_LITERAL_PURGE_AREA(info,ptr)) + if (in_literal_purge_area(ptr)) info->literal_size += size_object(obj); goto pop_next; } @@ -1298,6 +1328,8 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, Eterm* resp; Eterm *hbot, *hend; unsigned remaining; + Eterm *lit_purge_ptr = info->lit_purge_ptr; + Uint lit_purge_sz = info->lit_purge_sz; #ifdef DEBUG Eterm mypid = erts_get_current_pid(); Eterm saved_obj = obj; @@ -1347,11 +1379,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, ptr = list_val(obj); /* off heap list pointers are copied verbatim */ if (erts_is_literal(obj,ptr)) { - if (!IN_LITERAL_PURGE_AREA(info,ptr)) { + if (!in_literal_purge_area(ptr)) { *resp = obj; } else { Uint bsz = 0; - *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL, 0, 0); /* copy literal */ hbot -= bsz; } goto cleanup_next; @@ -1415,11 +1447,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, ptr = boxed_val(obj); /* off heap pointers to boxes are copied verbatim */ if (erts_is_literal(obj,ptr)) { - if (!IN_LITERAL_PURGE_AREA(info,ptr)) { + if (!in_literal_purge_area(ptr)) { *resp = obj; } else { Uint bsz = 0; - *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL, 0, 0); /* copy literal */ hbot -= bsz; } goto cleanup_next; @@ -1923,7 +1955,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite if (is_header(val)) { struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp; ASSERT(ptr + header_arity(val) < end); - MOVE_BOXED(ptr, val, hp, &dummy_ref); + MOVE_BOXED(ptr, val, hp, &dummy_ref); switch (val & _HEADER_SUBTAG_MASK) { case REFC_BINARY_SUBTAG: case FUN_SUBTAG: diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index a224383493..22a7d430c8 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -2155,26 +2155,18 @@ copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap, *hp++ = val; break; case TAG_PRIMARY_LIST: -#ifdef SHCOPY_SEND if (erts_is_literal(val,list_val(val))) { *hp++ = val; } else { *hp++ = offset_ptr(val, offs); } -#else - *hp++ = offset_ptr(val, offs); -#endif break; case TAG_PRIMARY_BOXED: -#ifdef SHCOPY_SEND if (erts_is_literal(val,boxed_val(val))) { *hp++ = val; } else { *hp++ = offset_ptr(val, offs); } -#else - *hp++ = offset_ptr(val, offs); -#endif break; case TAG_PRIMARY_HEADER: *hp++ = val; diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 91e06cde2d..14ee07a304 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -696,6 +696,9 @@ erts_send_message(Process* sender, erts_aint32_t receiver_state; #ifdef SHCOPY_SEND erts_shcopy_t info; +#else + Eterm *lit_purge_ptr = erts_clrange.ptr; + Uint lit_purge_sz = erts_clrange.sz; #endif #ifdef USE_VM_PROBES @@ -725,7 +728,7 @@ erts_send_message(Process* sender, */ if (have_seqtrace(stoken)) { seq_trace_update_send(sender); - seq_trace_output(stoken, message, SEQ_TRACE_SEND, + seq_trace_output(stoken, message, SEQ_TRACE_SEND, receiver->common.id, sender); seq_trace_size = 6; /* TUPLE5 */ } @@ -741,7 +744,7 @@ erts_send_message(Process* sender, INITIALIZE_SHCOPY(info); msize = copy_shared_calculate(message, &info); #else - msize = size_object(message); + msize = size_object_litopt(message, lit_purge_ptr, lit_purge_sz); #endif mp = erts_alloc_message_heap_state(receiver, &receiver_state, @@ -760,7 +763,8 @@ erts_send_message(Process* sender, DESTROY_SHCOPY(info); #else if (is_not_immed(message)) - message = copy_struct(message, msize, &hp, ohp); + message = copy_struct_litopt(message, msize, &hp, ohp, + lit_purge_ptr, lit_purge_sz); #endif if (is_immed(stoken)) token = stoken; @@ -796,7 +800,7 @@ erts_send_message(Process* sender, INITIALIZE_SHCOPY(info); msize = copy_shared_calculate(message, &info); #else - msize = size_object(message); + msize = size_object_litopt(message, lit_purge_ptr, lit_purge_sz); #endif mp = erts_alloc_message_heap_state(receiver, &receiver_state, @@ -810,7 +814,8 @@ erts_send_message(Process* sender, DESTROY_SHCOPY(info); #else if (is_not_immed(message)) - message = copy_struct(message, msize, &hp, ohp); + message = copy_struct_litopt(message, msize, &hp, ohp, + lit_purge_ptr, lit_purge_sz); #endif } #ifdef USE_VM_PROBES @@ -998,8 +1003,8 @@ erts_move_messages_off_heap(Process *c_p) hp = hfrag->mem; if (is_not_immed(ERL_MESSAGE_TERM(mp))) ERL_MESSAGE_TERM(mp) = copy_struct(ERL_MESSAGE_TERM(mp), - msg_sz, &hp, - &hfrag->off_heap); + msg_sz, &hp, + &hfrag->off_heap); if (is_not_immed(ERL_MESSAGE_TOKEN(mp))) ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp), token_sz, &hp, diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index b5d8c5bc75..d6037e9f5d 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -11099,6 +11099,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #ifdef SHCOPY_SPAWN erts_shcopy_t info; INITIALIZE_SHCOPY(info); +#else + Eterm *lit_purge_ptr = erts_clrange.ptr; + Uint lit_purge_sz = erts_clrange.sz; #endif erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR); @@ -11157,7 +11160,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #ifdef SHCOPY_SPAWN arg_size = copy_shared_calculate(args, &info); #else - arg_size = size_object(args); + arg_size = size_object_litopt(args, lit_purge_ptr, lit_purge_sz); #endif heap_need = arg_size; @@ -11181,7 +11184,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). } p->schedule_count = 0; ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0)); - + p->u.initial[INITIAL_MOD] = mod; p->u.initial[INITIAL_FUN] = func; p->u.initial[INITIAL_ARI] = (Uint) arity; @@ -11239,7 +11242,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap); DESTROY_SHCOPY(info); #else - p->arg_reg[2] = copy_struct(args, arg_size, &p->htop, &p->off_heap); + p->arg_reg[2] = copy_struct_litopt(args, arg_size, &p->htop, &p->off_heap, + lit_purge_ptr, lit_purge_sz); #endif p->arity = 3; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index bf2e50e52f..67ce0d0723 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1085,8 +1085,8 @@ typedef struct { Eterm* shtable_start; ErtsAlcType_t shtable_alloc_type; Uint literal_size; - Eterm *range_ptr; - Uint range_sz; + Eterm *lit_purge_ptr; + Uint lit_purge_sz; } erts_shcopy_t; #define INITIALIZE_SHCOPY(info) \ @@ -1095,8 +1095,8 @@ do { \ info.bitstore_start = info.bitstore_default; \ info.shtable_start = info.shtable_default; \ info.literal_size = 0; \ - info.range_ptr = erts_clrange.ptr; \ - info.range_sz = erts_clrange.sz; \ + info.lit_purge_ptr = erts_clrange.ptr; \ + info.lit_purge_sz = erts_clrange.sz; \ } while(0) #define DESTROY_SHCOPY(info) \ @@ -1116,15 +1116,21 @@ do { \ Eterm copy_object_x(Eterm, Process*, Uint); #define copy_object(Term, Proc) copy_object_x(Term,Proc,0) -Uint size_object(Eterm); +Uint size_object_x(Eterm,Eterm*,Uint,Uint); +#define size_object(Term) size_object_x(Term,NULL,0,0) +#define size_object_litopt(Term,LitPtr,LitSz) size_object_x(Term,LitPtr,LitSz,1) + Uint copy_shared_calculate(Eterm, erts_shcopy_t*); Eterm copy_shared_perform(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*); Uint size_shared(Eterm); -Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint* bsz); +Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint* bsz, Eterm *lit_ptr, Uint lit_sz, Uint litopt); #define copy_struct(Obj,Sz,HPP,OH) \ - copy_struct_x(Obj,Sz,HPP,OH,NULL) + copy_struct_x(Obj,Sz,HPP,OH,NULL,NULL,0,0) +#define copy_struct_litopt(Obj,Sz,HPP,OH,LitPtr,LitSz) \ + copy_struct_x(Obj,Sz,HPP,OH,NULL,LitPtr,LitSz,1) + Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*); void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, -- cgit v1.2.3 From a9368b3c1ee78a3c83fcff83590a3d94ea4ddbe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Wed, 27 Jul 2016 17:00:32 +0200 Subject: erts: Fix literal size calculation in check_process_code We want to know the total size of literals of all heap fragmens. --- erts/emulator/beam/beam_bif_load.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 15e878ba65..2a165bed00 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -873,12 +873,12 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls continue; { ErlHeapFragment *hf; - Uint lit_sz; + Uint lit_sz = 0; for (hf=hfrag; hf; hf = hf->next) { if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)) return am_true; - lit_sz = hfrag_literal_size(&hf->mem[0], &hf->mem[hf->used_size], - literals, lit_bsize); + lit_sz += hfrag_literal_size(&hf->mem[0], &hf->mem[hf->used_size], + literals, lit_bsize); } if (lit_sz > 0) { ErlHeapFragment *bp = new_message_buffer(lit_sz); -- cgit v1.2.3 From 13ec43880b8565a5ddd7da590c36a520d4ff4b26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 29 Jul 2016 12:19:11 +0200 Subject: erts: Don't copy literals in enif_send --- erts/emulator/beam/erl_nif.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index c6127a4967..e1944fff29 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -627,7 +627,9 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, MBUF(&menv->phony_proc) = NULL; } } else { - Uint sz = size_object(msg); + Eterm *lit_purge_ptr = erts_clrange.ptr; + Uint lit_purge_sz = erts_clrange.sz; + Uint sz = size_object_litopt(msg, lit_purge_ptr, lit_purge_sz); ErlOffHeap *ohp; Eterm *hp; if (env && !env->tracee) { @@ -649,7 +651,8 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ohp = &bp->off_heap; } } - msg = copy_struct(msg, sz, &hp, ohp); + msg = copy_struct_litopt(msg, sz, &hp, ohp, + lit_purge_ptr, lit_purge_sz); } ERL_MESSAGE_TERM(mp) = msg; -- cgit v1.2.3 From dbea5c4023ec6ac59bb1988711532682a032536d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 29 Jul 2016 15:59:16 +0200 Subject: erts: No need to literal opt message token --- erts/emulator/beam/erl_gc.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 22a7d430c8..cddcc847a3 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -2252,10 +2252,12 @@ move_msgq_to_heap(Process *p) ASSERT(mp->data.dist_ext->heap_size >= 0); if (is_not_nil(ERL_MESSAGE_TOKEN(mp))) { bp = erts_dist_ext_trailer(mp->data.dist_ext); + /* Tokens does not use literal optimization */ ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp), bp->used_size, - &factory.hp, - factory.off_heap); + &factory.hp, + factory.off_heap); + erts_cleanup_offheap(&bp->off_heap); } ERL_MESSAGE_TERM(mp) = erts_decode_dist_ext(&factory, -- cgit v1.2.3 From e52fa24451306080be39a35444cf52210982b41a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 29 Jul 2016 17:16:53 +0200 Subject: erts: Refactor literal purge area arguments --- erts/emulator/beam/copy.c | 14 ++++++++------ erts/emulator/beam/erl_message.c | 14 ++++++-------- erts/emulator/beam/erl_nif.c | 10 +++++----- erts/emulator/beam/erl_process.c | 9 ++++----- erts/emulator/beam/global.h | 25 ++++++++++++++++++------- 5 files changed, 41 insertions(+), 31 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 7d3777f37e..c4dcd6a3cc 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -76,16 +76,17 @@ Eterm copy_object_x(Eterm obj, Process* to, Uint extra) (lit_purge_ptr <= (PTR) && \ (PTR) < (lit_purge_ptr + lit_purge_sz)))) -Uint size_object_x(Eterm obj, Eterm *lit_purge_ptr, Uint lit_purge_sz, Uint litopt) +Uint size_object_x(Eterm obj, erts_literal_area_t *litopt) { Uint sum = 0; Eterm* ptr; int arity; + Eterm *lit_purge_ptr = litopt ? litopt->lit_purge_ptr : NULL; + Uint lit_purge_sz = litopt ? litopt->lit_purge_sz : 0; #ifdef DEBUG Eterm mypid = erts_get_current_pid(); #endif DECLARE_ESTACK(s); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size_object %p\n", mypid, obj)); for (;;) { @@ -608,8 +609,7 @@ cleanup: /* * Copy a structure to a heap. */ -Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz, - Eterm *lit_purge_ptr, Uint lit_purge_sz, Uint litopt) +Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz, erts_literal_area_t *litopt) { char* hstart; Uint hsize; @@ -626,6 +626,8 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint Eterm hdr; Eterm *hend; int i; + Eterm *lit_purge_ptr = litopt ? litopt->lit_purge_ptr : NULL; + Uint lit_purge_sz = litopt ? litopt->lit_purge_sz : 0; #ifdef DEBUG Eterm org_obj = obj; Uint org_sz = sz; @@ -1383,7 +1385,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, *resp = obj; } else { Uint bsz = 0; - *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL, 0, 0); /* copy literal */ + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL); /* copy literal */ hbot -= bsz; } goto cleanup_next; @@ -1451,7 +1453,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, *resp = obj; } else { Uint bsz = 0; - *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL, 0, 0); /* copy literal */ + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL); /* copy literal */ hbot -= bsz; } goto cleanup_next; diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 14ee07a304..e9f0586edd 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -697,8 +697,8 @@ erts_send_message(Process* sender, #ifdef SHCOPY_SEND erts_shcopy_t info; #else - Eterm *lit_purge_ptr = erts_clrange.ptr; - Uint lit_purge_sz = erts_clrange.sz; + erts_literal_area_t litarea; + INITIALIZE_LITERAL_PURGE_AREA(litarea); #endif #ifdef USE_VM_PROBES @@ -744,7 +744,7 @@ erts_send_message(Process* sender, INITIALIZE_SHCOPY(info); msize = copy_shared_calculate(message, &info); #else - msize = size_object_litopt(message, lit_purge_ptr, lit_purge_sz); + msize = size_object_litopt(message, &litarea); #endif mp = erts_alloc_message_heap_state(receiver, &receiver_state, @@ -763,8 +763,7 @@ erts_send_message(Process* sender, DESTROY_SHCOPY(info); #else if (is_not_immed(message)) - message = copy_struct_litopt(message, msize, &hp, ohp, - lit_purge_ptr, lit_purge_sz); + message = copy_struct_litopt(message, msize, &hp, ohp, &litarea); #endif if (is_immed(stoken)) token = stoken; @@ -800,7 +799,7 @@ erts_send_message(Process* sender, INITIALIZE_SHCOPY(info); msize = copy_shared_calculate(message, &info); #else - msize = size_object_litopt(message, lit_purge_ptr, lit_purge_sz); + msize = size_object_litopt(message, &litarea); #endif mp = erts_alloc_message_heap_state(receiver, &receiver_state, @@ -814,8 +813,7 @@ erts_send_message(Process* sender, DESTROY_SHCOPY(info); #else if (is_not_immed(message)) - message = copy_struct_litopt(message, msize, &hp, ohp, - lit_purge_ptr, lit_purge_sz); + message = copy_struct_litopt(message, msize, &hp, ohp, &litarea); #endif } #ifdef USE_VM_PROBES diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index e1944fff29..6bd27d11f4 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -627,11 +627,12 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, MBUF(&menv->phony_proc) = NULL; } } else { - Eterm *lit_purge_ptr = erts_clrange.ptr; - Uint lit_purge_sz = erts_clrange.sz; - Uint sz = size_object_litopt(msg, lit_purge_ptr, lit_purge_sz); + erts_literal_area_t litarea; ErlOffHeap *ohp; Eterm *hp; + Uint sz; + INITIALIZE_LITERAL_PURGE_AREA(litarea); + sz = size_object_litopt(msg, &litarea); if (env && !env->tracee) { flush_env(env); mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); @@ -651,8 +652,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ohp = &bp->off_heap; } } - msg = copy_struct_litopt(msg, sz, &hp, ohp, - lit_purge_ptr, lit_purge_sz); + msg = copy_struct_litopt(msg, sz, &hp, ohp, &litarea); } ERL_MESSAGE_TERM(mp) = msg; diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index d6037e9f5d..85c37e08be 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -11100,8 +11100,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). erts_shcopy_t info; INITIALIZE_SHCOPY(info); #else - Eterm *lit_purge_ptr = erts_clrange.ptr; - Uint lit_purge_sz = erts_clrange.sz; + erts_literal_area_t litarea; + INITIALIZE_LITERAL_PURGE_AREA(litarea); #endif erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR); @@ -11160,7 +11160,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #ifdef SHCOPY_SPAWN arg_size = copy_shared_calculate(args, &info); #else - arg_size = size_object_litopt(args, lit_purge_ptr, lit_purge_sz); + arg_size = size_object_litopt(args, &litarea); #endif heap_need = arg_size; @@ -11242,8 +11242,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap); DESTROY_SHCOPY(info); #else - p->arg_reg[2] = copy_struct_litopt(args, arg_size, &p->htop, &p->off_heap, - lit_purge_ptr, lit_purge_sz); + p->arg_reg[2] = copy_struct_litopt(args, arg_size, &p->htop, &p->off_heap, &litarea); #endif p->arity = 3; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 67ce0d0723..1423973739 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1113,23 +1113,34 @@ do { \ } while(0) /* copy.c */ +typedef struct { + Eterm *lit_purge_ptr; + Uint lit_purge_sz; +} erts_literal_area_t; + +#define INITIALIZE_LITERAL_PURGE_AREA(Area) \ + do { \ + (Area).lit_purge_ptr = erts_clrange.ptr; \ + (Area).lit_purge_sz = erts_clrange.sz; \ + } while(0) + Eterm copy_object_x(Eterm, Process*, Uint); #define copy_object(Term, Proc) copy_object_x(Term,Proc,0) -Uint size_object_x(Eterm,Eterm*,Uint,Uint); -#define size_object(Term) size_object_x(Term,NULL,0,0) -#define size_object_litopt(Term,LitPtr,LitSz) size_object_x(Term,LitPtr,LitSz,1) +Uint size_object_x(Eterm, erts_literal_area_t*); +#define size_object(Term) size_object_x(Term,NULL) +#define size_object_litopt(Term,LitArea) size_object_x(Term,LitArea) Uint copy_shared_calculate(Eterm, erts_shcopy_t*); Eterm copy_shared_perform(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*); Uint size_shared(Eterm); -Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint* bsz, Eterm *lit_ptr, Uint lit_sz, Uint litopt); +Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint*, erts_literal_area_t*); #define copy_struct(Obj,Sz,HPP,OH) \ - copy_struct_x(Obj,Sz,HPP,OH,NULL,NULL,0,0) -#define copy_struct_litopt(Obj,Sz,HPP,OH,LitPtr,LitSz) \ - copy_struct_x(Obj,Sz,HPP,OH,NULL,LitPtr,LitSz,1) + copy_struct_x(Obj,Sz,HPP,OH,NULL,NULL) +#define copy_struct_litopt(Obj,Sz,HPP,OH,LitArea) \ + copy_struct_x(Obj,Sz,HPP,OH,NULL,LitArea) Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*); -- cgit v1.2.3 From c2bc5b5204a482f3a04ce81d9ce57fe9ee845c82 Mon Sep 17 00:00:00 2001 From: Mahesh Baheti Date: Sat, 6 Aug 2016 22:11:57 +0900 Subject: Correct spelling of atomicity --- erts/emulator/beam/erl_db.c | 4 ++-- erts/emulator/beam/erl_db_hash.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index bad34211a5..128a7b3865 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1094,7 +1094,7 @@ BIF_RETTYPE ets_insert_2(BIF_ALIST_2) CHECK_TABLES(); - /* Write lock table if more than one object to keep atomicy */ + /* Write lock table if more than one object to keep atomicity */ kind = ((is_list(BIF_ARG_2) && CDR(list_val(BIF_ARG_2)) != NIL) ? LCK_WRITE : LCK_WRITE_REC); @@ -1164,7 +1164,7 @@ BIF_RETTYPE ets_insert_new_2(BIF_ALIST_2) Eterm lookup_ret; DbTableMethod* meth; - /* More than one object, use LCK_WRITE to keep atomicy */ + /* More than one object, use LCK_WRITE to keep atomicity */ kind = LCK_WRITE; tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, kind); if (tb == NULL) { diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 12ae086b31..5e6fe4f460 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -40,7 +40,7 @@ ** DB_FINE_LOCKED set. The table variable is_thread_safe will then indicate ** if operations need to obtain fine grained locks or not. Some operations ** will for example always use exclusive table lock to guarantee -** a higher level of atomicy. +** a higher level of atomicity. */ /* FIXATION: -- cgit v1.2.3 From aeb645a709b73e1bda0281f87dda2af3ce92dfe7 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 20 Jul 2016 17:30:39 +0200 Subject: Separate literal area from code --- erts/emulator/beam/beam_bif_load.c | 33 ++++++-------------------- erts/emulator/beam/beam_load.c | 47 +++++++++++++++++++++++++++++--------- erts/emulator/beam/beam_load.h | 14 +++++++++--- 3 files changed, 54 insertions(+), 40 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 15e878ba65..a94a3cc3d3 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -40,7 +40,6 @@ static void set_default_trace_pattern(Eterm module); static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls); static void delete_code(Module* modp); -static void decrement_refc(BeamCodeHeader*); static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); @@ -861,8 +860,8 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ); - literals = (char*) modp->old.code_hdr->literals_start; - lit_bsize = (char*) modp->old.code_hdr->literals_end - literals; + literals = (char*) modp->old.code_hdr->literal_area->start; + lit_bsize = (char*) modp->old.code_hdr->literal_area->end - literals; for (msgp = rp->msg.first; msgp; msgp = msgp->next) { if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) @@ -994,7 +993,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls } if (need_gc & ERTS_LITERAL_GC__) { struct erl_off_heap_header* oh; - oh = modp->old.code_hdr->literals_off_heap; + oh = modp->old.code_hdr->literal_area->off_heap; *redsp += lit_bsize / 64; /* Need, better value... */ erts_garbage_collect_literals(rp, (Eterm*)literals, lit_bsize, oh); done_gc |= ERTS_LITERAL_GC__; @@ -1186,8 +1185,8 @@ BIF_RETTYPE erts_internal_copy_literals_2(BIF_ALIST_2) res = am_aborted; goto done; } - erts_clrange.ptr = modp->old.code_hdr->literals_start; - erts_clrange.sz = modp->old.code_hdr->literals_end - erts_clrange.ptr; + erts_clrange.ptr = modp->old.code_hdr->literal_area->start; + erts_clrange.sz = modp->old.code_hdr->literal_area->end - erts_clrange.ptr; erts_clrange.pid = BIF_P->common.id; } else if (BIF_ARG_2 == am_false) { if (erts_clrange.pid != BIF_P->common.id) { @@ -1295,10 +1294,8 @@ BIF_RETTYPE erts_internal_purge_module_1(BIF_ALIST_1) erts_cleanup_funs_on_purge(code, end); beam_catches_delmod(modp->old.catches, code, modp->old.code_length, code_ix); - decrement_refc(modp->old.code_hdr); - if (modp->old.code_hdr->literals_start) { - erts_free(ERTS_ALC_T_LITERAL, modp->old.code_hdr->literals_start); - } + erts_release_literal_area(modp->old.code_hdr->literal_area); + modp->old.code_hdr->literal_area = NULL; erts_free(ERTS_ALC_T_CODE, (void *) code); modp->old.code_hdr = NULL; modp->old.code_length = 0; @@ -1316,22 +1313,6 @@ BIF_RETTYPE erts_internal_purge_module_1(BIF_ALIST_1) return ret; } -static void -decrement_refc(BeamCodeHeader* code_hdr) -{ - struct erl_off_heap_header* oh = code_hdr->literals_off_heap; - - while (oh) { - Binary* bptr; - ASSERT(thing_subtag(oh->thing_word) == REFC_BINARY_SUBTAG); - bptr = ((ProcBin*)oh)->val; - if (erts_refc_dectest(&bptr->refc, 0) == 0) { - erts_bin_free(bptr); - } - oh = oh->next; - } -} - /* * Move code from current to old and null all export entries for the module */ diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 0c2743beb2..b2c4fe5f5e 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -663,7 +663,7 @@ erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader, stp->hdr->compile_ptr = NULL; stp->hdr->compile_size = 0; stp->hdr->compile_size_on_heap = 0; - stp->hdr->literals_start = NULL; + stp->hdr->literal_area = NULL; stp->hdr->md5_ptr = NULL; /* @@ -1005,8 +1005,9 @@ loader_state_dtor(Binary* magic) stp->bin = 0; } if (stp->hdr != 0) { - if (stp->hdr->literals_start) { - erts_free(ERTS_ALC_T_LITERAL, stp->hdr->literals_start); + if (stp->hdr->literal_area) { + erts_release_literal_area(stp->hdr->literal_area); + stp->hdr->literal_area = NULL; } erts_free(ERTS_ALC_T_CODE, stp->hdr); stp->hdr = 0; @@ -4560,13 +4561,16 @@ freeze_code(LoaderState* stp) Eterm* ptr; LiteralPatch* lp; ErlOffHeap code_off_heap; + ErtsLiteralArea *literal_area; + Uint lit_asize; ERTS_INIT_OFF_HEAP(&code_off_heap); - ptr = (Eterm*)erts_alloc(ERTS_ALC_T_LITERAL, - stp->total_literal_size*sizeof(Eterm)); - code_hdr->literals_start = ptr; - code_hdr->literals_end = ptr + stp->total_literal_size; + lit_asize = ERTS_LITERAL_AREA_ALLOC_SIZE(stp->total_literal_size); + literal_area = erts_alloc(ERTS_ALC_T_LITERAL, lit_asize); + ptr = &literal_area->start[0]; + literal_area->end = ptr + stp->total_literal_size; + for (i = 0; i < stp->num_literals; i++) { if (is_not_immed(stp->literals[i].term)) { erts_move_multi_frags(&ptr, &code_off_heap, @@ -4576,7 +4580,7 @@ freeze_code(LoaderState* stp) ptr_val(stp->literals[i].term))); } } - code_hdr->literals_off_heap = code_off_heap.first; + literal_area->off_heap = code_off_heap.first; lp = stp->literal_patches; while (lp != 0) { BeamInstr* op_ptr; @@ -4587,6 +4591,7 @@ freeze_code(LoaderState* stp) op_ptr[0] = lit->term; lp = lp->next; } + code_hdr->literal_area = literal_area; } CHKBLK(ERTS_ALC_T_CODE,code); @@ -5642,6 +5647,28 @@ has_native(BeamCodeHeader *code_hdr) return result; } +void +erts_release_literal_area(ErtsLiteralArea* literal_area) +{ + struct erl_off_heap_header* oh; + + if (!literal_area) + return; + + oh = literal_area->off_heap; + + while (oh) { + Binary* bptr; + ASSERT(thing_subtag(oh->thing_word) == REFC_BINARY_SUBTAG); + bptr = ((ProcBin*)oh)->val; + if (erts_refc_dectest(&bptr->refc, 0) == 0) { + erts_bin_free(bptr); + } + oh = oh->next; + } + erts_free(ERTS_ALC_T_LITERAL, literal_area); +} + int erts_is_module_native(BeamCodeHeader* code_hdr) { @@ -6373,9 +6400,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) code_hdr->compile_ptr = NULL; code_hdr->compile_size = 0; code_hdr->compile_size_on_heap = 0; - code_hdr->literals_start = NULL; - code_hdr->literals_end = NULL; - code_hdr->literals_off_heap = 0; + code_hdr->literal_area = NULL; code_hdr->on_load_function_ptr = NULL; code_hdr->line_table = NULL; code_hdr->md5_ptr = NULL; diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index fd2dd97fee..336f097f0f 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -60,6 +60,15 @@ extern BeamInstr* em_call_nif; /* Total code size in bytes */ extern Uint erts_total_code_size; +typedef struct { + struct erl_off_heap_header *off_heap; + Eterm *end; + Eterm start[1]; /* beginning of area */ +} ErtsLiteralArea; + +#define ERTS_LITERAL_AREA_ALLOC_SIZE(N) \ + (sizeof(ErtsLiteralArea) + sizeof(Eterm)*((N) - 1)) + typedef struct BeamCodeLineTab_ BeamCodeLineTab; /* @@ -89,9 +98,7 @@ typedef struct beam_code_header { /* * Literal area (constant pool). */ - Eterm* literals_start; - Eterm* literals_end; - struct erl_off_heap_header* literals_off_heap; + ErtsLiteralArea *literal_area; /* * Pointer to the on_load function (or NULL if none). @@ -120,6 +127,7 @@ typedef struct beam_code_header { }BeamCodeHeader; +void erts_release_literal_area(ErtsLiteralArea* literal_area); int erts_is_module_native(BeamCodeHeader* code); /* -- cgit v1.2.3 From 17c838402c6d71aef041548ad36ee66db6f24a56 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 11 Aug 2016 15:45:05 +0200 Subject: Fix race causing lost wakeup on receive-after timeout --- erts/emulator/beam/beam_emu.c | 5 ++++- erts/emulator/beam/erl_hl_timer.c | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 8b409e139b..17df14acfe 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -2166,7 +2166,10 @@ void process_main(void) c_p->i = (BeamInstr *) Arg(0); /* L1 */ SWAPOUT; c_p->arity = 0; - erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE); + + if (!ERTS_PTMR_IS_TIMED_OUT(c_p)) + erts_smp_atomic32_read_band_relb(&c_p->state, + ~ERTS_PSFLG_ACTIVE); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); c_p->current = NULL; diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index 51a0d68247..96bb5ac9fa 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -735,7 +735,10 @@ proc_timeout_common(Process *proc, void *tmr) if (tmr == (void *) erts_smp_atomic_cmpxchg_mb(&proc->common.timer, ERTS_PTMR_TIMEDOUT, (erts_aint_t) tmr)) { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&proc->state); + erts_aint32_t state; + erts_smp_proc_lock(proc, ERTS_PROC_LOCKS_MSG_RECEIVE); + state = erts_smp_atomic32_read_acqb(&proc->state); + erts_smp_proc_unlock(proc, ERTS_PROC_LOCKS_MSG_RECEIVE); if (!(state & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_EXITING))) erts_schedule_process(proc, state, 0); return 1; -- cgit v1.2.3 From ee32dcfea7d93c81c3d03489d9888e4baf523f2e Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 11 Aug 2016 15:52:31 +0200 Subject: Avoid segfault when printing slogan after crashdumping --- erts/emulator/beam/erl_init.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index d9c3b0dcf4..51b72033ca 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -2262,14 +2262,15 @@ erl_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) if (erts_mtrace_enabled) erts_mtrace_exit((Uint32) an); + if (fmt != NULL && *fmt != '\0') + erl_error(fmt, args2); /* Print error message. */ + /* Produce an Erlang core dump if error */ if (((n > 0 && erts_no_crash_dump == 0) || n == ERTS_DUMP_EXIT) && erts_initialized) { erl_crash_dump_v((char*) NULL, 0, fmt, args1); } - if (fmt != NULL && *fmt != '\0') - erl_error(fmt, args2); /* Print error message. */ sys_tty_reset(n); if (n == ERTS_INTR_EXIT) -- cgit v1.2.3 From ca4ad75e8f2058d965465d612ee1b5dcd364200e Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Thu, 11 Aug 2016 10:35:24 +0200 Subject: erts: Align ErtsThrPrgrLaterOp when free'ing tracer On Sparc all structs with a double word element in are assumed to be allocated on a two word boundary. So we have to make sure that the ErtsThrPrgrLaterOp * points to 8 / 16 byte aligned memory. As it only costs one word I've done it on all architectures in case some unknown other arch also needs this. --- erts/emulator/beam/erl_trace.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index d14d237ca6..c4396488f5 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -3108,6 +3108,7 @@ erts_tracer_update(ErtsTracer *tracer, const ErtsTracer new_tracer) if (is_not_nil(*tracer)) { Uint offs = 2; UWord size = 2 * sizeof(Eterm) + sizeof(ErtsThrPrgrLaterOp); + ErtsThrPrgrLaterOp *lop; ASSERT(is_list(*tracer)); if (is_not_immed(ERTS_TRACER_STATE(*tracer))) { hf = (void*)(((char*)(ptr_val(*tracer)) - offsetof(ErlHeapFragment, mem))); @@ -3115,6 +3116,16 @@ erts_tracer_update(ErtsTracer *tracer, const ErtsTracer new_tracer) size = hf->alloc_size * sizeof(Eterm) + sizeof(ErlHeapFragment); ASSERT(offs == size_object(*tracer)); } + + /* sparc assumes that all structs are double word aligned, so we + have to align the ErtsThrPrgrLaterOp struct otherwise it may + segfault.*/ + if ((UWord)(ptr_val(*tracer) + offs) % (sizeof(UWord)*2) == sizeof(UWord)) + offs += 1; + + lop = (ErtsThrPrgrLaterOp*)(ptr_val(*tracer) + offs); + ASSERT((UWord)lop % (sizeof(UWord)*2) == 0); + /* We schedule the free:ing of the tracer until after a thread progress has been made so that we know that no schedulers have any references to it. Because we do this, it is possible to release all locks of a @@ -3122,9 +3133,7 @@ erts_tracer_update(ErtsTracer *tracer, const ErtsTracer new_tracer) without having to worry if it is free'd. */ erts_schedule_thr_prgr_later_cleanup_op( - free_tracer, (void*)(*tracer), - (ErtsThrPrgrLaterOp*)(ptr_val(*tracer) + offs), - size); + free_tracer, (void*)(*tracer), lop, size); } if (is_nil(new_tracer)) { @@ -3141,9 +3150,10 @@ erts_tracer_update(ErtsTracer *tracer, const ErtsTracer new_tracer) Eterm *hp, tracer_state = ERTS_TRACER_STATE(new_tracer), tracer_module = ERTS_TRACER_MODULE(new_tracer); Uint sz = size_object(tracer_state); - hf = new_message_buffer(sz + 2 /* cons cell */ + (sizeof(ErtsThrPrgrLaterOp)+sizeof(Eterm)-1)/sizeof(Eterm)); + hf = new_message_buffer(sz + 2 /* cons cell */ + + (sizeof(ErtsThrPrgrLaterOp)+sizeof(Eterm)-1)/sizeof(Eterm) + 1); hp = hf->mem + 2; - hf->used_size -= (sizeof(ErtsThrPrgrLaterOp)+sizeof(Eterm)-1)/sizeof(Eterm); + hf->used_size -= (sizeof(ErtsThrPrgrLaterOp)+sizeof(Eterm)-1)/sizeof(Eterm) + 1; *tracer = copy_struct(tracer_state, sz, &hp, &hf->off_heap); *tracer = CONS(hf->mem, tracer_module, *tracer); ASSERT((void*)(((char*)(ptr_val(*tracer)) - offsetof(ErlHeapFragment, mem))) == hf); -- cgit v1.2.3 From 58d8a3ac4d491c8cd962ac0839a56cd1a0e339f9 Mon Sep 17 00:00:00 2001 From: Dmytro Lytovchenko Date: Mon, 13 Jun 2016 12:05:55 +0200 Subject: Option to erlang:garbage_collect to request minor (generational) GC Note: Minor GC option is a hint, and GC may still decide to run fullsweep. Test case for major and minor gc on self Test case for major and minor gs on some other process + async gc test check docs fix --- erts/emulator/beam/atom.names | 2 ++ erts/emulator/beam/bif.c | 19 ++++++++++++++++++- erts/emulator/beam/bif.tab | 1 + erts/emulator/beam/erl_process.c | 23 ++++++++++++++++------- erts/emulator/beam/ops.tab | 4 ++++ 5 files changed, 41 insertions(+), 8 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index badd69856e..f88aeba49f 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -361,6 +361,7 @@ atom long_schedule atom low atom Lt='<' atom machine +atom major atom match atom match_limit atom match_limit_recursion @@ -388,6 +389,7 @@ atom microstate_accounting atom milli_seconds atom min_heap_size atom min_bin_vheap_size +atom minor atom minor_version atom Minus='-' atom module diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index d9048065c8..84eab5f651 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -3869,7 +3869,24 @@ BIF_RETTYPE garbage_collect_0(BIF_ALIST_0) { FLAGS(BIF_P) |= F_NEED_FULLSWEEP; erts_garbage_collect(BIF_P, 0, NULL, 0); - BIF_RET(am_true); + return am_true; +} + +/* + * Pass atom 'minor' for relaxed generational GC run. This is only + * recommendation, major run may still be chosen by VM. + * Pass atom 'major' for default behaviour - major GC run (fullsweep) + */ +BIF_RETTYPE +erts_internal_garbage_collect_1(BIF_ALIST_1) +{ + switch (BIF_ARG_1) { + case am_minor: break; + case am_major: FLAGS(BIF_P) |= F_NEED_FULLSWEEP; break; + default: BIF_ERROR(BIF_P, BADARG); + } + erts_garbage_collect(BIF_P, 0, NULL, 0); + return am_true; } /**********************************************************************/ diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 978bcd4bba..5bb7db5d47 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -69,6 +69,7 @@ bif erlang:float_to_list/1 bif erlang:float_to_list/2 bif erlang:fun_info/2 bif erlang:garbage_collect/0 +bif erts_internal:garbage_collect/1 bif erlang:get/0 bif erlang:get/1 bif erlang:get_keys/1 diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 85c37e08be..a0c5e247d4 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -446,7 +446,8 @@ int erts_system_profile_ts_type = ERTS_TRACE_FLG_NOW_TIMESTAMP; #endif typedef enum { - ERTS_PSTT_GC, /* Garbage Collect */ + ERTS_PSTT_GC_MAJOR, /* Garbage Collect: Fullsweep */ + ERTS_PSTT_GC_MINOR, /* Garbage Collect: Generational */ ERTS_PSTT_CPC, /* Check Process Code */ ERTS_PSTT_COHMQ, /* Change off heap message queue */ ERTS_PSTT_FTMQ /* Flush trace msg queue */ @@ -10358,7 +10359,8 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) break; switch (st->type) { - case ERTS_PSTT_GC: + case ERTS_PSTT_GC_MAJOR: + case ERTS_PSTT_GC_MINOR: if (c_p->flags & F_DISABLE_GC) { save_gc_task(c_p, st, st_prio); st = NULL; @@ -10366,7 +10368,9 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) } else { if (!garbage_collected) { - FLAGS(c_p) |= F_NEED_FULLSWEEP; + if (st->type == ERTS_PSTT_GC_MAJOR) { + FLAGS(c_p) |= F_NEED_FULLSWEEP; + } reds -= scheduler_gc_proc(c_p, reds); garbage_collected = 1; } @@ -10442,7 +10446,8 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) break; switch (st->type) { - case ERTS_PSTT_GC: + case ERTS_PSTT_GC_MAJOR: + case ERTS_PSTT_GC_MINOR: st_res = am_false; break; case ERTS_PSTT_CPC: @@ -10555,9 +10560,13 @@ erts_internal_request_system_task_3(BIF_ALIST_3) switch (req_type) { case am_garbage_collect: - st->type = ERTS_PSTT_GC; - noproc_res = am_false; - if (!rp) + switch (st->arg[0]) { + case am_minor: st->type = ERTS_PSTT_GC_MINOR; break; + case am_major: st->type = ERTS_PSTT_GC_MAJOR; break; + default: goto badarg; + } + noproc_res = am_false; + if (!rp) goto noproc; break; diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 879daaca0a..801d43e957 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -811,6 +811,10 @@ call_ext u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext Bif call_ext_last u==0 Bif=u$bif:erlang:garbage_collect/0 D => i_call_ext_last Bif D call_ext_only u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext_only Bif +call_ext u==1 Bif=u$bif:erts_internal:garbage_collect/1 => i_call_ext Bif +call_ext_last u==1 Bif=u$bif:erts_internal:garbage_collect/1 D => i_call_ext_last Bif D +call_ext_only u==1 Bif=u$bif:erts_internal:garbage_collect/1 => i_call_ext_only Bif + # # put/2 and erase/1 must be able to do garbage collection, so we must call # them like functions. -- cgit v1.2.3 From f5a1c56f0ab2a276b5c2add54b8b0d6d276e1361 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Fri, 12 Aug 2016 16:18:25 +0200 Subject: erts: Fix port monitor memory leak --- erts/emulator/beam/erl_process.c | 1 - 1 file changed, 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index b5d8c5bc75..aa6af7427c 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -12276,7 +12276,6 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) erts_port_demonitor(pcontext->p, ERTS_PORT_DEMONITOR_ORIGIN_ON_DEATHBED, prt, mon->ref, NULL); - return; /* let erts_port_demonitor do the deletion */ } else { /* remote by pid */ ASSERT(is_external_pid(mon->pid)); dep = external_pid_dist_entry(mon->pid); -- cgit v1.2.3 From 8f0d1988c0943491e89428b54cd457c8e8e580a2 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 22 Aug 2016 20:26:01 +0200 Subject: erts: Fix load of beam with invalid imports and atom numbers --- erts/emulator/beam/beam_load.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 0c2743beb2..30913910b6 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -303,13 +303,13 @@ typedef struct LoaderState { * Atom table. */ - int num_atoms; /* Number of atoms in atom table. */ + unsigned int num_atoms; /* Number of atoms in atom table. */ Eterm* atom; /* Atom table. */ int num_exps; /* Number of exports. */ ExportEntry* export; /* Pointer to export table. */ - int num_imports; /* Number of imports. */ + unsigned int num_imports; /* Number of imports. */ ImportEntry* import; /* Import entry (translated information). */ /* @@ -1375,13 +1375,13 @@ load_atom_table(LoaderState* stp) static int load_import_table(LoaderState* stp) { - int i; + unsigned int i; GetInt(stp, 4, stp->num_imports); stp->import = erts_alloc(ERTS_ALC_T_PREPARED_CODE, stp->num_imports * sizeof(ImportEntry)); for (i = 0; i < stp->num_imports; i++) { - int n; + unsigned int n; Eterm mod; Eterm func; Uint arity; @@ -1389,17 +1389,17 @@ load_import_table(LoaderState* stp) GetInt(stp, 4, n); if (n >= stp->num_atoms) { - LoadError2(stp, "import entry %d: invalid atom number %d", i, n); + LoadError2(stp, "import entry %u: invalid atom number %u", i, n); } mod = stp->import[i].module = stp->atom[n]; GetInt(stp, 4, n); if (n >= stp->num_atoms) { - LoadError2(stp, "import entry %d: invalid atom number %d", i, n); + LoadError2(stp, "import entry %u: invalid atom number %u", i, n); } func = stp->import[i].function = stp->atom[n]; GetInt(stp, 4, arity); if (arity > MAX_REG) { - LoadError2(stp, "import entry %d: invalid arity %d", i, arity); + LoadError2(stp, "import entry %u: invalid arity %d", i, arity); } stp->import[i].arity = arity; stp->import[i].patches = 0; -- cgit v1.2.3 From eb889b196ecb8128b4cb232ab2ebb2715c9b8993 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 23 Aug 2016 17:17:43 +0200 Subject: erts: Reject an invalid beam code header size --- erts/emulator/beam/beam_load.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 30913910b6..8ad7862daa 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -1783,6 +1783,10 @@ read_code_header(LoaderState* stp) */ GetInt(stp, 4, head_size); + if (head_size > stp->file_left) { + LoadError2(stp, "invalid code header size %u; bytes left %u", + head_size, stp->file_left); + } stp->code_start = stp->file_p + head_size; stp->code_size = stp->file_left - head_size; stp->file_left = head_size; -- cgit v1.2.3 From e86fe38df606996919516e20cbdcf307dd5400fe Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 23 Aug 2016 18:32:48 +0200 Subject: erts:: Unsignify a bunch of loader variables --- erts/emulator/beam/beam_load.c | 100 ++++++++++++++++++++--------------------- 1 file changed, 50 insertions(+), 50 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 8ad7862daa..e36a54c5af 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -105,7 +105,7 @@ typedef struct { */ typedef struct genop { - int op; /* Opcode. */ + unsigned int op; /* Opcode. */ int arity; /* Number of arguments. */ GenOpArg def_args[MAX_OPARGS]; /* Default buffer for arguments. */ GenOpArg* a; /* The arguments. */ @@ -283,8 +283,8 @@ typedef struct LoaderState { byte* code_start; /* Start of code file. */ unsigned code_size; /* Size of code file. */ int specific_op; /* Specific opcode (-1 if not found). */ - int num_functions; /* Number of functions in module. */ - int num_labels; /* Number of labels. */ + unsigned int num_functions; /* Number of functions in module. */ + unsigned int num_labels; /* Number of labels. */ BeamCodeHeader* hdr; /* Loaded code header */ BeamInstr* codev; /* Loaded code buffer */ int codev_size; /* Size of code buffer in words. */ @@ -306,7 +306,7 @@ typedef struct LoaderState { unsigned int num_atoms; /* Number of atoms in atom table. */ Eterm* atom; /* Atom table. */ - int num_exps; /* Number of exports. */ + unsigned int num_exps; /* Number of exports. */ ExportEntry* export; /* Pointer to export table. */ unsigned int num_imports; /* Number of imports. */ @@ -323,8 +323,8 @@ typedef struct LoaderState { * Lambda table. */ - int num_lambdas; /* Number of lambdas in table. */ - int lambdas_allocated; /* Size of allocated lambda table. */ + unsigned int num_lambdas; /* Number of lambdas in table. */ + unsigned int lambdas_allocated; /* Size of allocated lambda table. */ Lambda* lambdas; /* Pointer to lambdas. */ Lambda def_lambdas[16]; /* Default storage for lambda table. */ char* lambda_error; /* Delayed missing 'FunT' error. */ @@ -333,8 +333,8 @@ typedef struct LoaderState { * Literals (constant pool). */ - int num_literals; /* Number of literals in table. */ - int allocated_literals; /* Number of literal entries allocated. */ + unsigned int num_literals; /* Number of literals in table. */ + unsigned int allocated_literals; /* Number of literal entries allocated. */ Literal* literals; /* Array of literals. */ LiteralPatch* literal_patches; /* Operands that need to be patched. */ Uint total_literal_size; /* Total heap size for all literals. */ @@ -343,13 +343,13 @@ typedef struct LoaderState { * Line table. */ BeamInstr* line_item; /* Line items from the BEAM file. */ - int num_line_items; /* Number of line items. */ + unsigned int num_line_items;/* Number of line items. */ LineInstr* line_instr; /* Line instructions */ - int num_line_instrs; /* Maximum number of line instructions */ - int current_li; /* Current line instruction */ - int* func_line; /* Mapping from function to first line instr */ + unsigned int num_line_instrs; /* Maximum number of line instructions */ + unsigned int current_li; /* Current line instruction */ + unsigned int* func_line; /* Mapping from function to first line instr */ Eterm* fname; /* List of file names */ - int num_fnames; /* Number of filenames in fname table */ + unsigned int num_fnames; /* Number of filenames in fname table */ int loc_size; /* Size of location info in bytes (2/4) */ } LoaderState; @@ -1330,7 +1330,7 @@ verify_chunks(LoaderState* stp) static int load_atom_table(LoaderState* stp) { - int i; + unsigned int i; GetInt(stp, 4, stp->num_atoms); stp->num_atoms++; @@ -1427,12 +1427,12 @@ load_import_table(LoaderState* stp) static int read_export_table(LoaderState* stp) { - int i; + unsigned int i; BeamInstr* address; GetInt(stp, 4, stp->num_exps); if (stp->num_exps > stp->num_functions) { - LoadError2(stp, "%d functions exported; only %d functions defined", + LoadError2(stp, "%u functions exported; only %u functions defined", stp->num_exps, stp->num_functions); } stp->export @@ -1450,16 +1450,16 @@ read_export_table(LoaderState* stp) stp->export[i].function = func; GetInt(stp, 4, arity); if (arity > MAX_REG) { - LoadError2(stp, "export table entry %d: absurdly high arity %d", i, arity); + LoadError2(stp, "export table entry %u: absurdly high arity %u", i, arity); } stp->export[i].arity = arity; GetInt(stp, 4, n); if (n >= stp->num_labels) { - LoadError3(stp, "export table entry %d: invalid label %d (highest defined label is %d)", i, n, stp->num_labels); + LoadError3(stp, "export table entry %u: invalid label %u (highest defined label is %u)", i, n, stp->num_labels); } value = stp->labels[n].value; if (value == 0) { - LoadError2(stp, "export table entry %d: label %d not resolved", i, n); + LoadError2(stp, "export table entry %u: label %u not resolved", i, n); } stp->export[i].address = address = stp->codev + value; @@ -1520,7 +1520,7 @@ is_bif(Eterm mod, Eterm func, unsigned arity) static int read_lambda_table(LoaderState* stp) { - int i; + unsigned int i; GetInt(stp, 4, stp->num_lambdas); if (stp->num_lambdas > stp->lambdas_allocated) { @@ -1540,12 +1540,12 @@ read_lambda_table(LoaderState* stp) GetAtom(stp, n, stp->lambdas[i].function); GetInt(stp, 4, arity); if (arity > MAX_REG) { - LoadError2(stp, "lambda entry %d: absurdly high arity %d", i, arity); + LoadError2(stp, "lambda entry %u: absurdly high arity %u", i, arity); } stp->lambdas[i].arity = arity; GetInt(stp, 4, n); if (n >= stp->num_labels) { - LoadError3(stp, "lambda entry %d: invalid label %d (highest defined label is %d)", + LoadError3(stp, "lambda entry %u: invalid label %u (highest defined label is %u)", i, n, stp->num_labels); } stp->lambdas[i].label = n; @@ -1566,7 +1566,7 @@ read_lambda_table(LoaderState* stp) static int read_literal_table(LoaderState* stp) { - int i; + unsigned int i; uLongf uncompressed_sz; byte* uncompressed = 0; @@ -1588,7 +1588,7 @@ read_literal_table(LoaderState* stp) } for (i = 0; i < stp->num_literals; i++) { - int sz; + Uint sz; Sint heap_size; byte* p; Eterm val; @@ -1597,7 +1597,7 @@ read_literal_table(LoaderState* stp) GetInt(stp, 4, sz); /* Size of external term format. */ GetString(stp, p, sz); if ((heap_size = erts_decode_ext_size(p, sz)) < 0) { - LoadError1(stp, "literal %d: bad external format", i); + LoadError1(stp, "literal %u: bad external format", i); } if (heap_size > 0) { @@ -1607,7 +1607,7 @@ read_literal_table(LoaderState* stp) val = erts_decode_ext(&factory, &p, 0); if (is_non_value(val)) { - LoadError1(stp, "literal %d: bad external format", i); + LoadError1(stp, "literal %u: bad external format", i); } erts_factory_close(&factory); stp->literals[i].heap_frags = factory.heap_frags; @@ -1617,7 +1617,7 @@ read_literal_table(LoaderState* stp) erts_factory_dummy_init(&factory); val = erts_decode_ext(&factory, &p, 0); if (is_non_value(val)) { - LoadError1(stp, "literal %d: bad external format", i); + LoadError1(stp, "literal %u: bad external format", i); } ASSERT(is_immed(val)); stp->literals[i].heap_frags = NULL; @@ -1640,9 +1640,9 @@ read_line_table(LoaderState* stp) { unsigned version; ERTS_DECLARE_DUMMY(unsigned flags); - int num_line_items; + unsigned int num_line_items; BeamInstr* lp; - int i; + unsigned int i; BeamInstr fname_index; BeamInstr tag; @@ -1721,7 +1721,7 @@ read_line_table(LoaderState* stp) } } else if (tag == TAG_a) { if (val > stp->num_fnames) { - LoadError2(stp, "file index overflow (%d/%d)", + LoadError2(stp, "file index overflow (%u/%u)", val, stp->num_fnames); } fname_index = val; @@ -1757,9 +1757,9 @@ read_line_table(LoaderState* stp) stp->num_line_instrs * sizeof(LineInstr)); stp->current_li = 0; - stp->func_line = (int *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, - stp->num_functions * - sizeof(int)); + stp->func_line = (unsigned int *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, + stp->num_functions * + sizeof(int)); return 1; @@ -1891,7 +1891,7 @@ load_code(LoaderState* stp) ci = stp->ci; for (;;) { - int new_op; + unsigned int new_op; GenOp* tmp_op; ASSERT(ci <= codev_size); @@ -1899,10 +1899,10 @@ load_code(LoaderState* stp) get_next_instr: GetByte(stp, new_op); if (new_op >= NUM_GENERIC_OPS) { - LoadError1(stp, "invalid opcode %d", new_op); + LoadError1(stp, "invalid opcode %u", new_op); } if (gen_opc[new_op].name[0] == '\0') { - LoadError1(stp, "invalid opcode %d", new_op); + LoadError1(stp, "invalid opcode %u", new_op); } @@ -2372,7 +2372,7 @@ load_code(LoaderState* stp) VerifyTag(stp, tag, TAG_u); last_label = tmp_op->a[arg].val; if (!(0 < last_label && last_label < stp->num_labels)) { - LoadError2(stp, "invalid label num %d (0 < label < %d)", + LoadError2(stp, "invalid label num %u (0 < label < %u)", tmp_op->a[arg].val, stp->num_labels); } if (stp->labels[last_label].value != 0) { @@ -2516,7 +2516,7 @@ load_code(LoaderState* stp) { Sint offset; if (function_number >= stp->num_functions) { - LoadError1(stp, "too many functions in module (header said %d)", + LoadError1(stp, "too many functions in module (header said %u)", stp->num_functions); } @@ -2595,14 +2595,14 @@ load_code(LoaderState* stp) if (stp->line_item) { BeamInstr item = code[ci-1]; BeamInstr loc; - int li; + unsigned int li; if (item >= stp->num_line_items) { - LoadError2(stp, "line instruction index overflow (%d/%d)", + LoadError2(stp, "line instruction index overflow (%u/%u)", item, stp->num_line_items); } li = stp->current_li; if (li >= stp->num_line_instrs) { - LoadError2(stp, "line instruction table overflow (%d/%d)", + LoadError2(stp, "line instruction table overflow (%u/%u)", li, stp->num_line_instrs); } loc = stp->line_item[item]; @@ -4602,8 +4602,8 @@ freeze_code(LoaderState* stp) str_table = (byte *) (codev + stp->ci); } else { BeamCodeLineTab* const line_tab = (BeamCodeLineTab *) (codev+stp->ci); - const int ftab_size = stp->num_functions; - const int num_instrs = stp->current_li; + const unsigned int ftab_size = stp->num_functions; + const unsigned int num_instrs = stp->current_li; const BeamInstr** const line_items = (const BeamInstr**) &line_tab->func_tab[ftab_size + 1]; @@ -4763,7 +4763,7 @@ freeze_code(LoaderState* stp) static void final_touch(LoaderState* stp, struct erl_module_instance* inst_p) { - int i; + unsigned int i; int on_load = stp->on_load; unsigned catches; Uint index; @@ -5435,7 +5435,7 @@ new_genop(LoaderState* stp) static int new_label(LoaderState* stp) { - int num = stp->num_labels; + unsigned int num = stp->num_labels; stp->num_labels++; stp->labels = (Label *) erts_realloc(ERTS_ALC_T_PREPARED_CODE, @@ -6033,11 +6033,11 @@ stub_copy_info(LoaderState* stp, static int stub_read_export_table(LoaderState* stp) { - int i; + unsigned int i; GetInt(stp, 4, stp->num_exps); if (stp->num_exps > stp->num_functions) { - LoadError2(stp, "%d functions exported; only %d functions defined", + LoadError2(stp, "%u functions exported; only %u functions defined", stp->num_exps, stp->num_functions); } stp->export @@ -6051,7 +6051,7 @@ stub_read_export_table(LoaderState* stp) GetAtom(stp, n, stp->export[i].function); GetInt(stp, 4, n); if (n > MAX_REG) { - LoadError2(stp, "export table entry %d: absurdly high arity %d", i, n); + LoadError2(stp, "export table entry %u: absurdly high arity %u", i, n); } stp->export[i].arity = n; GetInt(stp, 4, n); /* Ignore label */ @@ -6065,8 +6065,8 @@ stub_read_export_table(LoaderState* stp) static void stub_final_touch(LoaderState* stp, BeamInstr* fp) { - int i; - int n = stp->num_exps; + unsigned int i; + unsigned int n = stp->num_exps; Eterm mod = fp[2]; Eterm function = fp[3]; int arity = fp[4]; -- cgit v1.2.3 From ba12a91c70729048e5496a66587e56e17b6ad3f8 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Wed, 24 Aug 2016 13:21:20 +0200 Subject: erts: Immed tracer states also have to be alignable --- erts/emulator/beam/erl_trace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index c4396488f5..6b07ab87e4 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -3143,7 +3143,7 @@ erts_tracer_update(ErtsTracer *tracer, const ErtsTracer new_tracer) Not sure if it is worth it, we save 4 words (sizeof(ErlHeapFragment)) per tracer. */ Eterm *hp = erts_alloc(ERTS_ALC_T_HEAP_FRAG, - 2*sizeof(Eterm) + sizeof(ErtsThrPrgrLaterOp)); + 3*sizeof(Eterm) + sizeof(ErtsThrPrgrLaterOp)); *tracer = CONS(hp, ERTS_TRACER_MODULE(new_tracer), ERTS_TRACER_STATE(new_tracer)); } else { -- cgit v1.2.3 From 6cba7d35d3322db8acc25c45889c2b03f1b2f4c2 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 19 Aug 2016 20:16:20 +0200 Subject: erts: Add ErtsContainerStruct_ for array members that otherwise may produce warning from compilers that think T* and T[] are incompatible types (?). --- erts/emulator/beam/sys.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index dfe82cab44..2d0628f70e 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -99,6 +99,10 @@ #define ErtsContainerStruct(ptr, type, member) \ ((type *)((char *)(1 ? (ptr) : &((type *)0)->member) - offsetof(type, member))) +/* Use this variant when the member is an array */ +#define ErtsContainerStruct_(ptr, type, memberv) \ + ((type *)((char *)(1 ? (ptr) : ((type *)0)->memberv) - offsetof(type, memberv))) + #if defined (__WIN32__) # include "erl_win_sys.h" #else -- cgit v1.2.3 From 412b5aaa5810cbac75ef41d29a36898ff1480129 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 24 Aug 2016 11:46:34 +0200 Subject: Replace misspelled symbolic time units Besides using two words for 'milliseconds' et. al. they are also changed from plural to singular. --- erts/emulator/beam/atom.names | 4 ++++ erts/emulator/beam/erl_time_sup.c | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index badd69856e..263c7d4ac2 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -384,8 +384,10 @@ atom merge_trap atom meta atom meta_match_spec atom micro_seconds +atom microsecond atom microstate_accounting atom milli_seconds +atom millisecond atom min_heap_size atom min_bin_vheap_size atom minor_version @@ -402,6 +404,7 @@ atom more atom multi_scheduling atom multiline atom nano_seconds +atom nanosecond atom name atom named_table atom namelist @@ -554,6 +557,7 @@ atom schedulers_online atom scheme atom scientific atom scope +atom second atom seconds atom send_to_non_existing_process atom sensitive diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index 9e37106b88..6aa2a7500f 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -2133,22 +2133,26 @@ time_unit_conversion(Process *c_p, Eterm term, ErtsMonotonicTime val, ErtsMonoto /* Convert to common user specified time units */ switch (term) { + case am_second: case am_seconds: case make_small(1): result = ERTS_MONOTONIC_TO_SEC(val) + muloff*ERTS_MONOTONIC_OFFSET_SEC; ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); break; + case am_millisecond: case am_milli_seconds: case make_small(1000): result = ERTS_MONOTONIC_TO_MSEC(val) + muloff*ERTS_MONOTONIC_OFFSET_MSEC; ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); break; + case am_microsecond: case am_micro_seconds: case make_small(1000*1000): result = ERTS_MONOTONIC_TO_USEC(val) + muloff*ERTS_MONOTONIC_OFFSET_USEC; ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); break; #ifdef ARCH_64 + case am_nanosecond: case am_nano_seconds: case make_small(1000*1000*1000): result = ERTS_MONOTONIC_TO_NSEC(val) + muloff*ERTS_MONOTONIC_OFFSET_NSEC; @@ -2159,7 +2163,7 @@ time_unit_conversion(Process *c_p, Eterm term, ErtsMonotonicTime val, ErtsMonoto Eterm value, native_res; #ifndef ARCH_64 Sint user_res; - if (term == am_nano_seconds) + if (term == am_nanosecond || term == am_nano_seconds) goto to_nano_seconds; if (term_to_Sint(term, &user_res)) { if (user_res == 1000*1000*1000) { -- cgit v1.2.3 From 2fe03e832adb11c50bcfc62679cf17779b284124 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 27 Jul 2016 19:45:41 +0200 Subject: Reclaim literal area after purge has completed --- erts/emulator/beam/atom.names | 2 + erts/emulator/beam/beam_bif_load.c | 478 ++++++++++++++++++++++++++++++++++-- erts/emulator/beam/beam_load.h | 16 +- erts/emulator/beam/bif.tab | 2 + erts/emulator/beam/erl_alloc.types | 1 + erts/emulator/beam/erl_init.c | 28 ++- erts/emulator/beam/erl_lock_check.c | 3 + erts/emulator/beam/erl_process.c | 44 +++- erts/emulator/beam/global.h | 37 ++- 9 files changed, 560 insertions(+), 51 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index badd69856e..dd37c682b1 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -176,6 +176,7 @@ atom const atom context_switches atom control atom copy +atom copy_literals atom counters atom cpu atom cpu_timestamp @@ -407,6 +408,7 @@ atom named_table atom namelist atom native atom native_addresses +atom need_gc atom Neq='=/=' atom Neqeq='/=' atom net_kernel diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index a94a3cc3d3..484198b339 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -37,12 +37,41 @@ #include "erl_bits.h" #include "erl_thr_progress.h" +Process *erts_code_purger = NULL; + +ErtsLiteralArea *erts_copy_literal_area = NULL; +#ifdef ERTS_NEW_PURGE_STRATEGY +Process *erts_literal_area_collector = NULL; + +typedef struct ErtsLiteralAreaRef_ ErtsLiteralAreaRef; +struct ErtsLiteralAreaRef_ { + ErtsLiteralAreaRef *next; + ErtsLiteralArea *literal_area; +}; + +struct { + erts_smp_mtx_t mtx; + ErtsLiteralAreaRef *first; + ErtsLiteralAreaRef *last; +} release_literal_areas; +#endif + static void set_default_trace_pattern(Eterm module); static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls); static void delete_code(Module* modp); static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); +void +erts_beam_bif_load_init(void) +{ +#ifdef ERTS_NEW_PURGE_STRATEGY + erts_smp_mtx_init(&release_literal_areas.mtx, "release_literal_areas"); + release_literal_areas.first = NULL; + release_literal_areas.last = NULL; +#endif +} + BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) { Module* modp; @@ -767,6 +796,311 @@ static void hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp, Eterm *start, Eterm *end, char *lit_start, Uint lit_size); +#ifdef ERTS_NEW_PURGE_STRATEGY + +Eterm +erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed) +{ + ErtsLiteralArea *la; + ErtsMessage *msgp; + struct erl_off_heap_header* oh; + char *literals; + Uint lit_bsize; + ErlHeapFragment *hfrag; + + la = erts_copy_literal_area; + if (!la) + return am_ok; + + oh = la->off_heap; + literals = (char *) &la->start[0]; + lit_bsize = (char *) la->end - literals; + + /* + * If a literal is in the message queue we make an explicit copy of + * it and attach it to the heap fragment. Each message needs to be + * self contained, we cannot save the literal in the old_heap or + * any other heap than the message it self. + */ + + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ); + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ); + + for (msgp = c_p->msg.first; msgp; msgp = msgp->next) { + ErlHeapFragment *hf; + Uint lit_sz = 0; + + *redsp += 1; + + if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) + hfrag = &msgp->hfrag; + else if (is_value(ERL_MESSAGE_TERM(msgp)) && msgp->data.heap_frag) + hfrag = msgp->data.heap_frag; + else + continue; /* Content on heap or in external term format... */ + + for (hf = hfrag; hf; hf = hf->next) { + lit_sz += hfrag_literal_size(&hf->mem[0], &hf->mem[hf->used_size], + literals, lit_bsize); + *redsp += 1; + } + + *redsp += lit_sz / 16; /* Better value needed... */ + if (lit_sz > 0) { + ErlHeapFragment *bp = new_message_buffer(lit_sz); + Eterm *hp = bp->mem; + + for (hf = hfrag; hf; hf = hf->next) { + hfrag_literal_copy(&hp, &bp->off_heap, + &hf->mem[0], &hf->mem[hf->used_size], + literals, lit_bsize); + hfrag = hf; + } + + /* link new hfrag last */ + ASSERT(hfrag->next == NULL); + hfrag->next = bp; + bp->next = NULL; + } + } + + if (gc_allowed) { + /* + * Current implementation first tests without + * allowing GC, and then restarts the operation + * allowing GC if it is needed. It is therfore + * very likely that we will need the GC (although + * this is not completely certain). We go for + * the GC directly instead of scanning everything + * one more time... + */ + goto literal_gc; + } + + *redsp += 2; + if (any_heap_ref_ptrs(&c_p->fvalue, &c_p->fvalue+1, literals, lit_bsize)) { + c_p->freason = EXC_NULL; + c_p->fvalue = NIL; + c_p->ftrace = NIL; + } + + if (any_heap_ref_ptrs(c_p->stop, c_p->hend, literals, lit_bsize)) + goto literal_gc; + *redsp += 1; + if (any_heap_refs(c_p->heap, c_p->htop, literals, lit_bsize)) + goto literal_gc; + *redsp += 1; + if (any_heap_refs(c_p->old_heap, c_p->old_htop, literals, lit_bsize)) + goto literal_gc; + + /* Check dictionary */ + *redsp += 1; + if (c_p->dictionary) { + Eterm* start = ERTS_PD_START(c_p->dictionary); + Eterm* end = start + ERTS_PD_SIZE(c_p->dictionary); + + if (any_heap_ref_ptrs(start, end, literals, lit_bsize)) + goto literal_gc; + } + + /* Check heap fragments */ + for (hfrag = c_p->mbuf; hfrag; hfrag = hfrag->next) { + Eterm *hp, *hp_end; + + *redsp += 1; + + hp = &hfrag->mem[0]; + hp_end = &hfrag->mem[hfrag->used_size]; + if (any_heap_refs(hp, hp_end, literals, lit_bsize)) + goto literal_gc; + } + + /* + * Message buffer fragments (matched messages) + * - off heap lists should already have been moved into + * process off heap structure. + * - Check for literals + */ + for (msgp = c_p->msg_frag; msgp; msgp = msgp->next) { + hfrag = erts_message_to_heap_frag(msgp); + for (; hfrag; hfrag = hfrag->next) { + Eterm *hp, *hp_end; + ASSERT(!check_mod_funs(c_p, &hfrag->off_heap, mod_start, mod_size)); + + *redsp += 1; + + hp = &hfrag->mem[0]; + hp_end = &hfrag->mem[hfrag->used_size]; + + if (any_heap_refs(hp, hp_end, literals, lit_bsize)) + goto literal_gc; + } + } + + return am_ok; + +literal_gc: + + if (!gc_allowed) + return am_need_gc; + + if (c_p->flags & F_DISABLE_GC) + return THE_NON_VALUE; + + FLAGS(c_p) |= F_NEED_FULLSWEEP; + + *redsp += erts_garbage_collect_nobump(c_p, 0, c_p->arg_reg, c_p->arity, fcalls); + + erts_garbage_collect_literals(c_p, (Eterm *) literals, lit_bsize, oh); + + *redsp += lit_bsize / 64; /* Need, better value... */ + + return am_ok; +} + +static Eterm +check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls) +{ + BeamInstr* start; + char* mod_start; + Uint mod_size; + Eterm* sp; + int done_gc = 0; + ErtsMessage *msgp; + ErlHeapFragment *hfrag; + + /* + * Pick up limits for the module. + */ + start = (BeamInstr*) modp->old.code_hdr; + mod_start = (char *) start; + mod_size = modp->old.code_length; + + /* + * Check if current instruction or continuation pointer points into module. + */ + if (ErtsInArea(rp->i, mod_start, mod_size) + || ErtsInArea(rp->cp, mod_start, mod_size)) { + return am_true; + } + + /* + * Check all continuation pointers stored on the stack. + */ + for (sp = rp->stop; sp < STACK_START(rp); sp++) { + if (is_CP(*sp) && ErtsInArea(cp_val(*sp), mod_start, mod_size)) { + return am_true; + } + } + + /* + * Check all continuation pointers stored in stackdump + * and clear exception stackdump if there is a pointer + * to the module. + */ + if (rp->ftrace != NIL) { + struct StackTrace *s; + ASSERT(is_list(rp->ftrace)); + s = (struct StackTrace *) big_val(CDR(list_val(rp->ftrace))); + if ((s->pc && ErtsInArea(s->pc, mod_start, mod_size)) || + (s->current && ErtsInArea(s->current, mod_start, mod_size))) { + rp->freason = EXC_NULL; + rp->fvalue = NIL; + rp->ftrace = NIL; + } else { + int i; + for (i = 0; i < s->depth; i++) { + if (ErtsInArea(s->trace[i], mod_start, mod_size)) { + rp->freason = EXC_NULL; + rp->fvalue = NIL; + rp->ftrace = NIL; + break; + } + } + } + } + + /* + * Message queue can contains funs. + */ + + erts_smp_proc_lock(rp, ERTS_PROC_LOCK_MSGQ); + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ); + + for (msgp = rp->msg.first; msgp; msgp = msgp->next) { + ErlHeapFragment *hf; + if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) + hfrag = &msgp->hfrag; + else if (is_value(ERL_MESSAGE_TERM(msgp)) && msgp->data.heap_frag) + hfrag = msgp->data.heap_frag; + else + continue; + + for (hf = hfrag; hf; hf = hf->next) { + if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)) + return am_true; + } + } + + while (1) { + + /* Check heap, stack etc... */ + if (check_mod_funs(rp, &rp->off_heap, mod_start, mod_size)) + goto try_gc; + +#ifdef DEBUG + /* Check heap fragments */ + for (hfrag = rp->mbuf; hfrag; hfrag = hfrag->next) { + /* Off heap lists should already have been moved into process */ + ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)); + } + + /* + * Message buffer fragments (matched messages) + * - off heap lists should already have been moved into + * process off heap structure. + * - Check for literals + */ + for (msgp = rp->msg_frag; msgp; msgp = msgp->next) { + hfrag = erts_message_to_heap_frag(msgp); + for (; hfrag; hfrag = hfrag->next) { + Eterm *hp, *hp_end; + ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)); + } + } +#endif + + return am_false; + + try_gc: + + if (rp->flags & F_DISABLE_GC) { + /* + * Cannot proceed. Process has disabled gc in order to + * safely leave inconsistent data on the heap and/or + * off heap lists. Need to wait for gc to be enabled + * again. + */ + return THE_NON_VALUE; + } + + if (done_gc) + return am_true; + + if (!(flags & ERTS_CPC_ALLOW_GC)) + return am_aborted; + + FLAGS(rp) |= F_NEED_FULLSWEEP; + *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity, fcalls); + done_gc = 1; + } + +} + +#else /* !ERTS_NEW_PURGE_STRATEGY, i.e, old style purge... */ + static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls) { @@ -860,8 +1194,14 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ); - literals = (char*) modp->old.code_hdr->literal_area->start; - lit_bsize = (char*) modp->old.code_hdr->literal_area->end - literals; + if (modp->old.code_hdr->literal_area) { + literals = (char*) modp->old.code_hdr->literal_area->start; + lit_bsize = (char*) modp->old.code_hdr->literal_area->end - literals; + } + else { + literals = NULL; + lit_bsize = 0; + } for (msgp = rp->msg.first; msgp; msgp = msgp->next) { if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) @@ -902,12 +1242,6 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls /* Check heap, stack etc... */ if (check_mod_funs(rp, &rp->off_heap, mod_start, mod_size)) goto try_gc; - if (!(flags & ERTS_CPC_COPY_LITERALS)) { - /* Process ok. May contain old literals but we will be called - * again before module is purged. - */ - return am_false; - } if (any_heap_ref_ptrs(&rp->fvalue, &rp->fvalue+1, literals, lit_bsize)) { rp->freason = EXC_NULL; rp->fvalue = NIL; @@ -1006,6 +1340,8 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls } +#endif /* !ERTS_NEW_PURGE_STRATEGY */ + static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) { @@ -1136,12 +1472,73 @@ hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp, #undef in_area +#ifdef ERTS_NEW_PURGE_STRATEGY + +ErtsThrPrgrLaterOp later_literal_area_switch; + +static void +complete_literal_area_switch(void *unused) +{ + Process *p = erts_literal_area_collector; + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_resume(p, ERTS_PROC_LOCK_STATUS); + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); +} + +#endif /* ERTS_NEW_PURGE_STRATEGY */ + +BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0) +{ +#ifndef ERTS_NEW_PURGE_STRATEGY + BIF_ERROR(BIF_P, EXC_NOTSUP); +#else + ErtsLiteralAreaRef *la_ref; + + if (BIF_P != erts_literal_area_collector) + BIF_ERROR(BIF_P, EXC_NOTSUP); + + erts_smp_mtx_lock(&release_literal_areas.mtx); + + la_ref = release_literal_areas.first; + if (la_ref) { + release_literal_areas.first = la_ref->next; + if (!release_literal_areas.first) + release_literal_areas.last = NULL; + } + + erts_smp_mtx_unlock(&release_literal_areas.mtx); + + if (erts_copy_literal_area) + erts_release_literal_area(erts_copy_literal_area); + + if (!la_ref) { + erts_copy_literal_area = NULL; + BIF_RET(am_false); + } + + erts_copy_literal_area = la_ref->literal_area; + + erts_free(ERTS_ALC_T_LITERAL_REF, la_ref); + +#ifndef ERTS_SMP + BIF_RET(am_true); +#else + erts_schedule_thr_prgr_later_op(complete_literal_area_switch, + NULL, + &later_literal_area_switch); + erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); + ERTS_BIF_YIELD_RETURN(BIF_P, am_true); +#endif + +#endif /* ERTS_NEW_PURGE_STRATEGY */ +} + +#ifndef ERTS_NEW_PURGE_STRATEGY + #ifdef ERTS_SMP static void copy_literals_commit(void*); #endif -copy_literals_t erts_clrange = {NULL, 0, THE_NON_VALUE}; - /* copy literals * * copy_literals.ptr = LitPtr @@ -1157,12 +1554,19 @@ copy_literals_t erts_clrange = {NULL, 0, THE_NON_VALUE}; * ... */ +#endif BIF_RETTYPE erts_internal_copy_literals_2(BIF_ALIST_2) { +#ifdef ERTS_NEW_PURGE_STRATEGY + BIF_ERROR(BIF_P, EXC_NOTSUP); +#else ErtsCodeIndex code_ix; Eterm res = am_true; + if (BIF_P != erts_code_purger) + BIF_ERROR(BIF_P, EXC_NOTSUP); + if (is_not_atom(BIF_ARG_1) || (am_true != BIF_ARG_2 && am_false != BIF_ARG_2)) { BIF_ERROR(BIF_P, BADARG); } @@ -1180,22 +1584,13 @@ BIF_RETTYPE erts_internal_copy_literals_2(BIF_ALIST_2) res = am_false; goto done; } - if (erts_clrange.ptr != NULL - && !(BIF_P->static_flags & ERTS_STC_FLG_SYSTEM_PROC)) { + if (erts_copy_literal_area) { res = am_aborted; goto done; } - erts_clrange.ptr = modp->old.code_hdr->literal_area->start; - erts_clrange.sz = modp->old.code_hdr->literal_area->end - erts_clrange.ptr; - erts_clrange.pid = BIF_P->common.id; + erts_copy_literal_area = modp->old.code_hdr->literal_area; } else if (BIF_ARG_2 == am_false) { - if (erts_clrange.pid != BIF_P->common.id) { - res = am_false; - goto done; - } - erts_clrange.ptr = NULL; - erts_clrange.sz = 0; - erts_clrange.pid = THE_NON_VALUE; + erts_copy_literal_area = NULL; } #ifdef ERTS_SMP @@ -1209,8 +1604,11 @@ BIF_RETTYPE erts_internal_copy_literals_2(BIF_ALIST_2) done: erts_release_code_write_permission(); BIF_RET(res); +#endif } +#ifndef ERTS_NEW_PURGE_STRATEGY + #ifdef ERTS_SMP static void copy_literals_commit(void* null) { Process* p = committer_state.stager; @@ -1227,6 +1625,7 @@ static void copy_literals_commit(void* null) { } #endif /* ERTS_SMP */ +#endif /* Do the actualy module purging and return: * true for success @@ -1241,6 +1640,7 @@ BIF_RETTYPE erts_internal_purge_module_1(BIF_ALIST_1) Module* modp; int is_blocking = 0; Eterm ret; + ErtsLiteralArea *literals = NULL; if (is_not_atom(BIF_ARG_1)) { BIF_ERROR(BIF_P, BADARG); @@ -1294,7 +1694,7 @@ BIF_RETTYPE erts_internal_purge_module_1(BIF_ALIST_1) erts_cleanup_funs_on_purge(code, end); beam_catches_delmod(modp->old.catches, code, modp->old.code_length, code_ix); - erts_release_literal_area(modp->old.code_hdr->literal_area); + literals = modp->old.code_hdr->literal_area; modp->old.code_hdr->literal_area = NULL; erts_free(ERTS_ALC_T_CODE, (void *) code); modp->old.code_hdr = NULL; @@ -1310,6 +1710,38 @@ BIF_RETTYPE erts_internal_purge_module_1(BIF_ALIST_1) erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); } erts_release_code_write_permission(); + +#ifdef ERTS_NEW_PURGE_STRATEGY + + if (literals) { + ErtsLiteralAreaRef *ref; + ref = erts_alloc(ERTS_ALC_T_LITERAL_REF, + sizeof(ErtsLiteralAreaRef)); + ref->literal_area = literals; + ref->next = NULL; + erts_smp_mtx_lock(&release_literal_areas.mtx); + if (release_literal_areas.last) { + release_literal_areas.last->next = ref; + release_literal_areas.last = ref; + } + else { + release_literal_areas.first = ref; + release_literal_areas.last = ref; + } + erts_smp_mtx_unlock(&release_literal_areas.mtx); + erts_queue_message(erts_literal_area_collector, + 0, + erts_alloc_message(0, NULL), + am_copy_literals, + BIF_P->common.id); + } + +#else /* !ERTS_NEW_PURGE_STRATEGY */ + + erts_release_literal_area(literals); + +#endif + return ret; } diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 336f097f0f..d52f12e6d0 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -50,6 +50,7 @@ extern BeamInstr* em_call_error_handler; extern BeamInstr* em_apply_bif; extern BeamInstr* em_call_nif; +struct ErtsLiteralArea_; /* * The following variables keep a sorted list of address ranges for @@ -60,15 +61,6 @@ extern BeamInstr* em_call_nif; /* Total code size in bytes */ extern Uint erts_total_code_size; -typedef struct { - struct erl_off_heap_header *off_heap; - Eterm *end; - Eterm start[1]; /* beginning of area */ -} ErtsLiteralArea; - -#define ERTS_LITERAL_AREA_ALLOC_SIZE(N) \ - (sizeof(ErtsLiteralArea) + sizeof(Eterm)*((N) - 1)) - typedef struct BeamCodeLineTab_ BeamCodeLineTab; /* @@ -98,7 +90,7 @@ typedef struct beam_code_header { /* * Literal area (constant pool). */ - ErtsLiteralArea *literal_area; + struct ErtsLiteralArea_ *literal_area; /* * Pointer to the on_load function (or NULL if none). @@ -127,9 +119,11 @@ typedef struct beam_code_header { }BeamCodeHeader; -void erts_release_literal_area(ErtsLiteralArea* literal_area); +void erts_release_literal_area(struct ErtsLiteralArea_* literal_area); int erts_is_module_native(BeamCodeHeader* code); +void erts_beam_bif_load_init(void); + /* * Layout of the line table. */ diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 065018514a..7ffcb26567 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -174,6 +174,8 @@ bif erts_internal:is_system_process/1 bif erts_internal:system_check/1 +bif erts_internal:release_literal_area_switch/0 + # inet_db support bif erlang:port_set_data/2 bif erlang:port_get_data/1 diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 227fedfb69..0a4a34a174 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -367,6 +367,7 @@ type MONITOR_LH STANDARD PROCESSES monitor_lh type NLINK_LH STANDARD PROCESSES nlink_lh type CODE LONG_LIVED CODE code type LITERAL LITERAL CODE literal +type LITERAL_REF SHORT_LIVED CODE literal_area_ref type DB_HEIR_DATA STANDARD ETS db_heir_data type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index fbdafec4ef..788dbc3328 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -384,6 +384,7 @@ erl_init(int ncpu, erts_init_unicode(); /* after RE to get access to PCRE unicode */ erts_init_external(); erts_init_map(); + erts_beam_bif_load_init(); erts_delay_trap = erts_export_put(am_erlang, am_delay_trap, 2); erts_late_init_process(); #if HAVE_ERTS_MSEG @@ -2250,7 +2251,27 @@ erl_start(int argc, char **argv) otp_ring0_pid = erl_first_process_otp("otp_ring0", NULL, 0, boot_argc, boot_argv); - (void) erl_system_process_otp(otp_ring0_pid, "erts_code_purger"); + { + /* + * The erts_code_purger and the erts_literal_area_collector + * system processes are *always* alive. If they terminate + * they bring the whole VM down. + */ + Eterm pid; + + pid = erl_system_process_otp(otp_ring0_pid, "erts_code_purger"); + erts_code_purger = erts_proc_lookup(pid); + ASSERT(erts_code_purger); + erts_proc_inc_refc(erts_code_purger); + +#ifdef ERTS_NEW_PURGE_STRATEGY + pid = erl_system_process_otp(otp_ring0_pid, "erts_literal_area_collector"); + erts_literal_area_collector = erts_proc_lookup(pid); + ASSERT(erts_literal_area_collector); + erts_proc_inc_refc(erts_literal_area_collector); +#endif + + } #ifdef ERTS_SMP erts_start_schedulers(); @@ -2350,14 +2371,15 @@ erts_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) if (erts_mtrace_enabled) erts_mtrace_exit((Uint32) n); + if (fmt != NULL && *fmt != '\0') + erl_error(fmt, args2); /* Print error message. */ + /* Produce an Erlang core dump if error */ if (((n == ERTS_ERROR_EXIT && erts_no_crash_dump == 0) || n == ERTS_DUMP_EXIT) && erts_initialized) { erl_crash_dump_v((char*) NULL, 0, fmt, args1); } - if (fmt != NULL && *fmt != '\0') - erl_error(fmt, args2); /* Print error message. */ sys_tty_reset(n); if (n == ERTS_INTR_EXIT) diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 39c0617143..4192cf389f 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -112,6 +112,9 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "export_tab", NULL }, { "fun_tab", NULL }, { "environ", NULL }, +#ifdef ERTS_NEW_PURGE_STRATEGY + { "release_literal_areas", NULL }, +#endif #endif { "efile_drv", "address" }, { "drv_ev_state_grow", NULL, }, diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 66f22979ad..42c30b935e 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -448,6 +448,9 @@ int erts_system_profile_ts_type = ERTS_TRACE_FLG_NOW_TIMESTAMP; typedef enum { ERTS_PSTT_GC, /* Garbage Collect */ ERTS_PSTT_CPC, /* Check Process Code */ +#ifdef ERTS_NEW_PURGE_STRATEGY + ERTS_PSTT_CLA, /* Copy Literal Area */ +#endif ERTS_PSTT_COHMQ, /* Change off heap message queue */ ERTS_PSTT_FTMQ /* Flush trace msg queue */ } ErtsProcSysTaskType; @@ -10393,6 +10396,27 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) } break; } +#ifdef ERTS_NEW_PURGE_STRATEGY + case ERTS_PSTT_CLA: { + int fcalls; + int cla_reds = 0; + if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) + fcalls = reds; + else + fcalls = reds - CONTEXT_REDS; + st_res = erts_proc_copy_literal_area(c_p, + &cla_reds, + fcalls, + st->arg[0] == am_true); + reds -= cla_reds; + if (is_non_value(st_res)) { + /* Needed gc, but gc was disabled */ + save_gc_task(c_p, st, st_prio); + st = NULL; + } + break; + } +#endif case ERTS_PSTT_COHMQ: reds -= erts_complete_off_heap_message_queue_change(c_p); st_res = am_true; @@ -10443,14 +10467,15 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) switch (st->type) { case ERTS_PSTT_GC: - st_res = am_false; - break; case ERTS_PSTT_CPC: - st_res = am_false; - break; case ERTS_PSTT_COHMQ: st_res = am_false; break; +#ifdef ERTS_NEW_PURGE_STRATEGY + case ERTS_PSTT_CLA: + st_res = am_ok; + break; +#endif #ifdef ERTS_SMP case ERTS_PSTT_FTMQ: reds -= erts_flush_trace_messages(c_p, ERTS_PROC_LOCK_MAIN); @@ -10572,6 +10597,17 @@ erts_internal_request_system_task_3(BIF_ALIST_3) goto noproc; break; +#ifdef ERTS_NEW_PURGE_STRATEGY + case am_copy_literals: + if (st->arg[0] != am_true && st->arg[0] != am_false) + goto badarg; + st->type = ERTS_PSTT_CLA; + noproc_res = am_ok; + if (!rp) + goto noproc; + break; +#endif + default: goto badarg; } diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index f3d4ac56cd..00dfc421d5 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -999,17 +999,27 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2); /* beam_bif_load.c */ #define ERTS_CPC_ALLOW_GC (1 << 0) -#define ERTS_CPC_COPY_LITERALS (1 << 1) -#define ERTS_CPC_ALL (ERTS_CPC_ALLOW_GC | ERTS_CPC_COPY_LITERALS) +#define ERTS_CPC_ALL ERTS_CPC_ALLOW_GC Eterm erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls); +#ifdef ERTS_NEW_PURGE_STRATEGY +Eterm erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed); +#endif -typedef struct { - Eterm *ptr; - Uint sz; - Eterm pid; -} copy_literals_t; +typedef struct ErtsLiteralArea_ { + struct erl_off_heap_header *off_heap; + Eterm *end; + Eterm start[1]; /* beginning of area */ +} ErtsLiteralArea; -extern copy_literals_t erts_clrange; +#define ERTS_LITERAL_AREA_ALLOC_SIZE(N) \ + (sizeof(ErtsLiteralArea) + sizeof(Eterm)*((N) - 1)) + +extern ErtsLiteralArea *erts_copy_literal_area; +#ifdef ERTS_NEW_PURGE_STRATEGY +extern Process *erts_literal_area_collector; +#endif + +extern Process *erts_code_purger; /* beam_load.c */ typedef struct { @@ -1092,12 +1102,19 @@ typedef struct { #define INITIALIZE_SHCOPY(info) \ do { \ + ErtsLiteralArea *larea__ = erts_copy_literal_area; \ info.queue_start = info.queue_default; \ info.bitstore_start = info.bitstore_default; \ info.shtable_start = info.shtable_default; \ info.literal_size = 0; \ - info.range_ptr = erts_clrange.ptr; \ - info.range_sz = erts_clrange.sz; \ + if (larea__) { \ + info.range_ptr = &larea__->start[0]; \ + info.range_sz = larea__->end - info.range_ptr; \ + } \ + else { \ + info.range_ptr = NULL; \ + info.range_sz = 0; \ + } \ } while(0) #define DESTROY_SHCOPY(info) \ -- cgit v1.2.3 From 9d0638216d35ca0f21c1eea20f8daa3992ac4f71 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 2 Aug 2016 15:58:06 +0200 Subject: Fix purge of code Ensure that we cannot get any dangling pointers into code that has been purged. This is done by a two phase purge. At first phase all fun entries pointing into the code to purge are marked for purge. All processes trying to call these funs will be suspended and by this we avoid getting new direct references into the code. When all processes has been checked, these processes are resumed. The new purge strategy now also completely ignore the existence of indirect references to the code (funs). If such exist, they will cause bad fun exceptions to the caller, but will not prevent a soft purge or cause a kill of a process having such live references during a hard purge. This since it is impossible to give any guarantees that no processes in the system have such indirect references. Even when the system is completely clean from such references, new ones can appear via distribution and/or disk. --- erts/emulator/beam/atom.names | 5 + erts/emulator/beam/beam_bif_load.c | 614 ++++++++++++++++++++++-------------- erts/emulator/beam/beam_emu.c | 49 ++- erts/emulator/beam/beam_load.h | 4 +- erts/emulator/beam/bif.tab | 3 +- erts/emulator/beam/erl_alloc.types | 1 + erts/emulator/beam/erl_bif_info.c | 21 ++ erts/emulator/beam/erl_fun.c | 53 +++- erts/emulator/beam/erl_fun.h | 6 +- erts/emulator/beam/erl_init.c | 13 +- erts/emulator/beam/erl_lock_check.c | 1 + 11 files changed, 489 insertions(+), 281 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index dd37c682b1..ace3dc5230 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -65,6 +65,7 @@ atom undefined_lambda atom DOWN='DOWN' atom UP='UP' atom EXIT='EXIT' +atom abort atom aborted atom abs_path atom absoluteURI @@ -165,6 +166,7 @@ atom commandv atom compact atom compat_rel atom compile +atom complete atom compressed atom config_h atom convert_time_unit @@ -236,6 +238,7 @@ atom erlang atom ERROR='ERROR' atom error_handler atom error_logger +atom erts_code_purger atom erts_internal atom ets atom ETS_TRANSFER='ETS-TRANSFER' @@ -487,6 +490,7 @@ atom pause atom pending atom pending_driver atom pending_process +atom pending_purge_lambda atom pending_reload atom permanent atom pid @@ -496,6 +500,7 @@ atom port_count atom port_limit atom port_op atom positive +atom prepare atom print atom priority atom private diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 484198b339..2413ccdba9 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -37,6 +37,20 @@ #include "erl_bits.h" #include "erl_thr_progress.h" +static struct { + Eterm module; + erts_smp_mtx_t mtx; + Export *pending_purge_lambda; + Eterm *sprocs; + Eterm def_sprocs[10]; + Uint sp_size; + Uint sp_ix; + ErlFunEntry **funs; + ErlFunEntry *def_funs[10]; + Uint fe_size; + Uint fe_ix; +} purge_state; + Process *erts_code_purger = NULL; ErtsLiteralArea *erts_copy_literal_area = NULL; @@ -62,6 +76,27 @@ static void delete_code(Module* modp); static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); +static void +init_purge_state(void) +{ + purge_state.module = THE_NON_VALUE; + + erts_smp_mtx_init(&purge_state.mtx, "purge_state"); + + purge_state.pending_purge_lambda = + erts_export_put(am_erts_code_purger, am_pending_purge_lambda, 3); + + purge_state.sprocs = &purge_state.def_sprocs[0]; + purge_state.sp_size = sizeof(purge_state.def_sprocs); + purge_state.sp_size /= sizeof(purge_state.def_sprocs[0]); + purge_state.sp_ix = 0; + + purge_state.funs = &purge_state.def_funs[0]; + purge_state.fe_size = sizeof(purge_state.def_funs); + purge_state.fe_size /= sizeof(purge_state.def_funs[0]); + purge_state.fe_ix = 0; +} + void erts_beam_bif_load_init(void) { @@ -70,6 +105,8 @@ erts_beam_bif_load_init(void) release_literal_areas.first = NULL; release_literal_areas.last = NULL; #endif + + init_purge_state(); } BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) @@ -776,6 +813,8 @@ set_default_trace_pattern(Eterm module) } } +#ifndef ERTS_NEW_PURGE_STRATEGY + static ERTS_INLINE int check_mod_funs(Process *p, ErlOffHeap *off_heap, char *area, size_t area_size) { @@ -790,6 +829,8 @@ check_mod_funs(Process *p, ErlOffHeap *off_heap, char *area, size_t area_size) return 0; } +#endif + static Uint hfrag_literal_size(Eterm* start, Eterm* end, char* lit_start, Uint lit_size); static void hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp, @@ -926,7 +967,6 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed hfrag = erts_message_to_heap_frag(msgp); for (; hfrag; hfrag = hfrag->next) { Eterm *hp, *hp_end; - ASSERT(!check_mod_funs(c_p, &hfrag->off_heap, mod_start, mod_size)); *redsp += 1; @@ -966,9 +1006,8 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls char* mod_start; Uint mod_size; Eterm* sp; - int done_gc = 0; - ErtsMessage *msgp; - ErlHeapFragment *hfrag; + + *redsp += 1; /* * Pick up limits for the module. @@ -985,6 +1024,8 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls return am_true; } + *redsp += (STACK_START(rp) - rp->stop) / 32; + /* * Check all continuation pointers stored on the stack. */ @@ -1021,82 +1062,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls } } - /* - * Message queue can contains funs. - */ - - erts_smp_proc_lock(rp, ERTS_PROC_LOCK_MSGQ); - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ); - - for (msgp = rp->msg.first; msgp; msgp = msgp->next) { - ErlHeapFragment *hf; - if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) - hfrag = &msgp->hfrag; - else if (is_value(ERL_MESSAGE_TERM(msgp)) && msgp->data.heap_frag) - hfrag = msgp->data.heap_frag; - else - continue; - - for (hf = hfrag; hf; hf = hf->next) { - if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)) - return am_true; - } - } - - while (1) { - - /* Check heap, stack etc... */ - if (check_mod_funs(rp, &rp->off_heap, mod_start, mod_size)) - goto try_gc; - -#ifdef DEBUG - /* Check heap fragments */ - for (hfrag = rp->mbuf; hfrag; hfrag = hfrag->next) { - /* Off heap lists should already have been moved into process */ - ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)); - } - - /* - * Message buffer fragments (matched messages) - * - off heap lists should already have been moved into - * process off heap structure. - * - Check for literals - */ - for (msgp = rp->msg_frag; msgp; msgp = msgp->next) { - hfrag = erts_message_to_heap_frag(msgp); - for (; hfrag; hfrag = hfrag->next) { - Eterm *hp, *hp_end; - ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)); - } - } -#endif - - return am_false; - - try_gc: - - if (rp->flags & F_DISABLE_GC) { - /* - * Cannot proceed. Process has disabled gc in order to - * safely leave inconsistent data on the heap and/or - * off heap lists. Need to wait for gc to be enabled - * again. - */ - return THE_NON_VALUE; - } - - if (done_gc) - return am_true; - - if (!(flags & ERTS_CPC_ALLOW_GC)) - return am_aborted; - - FLAGS(rp) |= F_NEED_FULLSWEEP; - *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity, fcalls); - done_gc = 1; - } - + return am_false; } #else /* !ERTS_NEW_PURGE_STRATEGY, i.e, old style purge... */ @@ -1470,12 +1436,11 @@ hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp, } } -#undef in_area - #ifdef ERTS_NEW_PURGE_STRATEGY ErtsThrPrgrLaterOp later_literal_area_switch; +#ifdef ERTS_SMP static void complete_literal_area_switch(void *unused) { @@ -1484,6 +1449,7 @@ complete_literal_area_switch(void *unused) erts_resume(p, ERTS_PROC_LOCK_STATUS); erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); } +#endif #endif /* ERTS_NEW_PURGE_STRATEGY */ @@ -1533,216 +1499,376 @@ BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0) #endif /* ERTS_NEW_PURGE_STRATEGY */ } -#ifndef ERTS_NEW_PURGE_STRATEGY +void +erts_purge_state_add_fun(ErlFunEntry *fe) +{ + ASSERT(is_value(purge_state.module)); + if (purge_state.fe_ix >= purge_state.fe_size) { + ErlFunEntry **funs; + purge_state.fe_size += 100; + funs = erts_alloc(ERTS_ALC_T_PURGE_DATA, + sizeof(ErlFunEntry *)*purge_state.fe_size); + sys_memcpy((void *) funs, + (void *) purge_state.funs, + purge_state.fe_ix*sizeof(ErlFunEntry *)); + if (purge_state.funs != &purge_state.def_funs[0]) + erts_free(ERTS_ALC_T_PURGE_DATA, purge_state.funs); + purge_state.funs = funs; + } + purge_state.funs[purge_state.fe_ix++] = fe; +} -#ifdef ERTS_SMP -static void copy_literals_commit(void*); -#endif +Export * +erts_suspend_process_on_pending_purge_lambda(Process *c_p) +{ + erts_smp_mtx_lock(&purge_state.mtx); + if (is_value(purge_state.module)) { + /* + * The process c_p is about to call a fun in the code + * that we are trying to purge. Suspend it and call + * erts_code_purger:pending_purge_lambda/3. The process + * will be resumed when the purge completes or aborts, + * and will then try to do the call again. + */ + if (purge_state.sp_ix >= purge_state.sp_size) { + Eterm *sprocs; + purge_state.sp_size += 100; + sprocs = erts_alloc(ERTS_ALC_T_PURGE_DATA, + (sizeof(ErlFunEntry *) + * purge_state.sp_size)); + sys_memcpy((void *) sprocs, + (void *) purge_state.sprocs, + purge_state.sp_ix*sizeof(ErlFunEntry *)); + if (purge_state.sprocs != &purge_state.def_sprocs[0]) + erts_free(ERTS_ALC_T_PURGE_DATA, purge_state.sprocs); + purge_state.sprocs = sprocs; + } + purge_state.sprocs[purge_state.sp_ix++] = c_p->common.id; + erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); + ERTS_VBUMP_ALL_REDS(c_p); + } + erts_smp_mtx_unlock(&purge_state.mtx); + return purge_state.pending_purge_lambda; +} -/* copy literals - * - * copy_literals.ptr = LitPtr - * copy_literals.sz = LitSz - * ------ THR PROG COMMIT ----- - * - * - check process code - * - check process code - * ... - * copy_literals.ptr = NULL - * copy_literals.sz = 0 - * ------ THR PROG COMMIT ----- - * ... - */ +static void +finalize_purge_operation(Process *c_p, int succeded) +{ + Uint ix; -#endif + if (c_p) + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); -BIF_RETTYPE erts_internal_copy_literals_2(BIF_ALIST_2) -{ -#ifdef ERTS_NEW_PURGE_STRATEGY - BIF_ERROR(BIF_P, EXC_NOTSUP); -#else - ErtsCodeIndex code_ix; - Eterm res = am_true; + erts_smp_mtx_lock(&purge_state.mtx); - if (BIF_P != erts_code_purger) - BIF_ERROR(BIF_P, EXC_NOTSUP); + ASSERT(purge_state.module != THE_NON_VALUE); - if (is_not_atom(BIF_ARG_1) || (am_true != BIF_ARG_2 && am_false != BIF_ARG_2)) { - BIF_ERROR(BIF_P, BADARG); - } + purge_state.module = THE_NON_VALUE; - if (!erts_try_seize_code_write_permission(BIF_P)) { - ERTS_BIF_YIELD2(bif_export[BIF_erts_internal_copy_literals_2], - BIF_P, BIF_ARG_1, BIF_ARG_2); + /* + * Resume all processes that have tried to call + * funs in this code. + */ + for (ix = 0; ix < purge_state.sp_ix; ix++) { + Process *rp = erts_pid2proc(NULL, 0, + purge_state.sprocs[ix], + ERTS_PROC_LOCK_STATUS); + if (rp) { + erts_resume(rp, ERTS_PROC_LOCK_STATUS); + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + } } - code_ix = erts_active_code_ix(); + erts_smp_mtx_unlock(&purge_state.mtx); - if (BIF_ARG_2 == am_true) { - Module* modp = erts_get_module(BIF_ARG_1, code_ix); - if (!modp || !modp->old.code_hdr) { - res = am_false; - goto done; - } - if (erts_copy_literal_area) { - res = am_aborted; - goto done; - } - erts_copy_literal_area = modp->old.code_hdr->literal_area; - } else if (BIF_ARG_2 == am_false) { - erts_copy_literal_area = NULL; + if (c_p) + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + + if (purge_state.sprocs != &purge_state.def_sprocs[0]) { + erts_free(ERTS_ALC_T_PURGE_DATA, purge_state.sprocs); + purge_state.sprocs = &purge_state.def_sprocs[0]; + purge_state.sp_size = sizeof(purge_state.def_sprocs); + purge_state.sp_size /= sizeof(purge_state.def_sprocs[0]); } + purge_state.sp_ix = 0; -#ifdef ERTS_SMP - ASSERT(committer_state.stager == NULL); - committer_state.stager = BIF_P; - erts_schedule_thr_prgr_later_op(copy_literals_commit, NULL, &committer_state.lop); - erts_proc_inc_refc(BIF_P); - erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); - ERTS_BIF_YIELD_RETURN(BIF_P, am_true); -#endif -done: - erts_release_code_write_permission(); - BIF_RET(res); -#endif + if (purge_state.funs != &purge_state.def_funs[0]) { + erts_free(ERTS_ALC_T_PURGE_DATA, purge_state.funs); + purge_state.funs = &purge_state.def_funs[0]; + purge_state.fe_size = sizeof(purge_state.def_funs); + purge_state.fe_size /= sizeof(purge_state.def_funs[0]); + } + purge_state.fe_ix = 0; } -#ifndef ERTS_NEW_PURGE_STRATEGY - #ifdef ERTS_SMP -static void copy_literals_commit(void* null) { - Process* p = committer_state.stager; -#ifdef DEBUG - committer_state.stager = NULL; -#endif - erts_release_code_write_permission(); + +static ErtsThrPrgrLaterOp purger_lop_data; + +static void +resume_purger(void *unused) +{ + Process *p = erts_code_purger; erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); - if (!ERTS_PROC_IS_EXITING(p)) { - erts_resume(p, ERTS_PROC_LOCK_STATUS); - } + erts_resume(p, ERTS_PROC_LOCK_STATUS); erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - erts_proc_dec_refc(p); } -#endif /* ERTS_SMP */ -#endif +static void +finalize_purge_abort(void *unused) +{ + erts_fun_purge_abort_finalize(purge_state.funs, purge_state.fe_ix); -/* Do the actualy module purging and return: - * true for success - * false if no such old module - * BADARG if not an atom - */ -BIF_RETTYPE erts_internal_purge_module_1(BIF_ALIST_1) + finalize_purge_operation(NULL, 0); + + resume_purger(NULL); +} + +#endif /* ERTS_SMP */ + +BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) { - ErtsCodeIndex code_ix; - BeamInstr* code; - BeamInstr* end; - Module* modp; - int is_blocking = 0; - Eterm ret; - ErtsLiteralArea *literals = NULL; + if (BIF_P != erts_code_purger) + BIF_ERROR(BIF_P, EXC_NOTSUP); - if (is_not_atom(BIF_ARG_1)) { + if (is_not_atom(BIF_ARG_1)) BIF_ERROR(BIF_P, BADARG); - } - if (!erts_try_seize_code_write_permission(BIF_P)) { - ERTS_BIF_YIELD1(bif_export[BIF_erts_internal_purge_module_1], - BIF_P, BIF_ARG_1); - } + switch (BIF_ARG_2) { - code_ix = erts_active_code_ix(); + case am_prepare: { + /* + * Prepare for purge by marking all fun + * entries referring to the code to purge + * with "pending purge" markers. + */ + ErtsCodeIndex code_ix; + Module* modp; + Eterm res; - /* - * Correct module? - */ + if (is_value(purge_state.module)) + BIF_ERROR(BIF_P, BADARG); - if ((modp = erts_get_module(BIF_ARG_1, code_ix)) == NULL) { - ERTS_BIF_PREP_RET(ret, am_false); - } - else { - erts_rwlock_old_code(code_ix); + code_ix = erts_active_code_ix(); /* - * Any code to purge? + * Correct module? */ - if (!modp->old.code_hdr) { - ERTS_BIF_PREP_RET(ret, am_false); - } + modp = erts_get_module(BIF_ARG_1, code_ix); + if (!modp) + res = am_false; else { /* - * Unload any NIF library + * Any code to purge? */ - if (modp->old.nif != NULL) { - /* ToDo: Do unload nif without blocking */ - erts_rwunlock_old_code(code_ix); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); - is_blocking = 1; - erts_rwlock_old_code(code_ix); - erts_unload_nif(modp->old.nif); - modp->old.nif = NULL; + erts_rlock_old_code(code_ix); + if (!modp->old.code_hdr) + res = am_false; + else { + BeamInstr* code; + BeamInstr* end; + erts_smp_mtx_lock(&purge_state.mtx); + purge_state.module = BIF_ARG_1; + erts_smp_mtx_unlock(&purge_state.mtx); + res = am_true; + code = (BeamInstr*) modp->old.code_hdr; + end = (BeamInstr *)((char *)code + modp->old.code_length); + erts_fun_purge_prepare(code, end); +#if !defined(ERTS_NEW_PURGE_STRATEGY) + ASSERT(!erts_copy_literal_area); + erts_copy_literal_area = modp->old.code_hdr->literal_area; +#endif } - + erts_runlock_old_code(code_ix); + } + +#ifndef ERTS_SMP + BIF_RET(res); +#else + if (res != am_true) + BIF_RET(res); + else { /* - * Remove the old code. + * We'll be resumed when all schedulers are guaranteed + * to see the "pending purge" markers that we've made on + * all fun entries of the code that we are about to purge. + * Processes trying to call these funs will be suspended + * before calling the funs. That is we are guaranteed not + * to get any more direct references into the code while + * checking for such references... */ - ASSERT(erts_total_code_size >= modp->old.code_length); - erts_total_code_size -= modp->old.code_length; - code = (BeamInstr*) modp->old.code_hdr; - end = (BeamInstr *)((char *)code + modp->old.code_length); - erts_cleanup_funs_on_purge(code, end); - beam_catches_delmod(modp->old.catches, code, modp->old.code_length, - code_ix); - literals = modp->old.code_hdr->literal_area; - modp->old.code_hdr->literal_area = NULL; - erts_free(ERTS_ALC_T_CODE, (void *) code); - modp->old.code_hdr = NULL; - modp->old.code_length = 0; - modp->old.catches = BEAM_CATCHES_NIL; - erts_remove_from_ranges(code); - ERTS_BIF_PREP_RET(ret, am_true); + erts_schedule_thr_prgr_later_op(resume_purger, + NULL, + &purger_lop_data); + erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); + ERTS_BIF_YIELD_RETURN(BIF_P, am_true); } - erts_rwunlock_old_code(code_ix); +#endif } - if (is_blocking) { - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + + case am_abort: { + /* + * Soft purge that detected direct references into the code + * we set out to purge. Abort the purge. + */ + + if (purge_state.module != BIF_ARG_1) + BIF_ERROR(BIF_P, BADARG); + + erts_fun_purge_abort_prepare(purge_state.funs, purge_state.fe_ix); + +#if !defined(ERTS_NEW_PURGE_STRATEGY) + ASSERT(erts_copy_literal_area); + erts_copy_literal_area = NULL; +#endif +#ifndef ERTS_SMP + erts_fun_purge_abort_finalize(purge_state.funs, purge_state.fe_ix); + finalize_purge_operation(BIF_P, 0); + BIF_RET(am_false); +#else + /* + * We need to restore the code addresses of the funs in + * two stages in order to ensure that we do not get any + * stale suspended processes due to the purge abort. + * Restore address pointer (erts_fun_purge_abort_prepare); + * wait for thread progress; clear pending purge address + * pointer (erts_fun_purge_abort_finalize), and then + * resume processes that got suspended + * (finalize_purge_operation). + */ + erts_schedule_thr_prgr_later_op(finalize_purge_abort, + NULL, + &purger_lop_data); + erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); + ERTS_BIF_YIELD_RETURN(BIF_P, am_false); +#endif } - erts_release_code_write_permission(); -#ifdef ERTS_NEW_PURGE_STRATEGY + case am_complete: { + ErtsCodeIndex code_ix; + BeamInstr* code; + Module* modp; + int is_blocking = 0; + Eterm ret; + ErtsLiteralArea *literals = NULL; - if (literals) { - ErtsLiteralAreaRef *ref; - ref = erts_alloc(ERTS_ALC_T_LITERAL_REF, - sizeof(ErtsLiteralAreaRef)); - ref->literal_area = literals; - ref->next = NULL; - erts_smp_mtx_lock(&release_literal_areas.mtx); - if (release_literal_areas.last) { - release_literal_areas.last->next = ref; - release_literal_areas.last = ref; + + /* + * We have no direct references into the code. + * Complete to purge. + */ + + if (purge_state.module != BIF_ARG_1) + BIF_ERROR(BIF_P, BADARG); + + if (!erts_try_seize_code_write_permission(BIF_P)) { + ERTS_BIF_YIELD2(bif_export[BIF_erts_internal_purge_module_2], + BIF_P, BIF_ARG_1, BIF_ARG_2); + } + + code_ix = erts_active_code_ix(); + + /* + * Correct module? + */ + + if ((modp = erts_get_module(BIF_ARG_1, code_ix)) == NULL) { + ERTS_BIF_PREP_RET(ret, am_false); } else { - release_literal_areas.first = ref; - release_literal_areas.last = ref; + + erts_rwlock_old_code(code_ix); + + /* + * Any code to purge? + */ + if (!modp->old.code_hdr) { + ERTS_BIF_PREP_RET(ret, am_false); + } + else { + /* + * Unload any NIF library + */ + if (modp->old.nif != NULL) { + /* ToDo: Do unload nif without blocking */ + erts_rwunlock_old_code(code_ix); + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_thr_progress_block(); + is_blocking = 1; + erts_rwlock_old_code(code_ix); + erts_unload_nif(modp->old.nif); + modp->old.nif = NULL; + } + + /* + * Remove the old code. + */ + ASSERT(erts_total_code_size >= modp->old.code_length); + erts_total_code_size -= modp->old.code_length; + code = (BeamInstr*) modp->old.code_hdr; + erts_fun_purge_complete(purge_state.funs, purge_state.fe_ix); + beam_catches_delmod(modp->old.catches, code, modp->old.code_length, + code_ix); + literals = modp->old.code_hdr->literal_area; + modp->old.code_hdr->literal_area = NULL; + erts_free(ERTS_ALC_T_CODE, (void *) code); + modp->old.code_hdr = NULL; + modp->old.code_length = 0; + modp->old.catches = BEAM_CATCHES_NIL; + erts_remove_from_ranges(code); + ERTS_BIF_PREP_RET(ret, am_true); + } + erts_rwunlock_old_code(code_ix); } - erts_smp_mtx_unlock(&release_literal_areas.mtx); - erts_queue_message(erts_literal_area_collector, - 0, - erts_alloc_message(0, NULL), - am_copy_literals, - BIF_P->common.id); - } + if (is_blocking) { + erts_smp_thr_progress_unblock(); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + } + + erts_release_code_write_permission(); -#else /* !ERTS_NEW_PURGE_STRATEGY */ + finalize_purge_operation(BIF_P, ret == am_true); - erts_release_literal_area(literals); +#if !defined(ERTS_NEW_PURGE_STRATEGY) -#endif + ASSERT(erts_copy_literal_area == literals); + erts_copy_literal_area = NULL; + erts_release_literal_area(literals); + +#else /* ERTS_NEW_PURGE_STRATEGY */ + + if (literals) { + ErtsLiteralAreaRef *ref; + ref = erts_alloc(ERTS_ALC_T_LITERAL_REF, + sizeof(ErtsLiteralAreaRef)); + ref->literal_area = literals; + ref->next = NULL; + erts_smp_mtx_lock(&release_literal_areas.mtx); + if (release_literal_areas.last) { + release_literal_areas.last->next = ref; + release_literal_areas.last = ref; + } + else { + release_literal_areas.first = ref; + release_literal_areas.last = ref; + } + erts_smp_mtx_unlock(&release_literal_areas.mtx); + erts_queue_message(erts_literal_area_collector, + 0, + erts_alloc_message(0, NULL), + am_copy_literals, + BIF_P->common.id); + } - return ret; +#endif /* ERTS_NEW_PURGE_STRATEGY */ + + return ret; + } + + default: + BIF_ERROR(BIF_P, BADARG); + + } } /* diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index c6258287c4..bc7c9165fd 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -6535,34 +6535,49 @@ call_fun(Process* p, /* Current process. */ * representation (the module has never been loaded), * or the module defining the fun has been unloaded. */ + module = fe->module; - if ((modp = erts_get_module(module, code_ix)) != NULL - && modp->curr.code_hdr != NULL) { + + ERTS_SMP_READ_MEMORY_BARRIER; + if (fe->pend_purge_address) { /* - * There is a module loaded, but obviously the fun is not - * defined in it. We must not call the error_handler - * (or we will get into an infinite loop). + * The system is currently trying to purge the + * module containing this fun. Suspend the process + * and let it try again when the purge operation is + * done (may succeed or not). */ - goto badfun; + ep = erts_suspend_process_on_pending_purge_lambda(p); + ASSERT(ep); } + else { + if ((modp = erts_get_module(module, code_ix)) != NULL + && modp->curr.code_hdr != NULL) { + /* + * There is a module loaded, but obviously the fun is not + * defined in it. We must not call the error_handler + * (or we will get into an infinite loop). + */ + goto badfun; + } - /* - * No current code for this module. Call the error_handler module - * to attempt loading the module. - */ + /* + * No current code for this module. Call the error_handler module + * to attempt loading the module. + */ - ep = erts_find_function(erts_proc_get_error_handler(p), - am_undefined_lambda, 3, code_ix); - if (ep == NULL) { /* No error handler */ - p->current = NULL; - p->freason = EXC_UNDEF; - return NULL; + ep = erts_find_function(erts_proc_get_error_handler(p), + am_undefined_lambda, 3, code_ix); + if (ep == NULL) { /* No error handler */ + p->current = NULL; + p->freason = EXC_UNDEF; + return NULL; + } } reg[0] = module; reg[1] = fun; reg[2] = args; reg[3] = NIL; - return ep->addressv[erts_active_code_ix()]; + return ep->addressv[code_ix]; } } } else if (is_export_header(hdr)) { diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index d52f12e6d0..1200bb9c6f 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -121,8 +121,10 @@ typedef struct beam_code_header { void erts_release_literal_area(struct ErtsLiteralArea_* literal_area); int erts_is_module_native(BeamCodeHeader* code); - void erts_beam_bif_load_init(void); +struct erl_fun_entry; +void erts_purge_state_add_fun(struct erl_fun_entry *fe); +Export *erts_suspend_process_on_pending_purge_lambda(Process *c_p); /* * Layout of the line table. diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 7ffcb26567..90ac5e9ed8 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -644,8 +644,7 @@ bif erts_debug:map_info/1 # New in 19.0 # -bif erts_internal:copy_literals/2 -bif erts_internal:purge_module/1 +bif erts_internal:purge_module/2 bif binary:split/2 bif binary:split/3 bif erts_debug:size_shared/1 diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 0a4a34a174..1b72a0cb59 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -368,6 +368,7 @@ type NLINK_LH STANDARD PROCESSES nlink_lh type CODE LONG_LIVED CODE code type LITERAL LITERAL CODE literal type LITERAL_REF SHORT_LIVED CODE literal_area_ref +type PURGE_DATA SHORT_LIVED CODE purge_data type DB_HEIR_DATA STANDARD ETS db_heir_data type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 3fb866733c..ef3fa07664 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -2886,6 +2886,27 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(AM_tag); #endif } + else if (ERTS_IS_ATOM_STR("check_process_code",BIF_ARG_1)) { + Eterm terms[3]; + Sint length = 1; + Uint sz = 0; + Eterm *hp, res; + DECL_AM(direct_references); + + terms[0] = AM_direct_references; +#if !defined(ERTS_NEW_PURGE_STRATEGY) + { + DECL_AM(indirect_references); + terms[1] = AM_indirect_references; + terms[2] = am_copy_literals; + length = 3; + } +#endif + erts_bld_list(NULL, &sz, length, terms); + hp = HAlloc(BIF_P, sz); + res = erts_bld_list(&hp, NULL, length, terms); + BIF_RET(res); + } BIF_ERROR(BIF_P, BADARG); } diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index 6ce1376c81..c639ba623f 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -199,14 +199,13 @@ erts_erase_fun_entry(ErlFunEntry* fe) } void -erts_cleanup_funs_on_purge(BeamInstr* start, BeamInstr* end) +erts_fun_purge_prepare(BeamInstr* start, BeamInstr* end) { int limit; HashBucket** bucket; - ErlFunEntry* to_delete = NULL; int i; - erts_fun_write_lock(); + erts_fun_read_lock(); limit = erts_fun_table.size; bucket = erts_fun_table.bucket; for (i = 0; i < limit; i++) { @@ -217,22 +216,51 @@ erts_cleanup_funs_on_purge(BeamInstr* start, BeamInstr* end) BeamInstr* addr = fe->address; if (start <= addr && addr < end) { + fe->pend_purge_address = addr; + ERTS_SMP_WRITE_MEMORY_BARRIER; fe->address = unloaded_fun; - if (erts_refc_dectest(&fe->refc, 0) == 0) { - fe->address = (void *) to_delete; - to_delete = fe; - } + erts_purge_state_add_fun(fe); } b = b->next; } } + erts_fun_read_unlock(); +} + +void +erts_fun_purge_abort_prepare(ErlFunEntry **funs, Uint no) +{ + Uint ix; - while (to_delete != NULL) { - ErlFunEntry* next = (ErlFunEntry *) to_delete->address; - erts_erase_fun_entry_unlocked(to_delete); - to_delete = next; + for (ix = 0; ix < no; ix++) { + ErlFunEntry *fe = funs[ix]; + if (fe->address == unloaded_fun) + fe->address = fe->pend_purge_address; + fe->pend_purge_address = NULL; } - erts_fun_write_unlock(); +} + +void +erts_fun_purge_abort_finalize(ErlFunEntry **funs, Uint no) +{ + Uint ix; + + for (ix = 0; ix < no; ix++) + funs[ix]->pend_purge_address = NULL; +} + +void +erts_fun_purge_complete(ErlFunEntry **funs, Uint no) +{ + Uint ix; + + for (ix = 0; ix < no; ix++) { + ErlFunEntry *fe = funs[ix]; + fe->pend_purge_address = NULL; + if (erts_refc_dectest(&fe->refc, 0) == 0) + erts_erase_fun_entry(fe); + } + ERTS_SMP_WRITE_MEMORY_BARRIER; } void @@ -294,6 +322,7 @@ fun_alloc(ErlFunEntry* template) obj->module = template->module; erts_refc_init(&obj->refc, -1); obj->address = unloaded_fun; + obj->pend_purge_address = NULL; #ifdef HIPE obj->native_address = NULL; #endif diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index 8c4deea7a0..73c3e19c1c 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -44,6 +44,7 @@ typedef struct erl_fun_entry { Eterm module; /* Tagged atom for module. */ erts_refc_t refc; /* Reference count: One for code + one for each fun object in each process. */ + BeamInstr *pend_purge_address; /* address stored during a pending purge */ } ErlFunEntry; /* @@ -81,7 +82,10 @@ ErlFunEntry* erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index, void erts_erase_fun_entry(ErlFunEntry* fe); void erts_cleanup_funs(ErlFunThing* funp); -void erts_cleanup_funs_on_purge(BeamInstr* start, BeamInstr* end); +void erts_fun_purge_prepare(BeamInstr* start, BeamInstr* end); +void erts_fun_purge_abort_prepare(ErlFunEntry **funs, Uint no); +void erts_fun_purge_abort_finalize(ErlFunEntry **funs, Uint no); +void erts_fun_purge_complete(ErlFunEntry **funs, Uint no); void erts_dump_fun_entries(int, void *); #endif diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 788dbc3328..a1b0a398da 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -2260,14 +2260,19 @@ erl_start(int argc, char **argv) Eterm pid; pid = erl_system_process_otp(otp_ring0_pid, "erts_code_purger"); - erts_code_purger = erts_proc_lookup(pid); - ASSERT(erts_code_purger); + erts_code_purger + = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, + internal_pid_index(pid)); + ASSERT(erts_code_purger && erts_code_purger->common.id == pid); erts_proc_inc_refc(erts_code_purger); #ifdef ERTS_NEW_PURGE_STRATEGY pid = erl_system_process_otp(otp_ring0_pid, "erts_literal_area_collector"); - erts_literal_area_collector = erts_proc_lookup(pid); - ASSERT(erts_literal_area_collector); + erts_literal_area_collector + = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, + internal_pid_index(pid)); + ASSERT(erts_literal_area_collector + && erts_literal_area_collector->common.id == pid); erts_proc_inc_refc(erts_literal_area_collector); #endif diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 4192cf389f..06266363b5 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -96,6 +96,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "dist_entry", "address" }, { "dist_entry_links", "address" }, { "code_write_permission", NULL }, + { "purge_state", NULL }, { "proc_status", "pid" }, { "proc_trace", "pid" }, { "ports_snapshot", NULL }, -- cgit v1.2.3 From 0e04e76df2ea71e2e2e116afef04c497d84b1024 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 8 Aug 2016 17:13:24 +0200 Subject: Perform check_process_code while process is executing dirty --- erts/emulator/beam/atom.names | 1 + erts/emulator/beam/beam_bif_load.c | 40 ++++++++ erts/emulator/beam/bif.tab | 3 + erts/emulator/beam/erl_init.c | 10 ++ erts/emulator/beam/erl_process.c | 198 ++++++++++++++++++++++++++++++++----- erts/emulator/beam/global.h | 3 + 6 files changed, 233 insertions(+), 22 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index ace3dc5230..89c42f4700 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -198,6 +198,7 @@ atom dgroup_leader atom dictionary atom dirty_cpu atom dirty_cpu_schedulers_online +atom dirty_execution atom dirty_io atom disable_trace atom disabled diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 2413ccdba9..01f7cdd88f 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -54,6 +54,9 @@ static struct { Process *erts_code_purger = NULL; ErtsLiteralArea *erts_copy_literal_area = NULL; +#ifdef ERTS_DIRTY_SCHEDULERS +Process *erts_dirty_process_code_checker; +#endif #ifdef ERTS_NEW_PURGE_STRATEGY Process *erts_literal_area_collector = NULL; @@ -581,6 +584,43 @@ badarg: BIF_ERROR(BIF_P, BADARG); } +BIF_RETTYPE erts_internal_check_dirty_process_code_2(BIF_ALIST_2) +{ +#if !defined(ERTS_DIRTY_SCHEDULERS) + BIF_ERROR(BIF_P, EXC_NOTSUP); +#else + Process *rp; + int reds = 0; + Eterm res; + + if (BIF_P != erts_dirty_process_code_checker) + BIF_ERROR(BIF_P, EXC_NOTSUP); + + if (is_not_internal_pid(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); + + if (is_not_atom(BIF_ARG_2)) + BIF_ERROR(BIF_P, BADARG); + + rp = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN, + BIF_ARG_1, ERTS_PROC_LOCK_MAIN); + if (rp == ERTS_PROC_LOCK_BUSY) + ERTS_BIF_YIELD2(bif_export[BIF_erts_internal_check_dirty_process_code_2], + BIF_P, BIF_ARG_1, BIF_ARG_2); + if (!rp) + BIF_RET(am_false); + + res = erts_check_process_code(rp, BIF_ARG_2, 0, &reds, BIF_P->fcalls); + + if (BIF_P != rp) + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); + + ASSERT(is_value(res)); + + BIF_RET2(res, reds); +#endif +} + BIF_RETTYPE delete_module_1(BIF_ALIST_1) { ErtsCodeIndex code_ix; diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 90ac5e9ed8..80db4eb6ff 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -161,6 +161,7 @@ bif erts_internal:port_close/1 bif erts_internal:port_connect/2 bif erts_internal:request_system_task/3 +bif erts_internal:request_system_task/4 bif erts_internal:check_process_code/2 bif erts_internal:map_to_tuple_keys/1 @@ -644,6 +645,8 @@ bif erts_debug:map_info/1 # New in 19.0 # +bif erts_internal:is_process_executing_dirty/1 +bif erts_internal:check_dirty_process_code/2 bif erts_internal:purge_module/2 bif binary:split/2 bif binary:split/3 diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index a1b0a398da..dea4cc2992 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -2276,6 +2276,16 @@ erl_start(int argc, char **argv) erts_proc_inc_refc(erts_literal_area_collector); #endif +#ifdef ERTS_DIRTY_SCHEDULERS + pid = erl_system_process_otp(otp_ring0_pid, "erts_dirty_process_code_checker"); + erts_dirty_process_code_checker + = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, + internal_pid_index(pid)); + ASSERT(erts_dirty_process_code_checker + && erts_dirty_process_code_checker->common.id == pid); + erts_proc_inc_refc(erts_dirty_process_code_checker); +#endif + } #ifdef ERTS_SMP diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 42c30b935e..f0a288c47b 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -6674,14 +6674,24 @@ erts_schedule_process(Process *p, erts_aint32_t state, ErtsProcLocks locks) } static int -schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st) +schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, + erts_aint32_t *fail_state_p) { int res; int locked; ErtsProcSysTaskQs *stqs, *free_stqs; - erts_aint32_t state, a, n, enq_prio; + erts_aint32_t fail_state, state, a, n, enq_prio; int enqueue; /* < 0 -> use proxy */ unsigned int prof_runnable_procs; + int strict_fail_state; + + fail_state = *fail_state_p; + /* + * If fail state something other than just exiting process, + * ensure that the task wont be scheduled when the + * receiver is in the failure state. + */ + strict_fail_state = fail_state != ERTS_PSFLG_EXITING; res = 1; /* prepare for success */ st->next = st->prev = st; /* Prep for empty prio queue */ @@ -6708,7 +6718,8 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st) erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); state = erts_smp_atomic32_read_nob(&p->state); - if (state & ERTS_PSFLG_EXITING) { + if (state & fail_state) { + *fail_state_p = (state & fail_state); free_stqs = stqs; res = 0; goto cleanup; @@ -6762,7 +6773,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st) /* Status lock prevents out of order "runnable proc" trace msgs */ ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - if (!prof_runnable_procs) { + if (!prof_runnable_procs && !strict_fail_state) { erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); locked = 0; } @@ -6773,6 +6784,11 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st) erts_aint32_t e; n = e = a; + if (strict_fail_state && (a & fail_state)) { + *fail_state_p = (a & fail_state); + goto cleanup; + } + if (a & ERTS_PSFLG_FREE) goto cleanup; /* We don't want to schedule free processes... */ @@ -9091,6 +9107,27 @@ resume_process_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } +BIF_RETTYPE +erts_internal_is_process_executing_dirty_1(BIF_ALIST_1) +{ + if (is_not_internal_pid(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); +#ifdef ERTS_DIRTY_SCHEDULERS + else { + Process *rp = erts_proc_lookup(BIF_ARG_1); + if (rp) { + erts_aint32_t state = erts_smp_atomic32_read_nob(&rp->state); + if (state & (ERTS_PSFLG_DIRTY_RUNNING + |ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + BIF_RET(am_true); + } + } + } +#endif + BIF_RET(am_false); +} + + Uint erts_run_queues_len(Uint *qlen, int atomic_queues_read, int incl_active_sched) { @@ -9987,6 +10024,11 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) if (state & (ERTS_PSFLG_ACTIVE_SYS | ERTS_PSFLG_PENDING_EXIT | ERTS_PSFLG_EXITING)) { + /* + * IMPORTANT! We need to take care of + * scheduled check-process-code requests + * before continuing with dirty execution! + */ /* Migrate to normal scheduler... */ goto sunlock_sched_out_proc; } @@ -10496,22 +10538,83 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) return reds; } -BIF_RETTYPE -erts_internal_request_system_task_3(BIF_ALIST_3) +#ifdef ERTS_DIRTY_SCHEDULERS + +static BIF_RETTYPE +dispatch_system_task(Process *c_p, erts_aint_t fail_state, + ErtsProcSysTask *st, Eterm target, + Eterm prio, Eterm operation) { - Process *rp = erts_proc_lookup(BIF_ARG_1); + Process *rp; + ErtsProcLocks rp_locks = 0; + ErlOffHeap *ohp; + ErtsMessage *mp; + Eterm *hp, msg; + Uint hsz, osz; + BIF_RETTYPE ret; + + switch (st->type) { + case ERTS_PSTT_CPC: + rp = erts_dirty_process_code_checker; + ASSERT(fail_state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)); + if (c_p == rp) { + ERTS_BIF_PREP_RET(ret, am_dirty_execution); + return ret; + } + break; + default: + rp = NULL; + ERTS_INTERNAL_ERROR("Non-dispatchable system task"); + break; + } + + ERTS_BIF_PREP_RET(ret, am_ok); + + /* + * Send message on the form: {Requester, Target, Operation} + */ + + ASSERT(is_immed(st->requester)); + ASSERT(is_immed(target)); + ASSERT(is_immed(prio)); + + osz = size_object(operation); + hsz = 5 /* 4-tuple */ + osz; + + mp = erts_alloc_message_heap(rp, &rp_locks, hsz, &hp, &ohp); + + msg = copy_struct(operation, osz, &hp, ohp); + msg = TUPLE4(hp, st->requester, target, prio, msg); + + erts_queue_message(rp, rp_locks, mp, msg, st->requester); + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + + return ret; +} + +#endif + +static BIF_RETTYPE +request_system_task(Process *c_p, Eterm requester, Eterm target, + Eterm priority, Eterm operation) +{ + BIF_RETTYPE ret; + Process *rp = erts_proc_lookup(target); ErtsProcSysTask *st = NULL; - erts_aint32_t prio; + erts_aint32_t prio, fail_state = ERTS_PSFLG_EXITING; Eterm noproc_res, req_type; - if (!rp && !is_internal_pid(BIF_ARG_1)) { - if (!is_external_pid(BIF_ARG_1)) + if (!rp && !is_internal_pid(target)) { + if (!is_external_pid(target)) goto badarg; - if (external_pid_dist_entry(BIF_ARG_1) != erts_this_dist_entry) + if (external_pid_dist_entry(target) != erts_this_dist_entry) goto badarg; } - switch (BIF_ARG_2) { + switch (priority) { case am_max: prio = PRIORITY_MAX; break; case am_high: prio = PRIORITY_HIGH; break; case am_normal: prio = PRIORITY_NORMAL; break; @@ -10519,11 +10622,11 @@ erts_internal_request_system_task_3(BIF_ALIST_3) default: goto badarg; } - if (is_not_tuple(BIF_ARG_3)) + if (is_not_tuple(operation)) goto badarg; else { int i; - Eterm *tp = tuple_val(BIF_ARG_3); + Eterm *tp = tuple_val(operation); Uint arity = arityval(*tp); Eterm req_id; Uint req_id_sz; @@ -10561,7 +10664,7 @@ erts_internal_request_system_task_3(BIF_ALIST_3) ERTS_INIT_OFF_HEAP(&st->off_heap); hp = &st->heap[0]; - st->requester = BIF_P->common.id; + st->requester = requester; st->reply_tag = req_type; st->req_id_sz = req_id_sz; st->req_id = req_id_sz == 0 ? req_id : copy_struct(req_id, @@ -10595,6 +10698,15 @@ erts_internal_request_system_task_3(BIF_ALIST_3) st->type = ERTS_PSTT_CPC; if (!rp) goto noproc; +#ifdef ERTS_DIRTY_SCHEDULERS + /* + * If the process should start executing dirty + * code it is important that this task is + * aborted. Therefore this strict fail state... + */ + fail_state |= (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS); +#endif break; #ifdef ERTS_NEW_PURGE_STRATEGY @@ -10612,20 +10724,59 @@ erts_internal_request_system_task_3(BIF_ALIST_3) goto badarg; } - if (!schedule_process_sys_task(rp, prio, st)) { - noproc: - notify_sys_task_executed(BIF_P, st, noproc_res); + if (!schedule_process_sys_task(rp, prio, st, &fail_state)) { + Eterm failure; + if (fail_state & ERTS_PSFLG_EXITING) { + noproc: + failure = noproc_res; + } +#ifdef ERTS_DIRTY_SCHEDULERS + else if (fail_state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + ret = dispatch_system_task(c_p, fail_state, st, + target, priority, operation); + goto cleanup_return; + } +#endif + else { + ERTS_INTERNAL_ERROR("Unknown failure schedule_process_sys_task()"); + failure = am_internal_error; + } + notify_sys_task_executed(c_p, st, failure); } - BIF_RET(am_ok); + ERTS_BIF_PREP_RET(ret, am_ok); + + return ret; badarg: + ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); + +#ifdef ERTS_DIRTY_SCHEDULERS +cleanup_return: +#endif + if (st) { erts_cleanup_offheap(&st->off_heap); erts_free(ERTS_ALC_T_PROC_SYS_TSK, st); } - BIF_ERROR(BIF_P, BADARG); + + return ret; +} + +BIF_RETTYPE +erts_internal_request_system_task_3(BIF_ALIST_3) +{ + return request_system_task(BIF_P, BIF_P->common.id, + BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); +} + +BIF_RETTYPE +erts_internal_request_system_task_4(BIF_ALIST_4) +{ + return request_system_task(BIF_P, BIF_ARG_1, + BIF_ARG_2, BIF_ARG_3, BIF_ARG_4); } static void @@ -10634,7 +10785,7 @@ erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type) Process *rp = erts_proc_lookup(pid); if (rp) { ErtsProcSysTask *st; - erts_aint32_t state; + erts_aint32_t state, fail_state; int i; st = erts_alloc(ERTS_ALC_T_PROC_SYS_TSK, @@ -10649,7 +10800,10 @@ erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type) ERTS_INIT_OFF_HEAP(&st->off_heap); state = erts_smp_atomic32_read_nob(&rp->state); - if (!schedule_process_sys_task(rp, ERTS_PSFLGS_GET_USR_PRIO(state), st)) + fail_state = ERTS_PSFLG_EXITING; + + if (!schedule_process_sys_task(rp, ERTS_PSFLGS_GET_USR_PRIO(state), + st, &fail_state)) erts_free(ERTS_ALC_T_PROC_SYS_TSK, st); } } diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 00dfc421d5..ca11d102e0 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1018,6 +1018,9 @@ extern ErtsLiteralArea *erts_copy_literal_area; #ifdef ERTS_NEW_PURGE_STRATEGY extern Process *erts_literal_area_collector; #endif +#ifdef ERTS_DIRTY_SCHEDULERS +extern Process *erts_dirty_process_code_checker; +#endif extern Process *erts_code_purger; -- cgit v1.2.3 From c429c3d042ba274d4225a6d79b2f1df8cfd19983 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 25 Aug 2016 15:34:43 +0200 Subject: Fix tracing of processes executing dirty --- erts/emulator/beam/beam_emu.c | 4 +- erts/emulator/beam/erl_alloc.types | 1 + erts/emulator/beam/erl_nif.c | 173 +++++++++++++++++++++------------- erts/emulator/beam/erl_process.c | 106 ++++++++++++++++++--- erts/emulator/beam/erl_process.h | 2 +- erts/emulator/beam/erl_thr_progress.c | 8 +- erts/emulator/beam/erl_trace.c | 14 +++ erts/emulator/beam/global.h | 5 +- 8 files changed, 230 insertions(+), 83 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index bc7c9165fd..1b1865c8c8 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -5391,7 +5391,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) ASSERT(!c_p->scheduler_data); erts_pre_dirty_nif(esdp, &env, c_p, - (struct erl_module_nif*)I[2], NULL); + (struct erl_module_nif*)I[2]); #ifdef DEBUG result = @@ -5400,7 +5400,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) #endif (*fp)(&env, arity, reg); - erts_post_nif(&env); + erts_post_dirty_nif(&env); ASSERT(!is_value(result)); ASSERT(c_p->freason == TRAP); diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 1b72a0cb59..971052b947 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -279,6 +279,7 @@ type TRACER_NIF LONG_LIVED SYSTEM tracer_nif type TRACE_MSG_QUEUE SHORT_LIVED SYSTEM trace_message_queue type SCHED_ASYNC_JOB SHORT_LIVED SYSTEM async_calls type DIRTY_START STANDARD PROCESSES dirty_start +type DIRTY_SL SHORT_LIVED SYSTEM dirty_short_lived +if threads_no_smp # Need thread safe allocs, but std_alloc and fix_alloc are not; diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 23931f0e54..8e0a9ca581 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -120,8 +120,10 @@ execution_state(ErlNifEnv *env, Process **c_pp, int *schedp) else { Process *c_p = env->proc; - if (!(c_p->static_flags & ERTS_STC_FLG_SHADOW_PROC)) - ASSERT(is_scheduler() > 0); + if (!(c_p->static_flags & ERTS_STC_FLG_SHADOW_PROC)) { + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + & ERTS_PROC_LOCK_MAIN); + } else { c_p = env->proc->next; ASSERT(is_scheduler() < 0); @@ -208,11 +210,16 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif, #endif } +static void full_cache_env(ErlNifEnv *env); +static void cache_env(ErlNifEnv* env); +static void full_flush_env(ErlNifEnv *env); +static void flush_env(ErlNifEnv* env); + +#ifdef ERTS_DIRTY_SCHEDULERS void erts_pre_dirty_nif(ErtsSchedulerData *esdp, - ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif, - Process* tracee) + ErlNifEnv* env, Process* p, + struct erl_module_nif* mod_nif) { -#ifdef ERTS_DIRTY_SCHEDULERS Process *sproc; #ifdef DEBUG erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); @@ -223,7 +230,7 @@ void erts_pre_dirty_nif(ErtsSchedulerData *esdp, ASSERT(esdp); #endif - erts_pre_nif(env, p, mod_nif, tracee); + erts_pre_nif(env, p, mod_nif, NULL); sproc = esdp->dirty_shadow_process; ASSERT(sproc); @@ -235,22 +242,10 @@ void erts_pre_dirty_nif(ErtsSchedulerData *esdp, sproc->next = p; sproc->common.id = p->common.id; - sproc->htop = p->htop; - sproc->stop = p->stop; - sproc->hend = p->hend; - sproc->heap = p->heap; - sproc->abandoned_heap = p->abandoned_heap; - sproc->heap_sz = p->heap_sz; - sproc->high_water = p->high_water; - sproc->old_hend = p->old_hend; - sproc->old_htop = p->old_htop; - sproc->old_heap = p->old_heap; - sproc->mbuf = NULL; - sproc->mbuf_sz = 0; - ERTS_INIT_OFF_HEAP(&sproc->off_heap); env->proc = sproc; -#endif + full_cache_env(env); } +#endif /* Temporary object header, auto-deallocated when NIF returns * or when independent environment is cleared. @@ -274,32 +269,37 @@ static ERTS_INLINE void free_tmp_objs(ErlNifEnv* env) void erts_post_nif(ErlNifEnv* env) { erts_unblock_fpe(env->fpe_was_unmasked); + full_flush_env(env); + free_tmp_objs(env); + env->exiting = ERTS_PROC_IS_EXITING(env->proc); +} #ifdef ERTS_DIRTY_SCHEDULERS - if (!(env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)) +void erts_post_dirty_nif(ErlNifEnv* env) +{ + Process *c_p; + ASSERT(env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC); + ASSERT(env->proc->next); + erts_unblock_fpe(env->fpe_was_unmasked); + full_flush_env(env); + free_tmp_objs(env); + c_p = env->proc->next; + env->exiting = ERTS_PROC_IS_EXITING(c_p); + ERTS_VBUMP_ALL_REDS(c_p); +} #endif - { - ASSERT(is_scheduler() > 0); - if (env->heap_frag == NULL) { - ASSERT(env->hp_end == HEAP_LIMIT(env->proc)); - ASSERT(env->hp >= HEAP_TOP(env->proc)); - ASSERT(env->hp <= HEAP_LIMIT(env->proc)); - HEAP_TOP(env->proc) = env->hp; - } - else { - ASSERT(env->hp_end != HEAP_LIMIT(env->proc)); - ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); - env->heap_frag->used_size = env->hp - env->heap_frag->mem; - ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); - } - env->exiting = ERTS_PROC_IS_EXITING(env->proc); - } + +static void full_flush_env(ErlNifEnv* env) +{ #ifdef ERTS_DIRTY_SCHEDULERS - else { /* Dirty nif call using shadow process struct */ + if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { + /* Dirty nif call using shadow process struct */ Process *c_p = env->proc->next; ASSERT(is_scheduler() < 0); ASSERT(env->proc->common.id == c_p->common.id); + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + & ERTS_PROC_LOCK_MAIN); if (!env->heap_frag) { ASSERT(env->hp_end == HEAP_LIMIT(c_p)); @@ -339,11 +339,48 @@ void erts_post_nif(ErlNifEnv* env) } c_p->off_heap.overhead += env->proc->off_heap.overhead; - env->exiting = ERTS_PROC_IS_EXITING(c_p); - BUMP_ALL_REDS(c_p); + return; } #endif - free_tmp_objs(env); + + flush_env(env); +} + +static void full_cache_env(ErlNifEnv* env) +{ +#ifdef ERTS_DIRTY_SCHEDULERS + if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { + /* Dirty nif call using shadow process struct */ + Process *sproc = env->proc; + Process *c_p = sproc->next; + ASSERT(c_p); + ASSERT(is_scheduler() < 0); + ASSERT(env->proc->common.id == c_p->common.id); + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + & ERTS_PROC_LOCK_MAIN); + + sproc->htop = c_p->htop; + sproc->stop = c_p->stop; + sproc->hend = c_p->hend; + sproc->heap = c_p->heap; + sproc->abandoned_heap = c_p->abandoned_heap; + sproc->heap_sz = c_p->heap_sz; + sproc->high_water = c_p->high_water; + sproc->old_hend = c_p->old_hend; + sproc->old_htop = c_p->old_htop; + sproc->old_heap = c_p->old_heap; + sproc->mbuf = NULL; + sproc->mbuf_sz = 0; + ERTS_INIT_OFF_HEAP(&sproc->off_heap); + + env->hp_end = HEAP_LIMIT(c_p); + env->hp = HEAP_TOP(c_p); + env->heap_frag = NULL; + return; + } +#endif + + cache_env(env); } /* Flush out our cached heap pointers to allow an ordinary HAlloc @@ -600,17 +637,32 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, if (scheduler > 0) { /* Normal scheduler */ rp = erts_proc_lookup(receiver); - if (c_p == rp) - rp_locks = ERTS_PROC_LOCK_MAIN; + if (!rp) + return 0; } else { - if (c_p && ERTS_PROC_IS_EXITING(c_p)) - return 0; - rp = erts_pid2proc_opt(c_p, 0, receiver, rp_locks, + if (c_p) { + ASSERT(scheduler < 0); /* Dirty scheduler */ + if (ERTS_PROC_IS_EXITING(c_p)) + return 0; + + if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + } + } + + rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, + receiver, rp_locks, ERTS_P2P_FLG_INC_REFC); + if (!rp) { + if (c_p && (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)) + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + return 0; + } } - if (rp == NULL) - return 0; + + if (c_p == rp) + rp_locks = ERTS_PROC_LOCK_MAIN; if (menv) { flush_env(msg_env); @@ -631,9 +683,9 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ErlOffHeap *ohp; Eterm *hp; if (env && !env->tracee) { - flush_env(env); + full_flush_env(env); mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); - cache_env(env); + full_cache_env(env); } else { erts_aint_t state = erts_smp_atomic32_read_nob(&rp->state); @@ -656,8 +708,11 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, if (!env || !env->tracee) { - if (c_p && IS_TRACED_FL(c_p, F_TRACE_SEND)) + if (c_p && IS_TRACED_FL(c_p, F_TRACE_SEND)) { + full_flush_env(env); trace_send(c_p, receiver, msg); + full_cache_env(env); + } } #ifdef ERTS_SMP else { @@ -690,10 +745,6 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, erts_smp_proc_trylock(rp, ERTS_PROC_LOCK_MSGQ) == EBUSY) { if (!msgq) { -#ifdef ERTS_SMP - ErtsThrPrgrDelayHandle dhndl; -#endif - msgq = erts_alloc(ERTS_ALC_T_TRACE_MSG_QUEUE, sizeof(ErlTraceMessageQueue)); msgq->receiver = receiver; @@ -707,15 +758,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, erts_smp_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE); -#ifdef ERTS_SMP - if (!scheduler) - dhndl = erts_thr_progress_unmanaged_delay(); -#endif - erts_schedule_flush_trace_messages(t_p->common.id); -#ifdef ERTS_SMP - if (!scheduler) - erts_thr_progress_unmanaged_continue(dhndl); -#endif + erts_schedule_flush_trace_messages(t_p, 0); } else { msgq->len++; *msgq->last = mp; @@ -740,6 +783,8 @@ done: rp_locks &= ~ERTS_PROC_LOCK_MAIN; if (rp_locks & ~lc_locks) erts_smp_proc_unlock(rp, rp_locks & ~lc_locks); + if (c_p && (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)) + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); #endif if (scheduler <= 0) erts_proc_dec_refc(rp); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index f0a288c47b..cd348a8541 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -9534,15 +9534,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) p->reds += actual_reds; -#ifdef ERTS_SMP - erts_smp_proc_lock(p, ERTS_PROC_LOCK_TRACE); - if (p->trace_msg_q) { - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_TRACE); - erts_schedule_flush_trace_messages(p->common.id); - } else - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_TRACE); -#endif - state = erts_smp_atomic32_read_nob(&p->state); if (IS_TRACED(p)) { @@ -9562,7 +9553,15 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) } } - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + +#ifdef ERTS_SMP + if (p->trace_msg_q) { + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_schedule_flush_trace_messages(p, 1); + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + } +#endif /* have to re-read state after taking lock */ state = erts_smp_atomic32_read_nob(&p->state); @@ -9570,9 +9569,11 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) #ifdef ERTS_SMP if (is_normal_sched && (state & ERTS_PSFLG_PENDING_EXIT)) erts_handle_pending_exit(p, (ERTS_PROC_LOCK_MAIN + | ERTS_PROC_LOCK_TRACE | ERTS_PROC_LOCK_STATUS)); if (p->pending_suspenders) handle_pending_suspend(p, (ERTS_PROC_LOCK_MAIN + | ERTS_PROC_LOCK_TRACE | ERTS_PROC_LOCK_STATUS)); #endif @@ -9592,7 +9593,9 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) p->scheduler_data = NULL; #endif - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); + erts_smp_proc_unlock(p, (ERTS_PROC_LOCK_MAIN + | ERTS_PROC_LOCK_STATUS + | ERTS_PROC_LOCK_TRACE)); ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER); @@ -10808,16 +10811,95 @@ erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type) } } + void erts_schedule_complete_off_heap_message_queue_change(Eterm pid) { erts_schedule_generic_sys_task(pid, ERTS_PSTT_COHMQ); } +#ifdef ERTS_DIRTY_SCHEDULERS + +static void +flush_dirty_trace_messages(void *vpid) +{ + Process *proc; + Eterm pid; +#ifdef ARCH_64 + pid = (Eterm) vpid; +#else + pid = *((Eterm *) vpid); + erts_free(ERTS_ALC_T_DIRTY_SL, vpid); +#endif + + proc = erts_proc_lookup(pid); + if (proc) + (void) erts_flush_trace_messages(proc, 0); +} + +#endif /* ERTS_DIRTY_SCHEDULERS */ + void -erts_schedule_flush_trace_messages(Eterm pid) +erts_schedule_flush_trace_messages(Process *proc, int force_on_proc) { +#ifdef ERTS_SMP + ErtsThrPrgrDelayHandle dhndl; +#endif + Eterm pid = proc->common.id; + +#ifdef ERTS_DIRTY_SCHEDULERS + erts_aint32_t state; + + if (!force_on_proc) { + state = erts_smp_atomic32_read_nob(&proc->state); + if (state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + goto sched_flush_dirty; + } + } +#endif + +#ifdef ERTS_SMP + dhndl = erts_thr_progress_unmanaged_delay(); +#endif + erts_schedule_generic_sys_task(pid, ERTS_PSTT_FTMQ); + +#ifdef ERTS_SMP + erts_thr_progress_unmanaged_continue(dhndl); +#endif + +#ifdef ERTS_DIRTY_SCHEDULERS + if (!force_on_proc) { + state = erts_smp_atomic32_read_mb(&proc->state); + if (state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + void *vargp; + + sched_flush_dirty: + /* + * We traced 'proc' from another thread than + * it is executing on, and it is executing + * on a dirty scheduler. It might take a + * significant amount of time before it is + * scheduled out (where it gets opportunity + * to flush messages). We therefore schedule + * the flush on the first ordinary scheduler. + */ + +#ifdef ARCH_64 + vargp = (void *) pid; +#else + { + Eterm *argp = erts_alloc(ERTS_ALC_T_DIRTY_SL, sizeof(Eterm)); + *argp = pid; + vargp = (void *) argp; + } +#endif + erts_schedule_misc_aux_work(1, flush_dirty_trace_messages, vargp); + } + } +#endif } static void diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 7c98b60647..bf3ed93deb 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1768,7 +1768,7 @@ void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *), ErtsThrPrgrLaterOp *, UWord); void erts_schedule_complete_off_heap_message_queue_change(Eterm pid); -void erts_schedule_flush_trace_messages(Eterm pid); +void erts_schedule_flush_trace_messages(Process *proc, int force_on_proc); int erts_flush_trace_messages(Process *c_p, ErtsProcLocks locks); #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c index 21938e7684..26d71f573f 100644 --- a/erts/emulator/beam/erl_thr_progress.c +++ b/erts/emulator/beam/erl_thr_progress.c @@ -969,8 +969,10 @@ erts_thr_progress_unmanaged_continue__(ErtsThrPrgrDelayHandle handle) #ifdef ERTS_ENABLE_LOCK_CHECK ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL); ERTS_LC_ASSERT(tpd && tpd->is_delaying); - tpd->is_delaying = 0; - return_tmp_thr_prgr_data(tpd); + tpd->is_delaying--; + ASSERT(tpd->is_delaying >= 0); + if (!tpd->is_delaying) + return_tmp_thr_prgr_data(tpd); #endif ASSERT(!erts_thr_progress_is_managed_thread()); @@ -995,7 +997,7 @@ erts_thr_progress_unmanaged_delay__(void) #ifdef ERTS_ENABLE_LOCK_CHECK { ErtsThrPrgrData *tpd = tmp_thr_prgr_data(NULL); - tpd->is_delaying = 1; + tpd->is_delaying++; } #endif return (ErtsThrPrgrDelayHandle) umrefc_ix; diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 4cf38bf894..48836be8eb 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -813,6 +813,9 @@ trace_send(Process *p, Eterm to, Eterm msg) ErtsTracerNif *tnif = NULL; ErtsTracingEvent* te; Eterm pam_result; +#ifdef ERTS_SMP + ErtsThrPrgrDelayHandle dhndl; +#endif ASSERT(ARE_TRACE_FLAGS_ON(p, F_TRACE_SEND)); @@ -837,6 +840,10 @@ trace_send(Process *p, Eterm to, Eterm msg) } else pam_result = am_true; +#ifdef ERTS_SMP + dhndl = erts_thr_progress_unmanaged_delay(); +#endif + if (is_internal_pid(to)) { if (!erts_proc_lookup(to)) goto send_to_non_existing_process; @@ -852,6 +859,11 @@ trace_send(Process *p, Eterm to, Eterm msg) send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_SEND, operation, msg, to, pam_result); } + +#ifdef ERTS_SMP + erts_thr_progress_unmanaged_continue(dhndl); +#endif + erts_match_set_release_result_trace(p, pam_result); } @@ -1167,6 +1179,8 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, Eterm transformed_args[MAX_ARG]; ErtsTracer pre_ms_tracer = erts_tracer_nil; + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) & ERTS_PROC_LOCK_MAIN); + ASSERT(tracer); if (ERTS_TRACER_COMPARE(*tracer, erts_tracer_true)) { /* Breakpoint trace enabled without specifying tracer => diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index ca11d102e0..9bc12c7b55 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -62,9 +62,12 @@ struct enif_environment_t /* ErlNifEnv */ extern void erts_pre_nif(struct enif_environment_t*, Process*, struct erl_module_nif*, Process* tracee); extern void erts_post_nif(struct enif_environment_t* env); +#ifdef ERTS_DIRTY_SCHEDULERS extern void erts_pre_dirty_nif(ErtsSchedulerData *, struct enif_environment_t*, Process*, - struct erl_module_nif*, Process* tracee); + struct erl_module_nif*); +extern void erts_post_dirty_nif(struct enif_environment_t* env); +#endif extern Eterm erts_nif_taints(Process* p); extern void erts_print_nif_taints(int to, void* to_arg); void erts_unload_nif(struct erl_module_nif* nif); -- cgit v1.2.3 From f36e430a1482ee91d7f8b5ff72477ee6882b876d Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 28 Jun 2016 17:40:45 +0200 Subject: Ensure correct mbuf_sz value in process structure --- erts/emulator/beam/beam_emu.c | 23 +++++++++++++++++++++++ erts/emulator/beam/erl_gc.c | 6 ++++++ erts/emulator/beam/erl_message.c | 17 +++++++++++++++++ erts/emulator/beam/erl_message.h | 11 +++++++++++ erts/emulator/beam/erl_nif.c | 14 +++++++++++--- erts/emulator/beam/erl_process.h | 5 ++++- erts/emulator/beam/utils.c | 3 ++- 7 files changed, 74 insertions(+), 5 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index b3e6ebf651..ef4cdf9d5a 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1960,6 +1960,8 @@ void process_main(void) ErtsMessage* msgp; PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_CHK_MBUF_SZ(c_p); + PreFetch(0, next); msgp = PEEK_MESSAGE(c_p); @@ -2047,6 +2049,7 @@ void process_main(void) } ERTS_DBG_CHK_REDS(c_p, FCALLS); + ERTS_CHK_MBUF_SZ(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2583,7 +2586,9 @@ do { \ c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, tmp_reg); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2614,7 +2619,9 @@ do { \ c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, tmp_reg); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2644,7 +2651,9 @@ do { \ SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, reg, live); + ERTS_CHK_MBUF_SZ(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2685,7 +2694,9 @@ do { \ SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, reg, live); + ERTS_CHK_MBUF_SZ(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2728,7 +2739,9 @@ do { \ SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, reg, live); + ERTS_CHK_MBUF_SZ(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2766,7 +2779,9 @@ do { \ c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, tmp_reg); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2793,7 +2808,9 @@ do { \ bf = (BifFunction) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, tmp_reg); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2843,7 +2860,9 @@ do { \ ASSERT(!ERTS_PROC_IS_EXITING(c_p)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); live_hf_end = c_p->mbuf; + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, reg, I); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_HOLE_CHECK(c_p); @@ -3556,11 +3575,13 @@ do { \ ASSERT(c_p->scheduler_data); #endif live_hf_end = c_p->mbuf; + ERTS_CHK_MBUF_SZ(c_p); erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL); nif_bif_result = (*fp)(&env, bif_nif_arity, reg); if (env.exception_thrown) nif_bif_result = THE_NON_VALUE; erts_post_nif(&env); + ERTS_CHK_MBUF_SZ(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -3612,7 +3633,9 @@ do { \ Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf; ASSERT(!ERTS_PROC_IS_EXITING(c_p)); live_hf_end = c_p->mbuf; + ERTS_CHK_MBUF_SZ(c_p); nif_bif_result = (*bf)(c_p, reg, I); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 8b0dd9a5a6..75f8ebefbd 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -517,6 +517,8 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int erts_proc_sched_data((p))->virtual_reds += vreds; } + ERTS_CHK_MBUF_SZ(p); + ASSERT(CONTEXT_REDS >= erts_proc_sched_data(p)->virtual_reds); return reds_left; } @@ -527,6 +529,8 @@ young_gen_usage(Process *p) Uint hsz; Eterm *aheap; + ERTS_CHK_MBUF_SZ(p); + hsz = p->mbuf_sz; if (p->flags & F_ON_HEAP_MSGQ) { @@ -590,6 +594,8 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); #endif + ERTS_CHK_MBUF_SZ(p); + ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls) >= erts_proc_sched_data(p)->virtual_reds); diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 91e06cde2d..e4c696ae3b 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -1842,3 +1842,20 @@ void erts_factory_undo(ErtsHeapFactory* factory) factory->heap_frags = NULL; #endif } + +Uint +erts_mbuf_size(Process *p) +{ + Uint sz = 0; + ErlHeapFragment* bp; + ErtsMessage* mp; + + for (bp = p->mbuf; bp; bp = bp->next) + sz += bp->used_size; + + for (mp = p->msg_frag; mp; mp = mp->next) + for (bp = erts_message_to_heap_frag(mp); bp; bp = bp->next) + sz += bp->used_size; + + return sz; +} diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 6df969367b..42ed14e69c 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -499,4 +499,15 @@ erts_msgq_replace_msg_ref(ErlMessageQueue *msgq, ErtsMessage *newp, ErtsMessage #endif +Uint erts_mbuf_size(Process *p); +#if defined(DEBUG) || 0 +# define ERTS_CHK_MBUF_SZ(P) \ + do { \ + Uint actual_mbuf_sz__ = erts_mbuf_size((P)); \ + ERTS_ASSERT((P)->mbuf_sz >= actual_mbuf_sz__); \ + } while (0) +#else +# define ERTS_CHK_MBUF_SZ(P) ((void) 1) +#endif + #endif diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index ef2fb93106..27392a5996 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -156,7 +156,9 @@ static Eterm* alloc_heap_heavy(ErlNifEnv* env, size_t need, Eterm* hp) HEAP_TOP(env->proc) = env->hp; } else { - env->heap_frag->used_size = hp - env->heap_frag->mem; + Uint usz = env->hp - env->heap_frag->mem; + env->proc->mbuf_sz += usz - env->heap_frag->used_size; + env->heap_frag->used_size = usz; ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); } hp = erts_heap_alloc(env->proc, need, MIN_HEAP_FRAG_SZ); @@ -308,11 +310,14 @@ static void full_flush_env(ErlNifEnv* env) HEAP_TOP(c_p) = env->hp; } else { + Uint usz; ASSERT(env->hp_end != HEAP_LIMIT(c_p)); ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); HEAP_TOP(c_p) = HEAP_TOP(env->proc); - env->heap_frag->used_size = env->hp - env->heap_frag->mem; + usz = env->hp - env->heap_frag->mem; + env->proc->mbuf_sz += usz - env->heap_frag->used_size; + env->heap_frag->used_size = usz; ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); @@ -394,9 +399,12 @@ static void flush_env(ErlNifEnv* env) HEAP_TOP(env->proc) = env->hp; } else { + Uint usz; ASSERT(env->hp_end != HEAP_LIMIT(env->proc)); ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); - env->heap_frag->used_size = env->hp - env->heap_frag->mem; + usz = env->hp - env->heap_frag->mem; + env->proc->mbuf_sz += usz - env->heap_frag->used_size; + env->heap_frag->used_size = usz; ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); } } diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 94c13bb948..f92c745050 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1328,10 +1328,13 @@ ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp); ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp) { ErlHeapFragment* hf = MBUF(p); + Uint sz; ASSERT(hf!=NULL && (hp - hf->mem < hf->alloc_size)); - hf->used_size = hp - hf->mem; + sz = hp - hf->mem; + p->mbuf_sz -= hf->used_size - sz; + hf->used_size = sz; } #endif /* inline */ diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 85647b8500..6786657faf 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -102,6 +102,7 @@ erts_heap_alloc(Process* p, Uint need, Uint xtra) if (bp != NULL && need <= (bp->alloc_size - bp->used_size)) { Eterm* ret = bp->mem + bp->used_size; bp->used_size += need; + p->mbuf_sz += need; return ret; } #ifdef DEBUG @@ -124,7 +125,7 @@ erts_heap_alloc(Process* p, Uint need, Uint xtra) MBUF(p) = bp; bp->alloc_size = n; bp->used_size = need; - MBUF_SIZE(p) += n; + MBUF_SIZE(p) += need; bp->off_heap.first = NULL; bp->off_heap.overhead = 0; return bp->mem; -- cgit v1.2.3 From cb8d2f366415fd46aeb87320f4fdf19adc9f0eb0 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 12 Aug 2016 15:16:57 +0200 Subject: Fix is-dirty-cpu/is-dirty-io scheduler predicates --- erts/emulator/beam/erl_process.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 94c13bb948..109d66012a 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1549,9 +1549,9 @@ extern int erts_system_profile_ts_type; #define ERTS_SCHEDULER_IS_DIRTY(ESDP) \ ((ESDP)->dirty_no.s.num != 0) #define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) \ - ((ESDP)->dirty_no.s.type == 0) + (ERTS_SCHEDULER_IS_DIRTY((ESDP)) & ((ESDP)->dirty_no.s.type == 0)) #define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) \ - ((ESDP)->dirty_no.s.type == 1) + (ERTS_SCHEDULER_IS_DIRTY((ESDP)) & ((ESDP)->dirty_no.s.type == 1)) #else #define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0 #define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0 -- cgit v1.2.3 From 1b4a59c405e6bd3532921d5c534e2264bb05b2eb Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 29 Aug 2016 19:21:53 +0200 Subject: Remove old purge strategy --- erts/emulator/beam/beam_bif_load.c | 324 ++---------------------------------- erts/emulator/beam/bif.tab | 2 +- erts/emulator/beam/erl_bif_info.c | 21 --- erts/emulator/beam/erl_init.c | 2 - erts/emulator/beam/erl_lock_check.c | 2 - erts/emulator/beam/erl_process.c | 11 -- erts/emulator/beam/global.h | 8 +- erts/emulator/beam/ops.tab | 8 +- 8 files changed, 16 insertions(+), 362 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 5c28fb1a98..10759c1f57 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -61,7 +61,7 @@ ErtsLiteralArea *erts_copy_literal_area = NULL; #ifdef ERTS_DIRTY_SCHEDULERS Process *erts_dirty_process_code_checker; #endif -#ifdef ERTS_NEW_PURGE_STRATEGY + Process *erts_literal_area_collector = NULL; typedef struct ErtsLiteralAreaRef_ ErtsLiteralAreaRef; @@ -75,10 +75,9 @@ struct { ErtsLiteralAreaRef *first; ErtsLiteralAreaRef *last; } release_literal_areas; -#endif static void set_default_trace_pattern(Eterm module); -static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls); +static Eterm check_process_code(Process* rp, Module* modp, int *redsp, int fcalls); static void delete_code(Module* modp); static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); @@ -107,11 +106,9 @@ init_purge_state(void) void erts_beam_bif_load_init(void) { -#ifdef ERTS_NEW_PURGE_STRATEGY erts_smp_mtx_init(&release_literal_areas.mtx, "release_literal_areas"); release_literal_areas.first = NULL; release_literal_areas.last = NULL; -#endif init_purge_state(); } @@ -539,7 +536,7 @@ check_old_code_1(BIF_ALIST_1) } Eterm -erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls) +erts_check_process_code(Process *c_p, Eterm module, int *redsp, int fcalls) { Module* modp; Eterm res; @@ -554,31 +551,23 @@ erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int if (!modp) return am_false; erts_rlock_old_code(code_ix); - res = (!modp->old.code_hdr ? am_false : - check_process_code(c_p, modp, flags, redsp, fcalls)); + res = (!modp->old.code_hdr + ? am_false + : check_process_code(c_p, modp, redsp, fcalls)); erts_runlock_old_code(code_ix); return res; } -BIF_RETTYPE erts_internal_check_process_code_2(BIF_ALIST_2) +BIF_RETTYPE erts_internal_check_process_code_1(BIF_ALIST_1) { int reds = 0; - Uint flags; Eterm res; if (is_not_atom(BIF_ARG_1)) goto badarg; - if (is_not_small(BIF_ARG_2)) - goto badarg; - - flags = unsigned_val(BIF_ARG_2); - if (flags & ~ERTS_CPC_ALL) { - goto badarg; - } - - res = erts_check_process_code(BIF_P, BIF_ARG_1, flags, &reds, BIF_P->fcalls); + res = erts_check_process_code(BIF_P, BIF_ARG_1, &reds, BIF_P->fcalls); ASSERT(is_value(res)); @@ -614,7 +603,7 @@ BIF_RETTYPE erts_internal_check_dirty_process_code_2(BIF_ALIST_2) if (!rp) BIF_RET(am_false); - res = erts_check_process_code(rp, BIF_ARG_2, 0, &reds, BIF_P->fcalls); + res = erts_check_process_code(rp, BIF_ARG_2, &reds, BIF_P->fcalls); if (BIF_P != rp) erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); @@ -857,32 +846,12 @@ set_default_trace_pattern(Eterm module) } } -#ifndef ERTS_NEW_PURGE_STRATEGY - -static ERTS_INLINE int -check_mod_funs(Process *p, ErlOffHeap *off_heap, char *area, size_t area_size) -{ - struct erl_off_heap_header* oh; - for (oh = off_heap->first; oh; oh = oh->next) { - if (thing_subtag(oh->thing_word) == FUN_SUBTAG) { - ErlFunThing* funp = (ErlFunThing*) oh; - if (ErtsInArea(funp->fe->address, area, area_size)) - return !0; - } - } - return 0; -} - -#endif - static Uint hfrag_literal_size(Eterm* start, Eterm* end, char* lit_start, Uint lit_size); static void hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp, Eterm *start, Eterm *end, char *lit_start, Uint lit_size); -#ifdef ERTS_NEW_PURGE_STRATEGY - Eterm erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed) { @@ -1044,7 +1013,7 @@ literal_gc: } static Eterm -check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls) +check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) { BeamInstr* start; char* mod_start; @@ -1109,253 +1078,6 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls return am_false; } -#else /* !ERTS_NEW_PURGE_STRATEGY, i.e, old style purge... */ - -static Eterm -check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls) -{ - BeamInstr* start; - char* literals; - Uint lit_bsize; - char* mod_start; - Uint mod_size; - Eterm* sp; - int done_gc = 0; - int need_gc = 0; - ErtsMessage *msgp; - ErlHeapFragment *hfrag; - -#define ERTS_ORDINARY_GC__ (1 << 0) -#define ERTS_LITERAL_GC__ (1 << 1) - - /* - * Pick up limits for the module. - */ - start = (BeamInstr*) modp->old.code_hdr; - mod_start = (char *) start; - mod_size = modp->old.code_length; - - /* - * Check if current instruction or continuation pointer points into module. - */ - if (ErtsInArea(rp->i, mod_start, mod_size) - || ErtsInArea(rp->cp, mod_start, mod_size)) { - return am_true; - } - - /* - * Check all continuation pointers stored on the stack. - */ - for (sp = rp->stop; sp < STACK_START(rp); sp++) { - if (is_CP(*sp) && ErtsInArea(cp_val(*sp), mod_start, mod_size)) { - return am_true; - } - } - - /* - * Check all continuation pointers stored in stackdump - * and clear exception stackdump if there is a pointer - * to the module. - */ - if (rp->ftrace != NIL) { - struct StackTrace *s; - ASSERT(is_list(rp->ftrace)); - s = (struct StackTrace *) big_val(CDR(list_val(rp->ftrace))); - if ((s->pc && ErtsInArea(s->pc, mod_start, mod_size)) || - (s->current && ErtsInArea(s->current, mod_start, mod_size))) { - rp->freason = EXC_NULL; - rp->fvalue = NIL; - rp->ftrace = NIL; - } else { - int i; - for (i = 0; i < s->depth; i++) { - if (ErtsInArea(s->trace[i], mod_start, mod_size)) { - rp->freason = EXC_NULL; - rp->fvalue = NIL; - rp->ftrace = NIL; - break; - } - } - } - } - - if (rp->flags & F_DISABLE_GC) { - /* - * Cannot proceed. Process has disabled gc in order to - * safely leave inconsistent data on the heap and/or - * off heap lists. Need to wait for gc to be enabled - * again. - */ - return THE_NON_VALUE; - } - - /* - * Message queue can contains funs, and may contain - * literals. If we got references to this module from the message - * queue. - * - * If a literal is in the message queue we maka an explicit copy of - * and attach it to the heap fragment. Each message needs to be - * self contained, we cannot save the literal in the old_heap or - * any other heap than the message it self. - */ - - erts_smp_proc_lock(rp, ERTS_PROC_LOCK_MSGQ); - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ); - - if (modp->old.code_hdr->literal_area) { - literals = (char*) modp->old.code_hdr->literal_area->start; - lit_bsize = (char*) modp->old.code_hdr->literal_area->end - literals; - } - else { - literals = NULL; - lit_bsize = 0; - } - - for (msgp = rp->msg.first; msgp; msgp = msgp->next) { - if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) - hfrag = &msgp->hfrag; - else if (is_value(ERL_MESSAGE_TERM(msgp)) && msgp->data.heap_frag) - hfrag = msgp->data.heap_frag; - else - continue; - { - ErlHeapFragment *hf; - Uint lit_sz = 0; - for (hf=hfrag; hf; hf = hf->next) { - if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)) - return am_true; - lit_sz += hfrag_literal_size(&hf->mem[0], &hf->mem[hf->used_size], - literals, lit_bsize); - } - if (lit_sz > 0) { - ErlHeapFragment *bp = new_message_buffer(lit_sz); - Eterm *hp = bp->mem; - - for (hf=hfrag; hf; hf = hf->next) { - hfrag_literal_copy(&hp, &bp->off_heap, - &hf->mem[0], &hf->mem[hf->used_size], - literals, lit_bsize); - hfrag=hf; - } - /* link new hfrag last */ - ASSERT(hfrag->next == NULL); - hfrag->next = bp; - bp->next = NULL; - } - } - } - - while (1) { - - /* Check heap, stack etc... */ - if (check_mod_funs(rp, &rp->off_heap, mod_start, mod_size)) - goto try_gc; - if (any_heap_ref_ptrs(&rp->fvalue, &rp->fvalue+1, literals, lit_bsize)) { - rp->freason = EXC_NULL; - rp->fvalue = NIL; - rp->ftrace = NIL; - } - if (any_heap_ref_ptrs(rp->stop, rp->hend, literals, lit_bsize)) - goto try_literal_gc; -#ifdef HIPE - if (nstack_any_heap_ref_ptrs(rp, literals, lit_bsize)) - goto try_literal_gc; -#endif - if (any_heap_refs(rp->heap, rp->htop, literals, lit_bsize)) - goto try_literal_gc; - if (any_heap_refs(rp->old_heap, rp->old_htop, literals, lit_bsize)) - goto try_literal_gc; - - /* Check dictionary */ - if (rp->dictionary) { - Eterm* start = ERTS_PD_START(rp->dictionary); - Eterm* end = start + ERTS_PD_SIZE(rp->dictionary); - - if (any_heap_ref_ptrs(start, end, literals, lit_bsize)) - goto try_literal_gc; - } - - /* Check heap fragments */ - for (hfrag = rp->mbuf; hfrag; hfrag = hfrag->next) { - Eterm *hp, *hp_end; - /* Off heap lists should already have been moved into process */ - ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)); - - hp = &hfrag->mem[0]; - hp_end = &hfrag->mem[hfrag->used_size]; - if (any_heap_refs(hp, hp_end, literals, lit_bsize)) - goto try_literal_gc; - } - - /* - * Message buffer fragments (matched messages) - * - off heap lists should already have been moved into - * process off heap structure. - * - Check for literals - */ - for (msgp = rp->msg_frag; msgp; msgp = msgp->next) { - hfrag = erts_message_to_heap_frag(msgp); - for (; hfrag; hfrag = hfrag->next) { - Eterm *hp, *hp_end; - ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)); - - hp = &hfrag->mem[0]; - hp_end = &hfrag->mem[hfrag->used_size]; - - if (any_heap_refs(hp, hp_end, literals, lit_bsize)) - goto try_literal_gc; - } - } - - return am_false; - - try_literal_gc: - need_gc |= ERTS_LITERAL_GC__; - - try_gc: - need_gc |= ERTS_ORDINARY_GC__; - - if ((done_gc & need_gc) == need_gc) - return am_true; - - if (!(flags & ERTS_CPC_ALLOW_GC)) - return am_aborted; - - need_gc &= ~done_gc; - - /* - * Try to get rid of literals by by garbage collecting. - * Clear both fvalue and ftrace. - */ - - rp->freason = EXC_NULL; - rp->fvalue = NIL; - rp->ftrace = NIL; - - if (need_gc & ERTS_ORDINARY_GC__) { - FLAGS(rp) |= F_NEED_FULLSWEEP; - *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity, fcalls); - done_gc |= ERTS_ORDINARY_GC__; - } - if (need_gc & ERTS_LITERAL_GC__) { - struct erl_off_heap_header* oh; - oh = modp->old.code_hdr->literal_area->off_heap; - *redsp += lit_bsize / 64; /* Need, better value... */ - erts_garbage_collect_literals(rp, (Eterm*)literals, lit_bsize, oh); - done_gc |= ERTS_LITERAL_GC__; - } - need_gc = 0; - } - -#undef ERTS_ORDINARY_GC__ -#undef ERTS_LITERAL_GC__ - -} - -#endif /* !ERTS_NEW_PURGE_STRATEGY */ - static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) { @@ -1484,8 +1206,6 @@ hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp, } } -#ifdef ERTS_NEW_PURGE_STRATEGY - ErtsThrPrgrLaterOp later_literal_area_switch; #ifdef ERTS_SMP @@ -1499,13 +1219,8 @@ complete_literal_area_switch(void *unused) } #endif -#endif /* ERTS_NEW_PURGE_STRATEGY */ - BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0) { -#ifndef ERTS_NEW_PURGE_STRATEGY - BIF_ERROR(BIF_P, EXC_NOTSUP); -#else ErtsLiteralAreaRef *la_ref; if (BIF_P != erts_literal_area_collector) @@ -1544,7 +1259,6 @@ BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0) ERTS_BIF_YIELD_RETURN(BIF_P, am_true); #endif -#endif /* ERTS_NEW_PURGE_STRATEGY */ } void @@ -1722,10 +1436,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) code = (BeamInstr*) modp->old.code_hdr; end = (BeamInstr *)((char *)code + modp->old.code_length); erts_fun_purge_prepare(code, end); -#if !defined(ERTS_NEW_PURGE_STRATEGY) - ASSERT(!erts_copy_literal_area); - erts_copy_literal_area = modp->old.code_hdr->literal_area; -#endif } erts_runlock_old_code(code_ix); } @@ -1765,10 +1475,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) erts_fun_purge_abort_prepare(purge_state.funs, purge_state.fe_ix); -#if !defined(ERTS_NEW_PURGE_STRATEGY) - ASSERT(erts_copy_literal_area); - erts_copy_literal_area = NULL; -#endif #ifndef ERTS_SMP erts_fun_purge_abort_finalize(purge_state.funs, purge_state.fe_ix); finalize_purge_operation(BIF_P, 0); @@ -1877,14 +1583,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) finalize_purge_operation(BIF_P, ret == am_true); -#if !defined(ERTS_NEW_PURGE_STRATEGY) - - ASSERT(erts_copy_literal_area == literals); - erts_copy_literal_area = NULL; - erts_release_literal_area(literals); - -#else /* ERTS_NEW_PURGE_STRATEGY */ - if (literals) { ErtsLiteralAreaRef *ref; ref = erts_alloc(ERTS_ALC_T_LITERAL_REF, @@ -1908,8 +1606,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) BIF_P->common.id); } -#endif /* ERTS_NEW_PURGE_STRATEGY */ - return ret; } diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index ea66f165a0..7a35c02934 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -164,7 +164,7 @@ bif erts_internal:port_connect/2 bif erts_internal:request_system_task/3 bif erts_internal:request_system_task/4 -bif erts_internal:check_process_code/2 +bif erts_internal:check_process_code/1 bif erts_internal:map_to_tuple_keys/1 bif erts_internal:term_type/1 diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 107c0c01d9..cb7278696f 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -2883,27 +2883,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(AM_tag); #endif } - else if (ERTS_IS_ATOM_STR("check_process_code",BIF_ARG_1)) { - Eterm terms[3]; - Sint length = 1; - Uint sz = 0; - Eterm *hp, res; - DECL_AM(direct_references); - - terms[0] = AM_direct_references; -#if !defined(ERTS_NEW_PURGE_STRATEGY) - { - DECL_AM(indirect_references); - terms[1] = AM_indirect_references; - terms[2] = am_copy_literals; - length = 3; - } -#endif - erts_bld_list(NULL, &sz, length, terms); - hp = HAlloc(BIF_P, sz); - res = erts_bld_list(&hp, NULL, length, terms); - BIF_RET(res); - } BIF_ERROR(BIF_P, BADARG); } diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 781bf024dd..fb85bbff34 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -2264,7 +2264,6 @@ erl_start(int argc, char **argv) ASSERT(erts_code_purger && erts_code_purger->common.id == pid); erts_proc_inc_refc(erts_code_purger); -#ifdef ERTS_NEW_PURGE_STRATEGY pid = erl_system_process_otp(otp_ring0_pid, "erts_literal_area_collector"); erts_literal_area_collector = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, @@ -2272,7 +2271,6 @@ erl_start(int argc, char **argv) ASSERT(erts_literal_area_collector && erts_literal_area_collector->common.id == pid); erts_proc_inc_refc(erts_literal_area_collector); -#endif #ifdef ERTS_DIRTY_SCHEDULERS pid = erl_system_process_otp(otp_ring0_pid, "erts_dirty_process_code_checker"); diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 06266363b5..356f5ca71e 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -113,9 +113,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "export_tab", NULL }, { "fun_tab", NULL }, { "environ", NULL }, -#ifdef ERTS_NEW_PURGE_STRATEGY { "release_literal_areas", NULL }, -#endif #endif { "efile_drv", "address" }, { "drv_ev_state_grow", NULL, }, diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index fa8290028a..91eb82af7d 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -448,9 +448,7 @@ int erts_system_profile_ts_type = ERTS_TRACE_FLG_NOW_TIMESTAMP; typedef enum { ERTS_PSTT_GC, /* Garbage Collect */ ERTS_PSTT_CPC, /* Check Process Code */ -#ifdef ERTS_NEW_PURGE_STRATEGY ERTS_PSTT_CLA, /* Copy Literal Area */ -#endif ERTS_PSTT_COHMQ, /* Change off heap message queue */ ERTS_PSTT_FTMQ /* Flush trace msg queue */ } ErtsProcSysTaskType; @@ -10430,7 +10428,6 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) fcalls = reds - CONTEXT_REDS; st_res = erts_check_process_code(c_p, st->arg[0], - unsigned_val(st->arg[1]), &cpc_reds, fcalls); reds -= cpc_reds; @@ -10441,7 +10438,6 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) } break; } -#ifdef ERTS_NEW_PURGE_STRATEGY case ERTS_PSTT_CLA: { int fcalls; int cla_reds = 0; @@ -10461,7 +10457,6 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) } break; } -#endif case ERTS_PSTT_COHMQ: reds -= erts_complete_off_heap_message_queue_change(c_p); st_res = am_true; @@ -10516,11 +10511,9 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) case ERTS_PSTT_COHMQ: st_res = am_false; break; -#ifdef ERTS_NEW_PURGE_STRATEGY case ERTS_PSTT_CLA: st_res = am_ok; break; -#endif #ifdef ERTS_SMP case ERTS_PSTT_FTMQ: reds -= erts_flush_trace_messages(c_p, ERTS_PROC_LOCK_MAIN); @@ -10695,8 +10688,6 @@ request_system_task(Process *c_p, Eterm requester, Eterm target, case am_check_process_code: if (is_not_atom(st->arg[0])) goto badarg; - if (is_not_small(st->arg[1]) || (unsigned_val(st->arg[1]) & ~ERTS_CPC_ALL)) - goto badarg; noproc_res = am_false; st->type = ERTS_PSTT_CPC; if (!rp) @@ -10712,7 +10703,6 @@ request_system_task(Process *c_p, Eterm requester, Eterm target, #endif break; -#ifdef ERTS_NEW_PURGE_STRATEGY case am_copy_literals: if (st->arg[0] != am_true && st->arg[0] != am_false) goto badarg; @@ -10721,7 +10711,6 @@ request_system_task(Process *c_p, Eterm requester, Eterm target, if (!rp) goto noproc; break; -#endif default: goto badarg; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index d7dd6371b7..0fd9ab5e58 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1000,12 +1000,8 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg); Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2); /* beam_bif_load.c */ -#define ERTS_CPC_ALLOW_GC (1 << 0) -#define ERTS_CPC_ALL ERTS_CPC_ALLOW_GC -Eterm erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls); -#ifdef ERTS_NEW_PURGE_STRATEGY +Eterm erts_check_process_code(Process *c_p, Eterm module, int *redsp, int fcalls); Eterm erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed); -#endif typedef struct ErtsLiteralArea_ { struct erl_off_heap_header *off_heap; @@ -1017,9 +1013,7 @@ typedef struct ErtsLiteralArea_ { (sizeof(ErtsLiteralArea) + sizeof(Eterm)*((N) - 1)) extern ErtsLiteralArea *erts_copy_literal_area; -#ifdef ERTS_NEW_PURGE_STRATEGY extern Process *erts_literal_area_collector; -#endif #ifdef ERTS_DIRTY_SCHEDULERS extern Process *erts_dirty_process_code_checker; #endif diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 879daaca0a..b467c5a9b6 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -792,14 +792,14 @@ allocate_init t I y ################################################################# # -# The BIFs erts_internal:check_process_code/2 must be called like a function, +# The BIFs erts_internal:check_process_code/1 must be called like a function, # to ensure that c_p->i (program counter) is set correctly (an ordinary # BIF call doesn't set it). # -call_ext u==2 Bif=u$bif:erts_internal:check_process_code/2 => i_call_ext Bif -call_ext_last u==2 Bif=u$bif:erts_internal:check_process_code/2 D => i_call_ext_last Bif D -call_ext_only u==2 Bif=u$bif:erts_internal:check_process_code/2 => i_call_ext_only Bif +call_ext u==1 Bif=u$bif:erts_internal:check_process_code/1 => i_call_ext Bif +call_ext_last u==1 Bif=u$bif:erts_internal:check_process_code/1 D => i_call_ext_last Bif D +call_ext_only u==1 Bif=u$bif:erts_internal:check_process_code/1 => i_call_ext_only Bif # # The BIFs erlang:garbage_collect/0 must be called like a function, -- cgit v1.2.3 From c46ba734b6787e254b45b35ea135fc2da23190bb Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Tue, 23 Feb 2016 11:24:12 +0100 Subject: erts: Fix install of suspend handler This commit makes sure to setup the suspend handler to matter what +B option is given at the command line. --- erts/emulator/beam/break.c | 4 ++-- erts/emulator/beam/erl_init.c | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 0ddf7f4e6d..298b30fff3 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -684,7 +684,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) crash dump. */ erts_thr_progress_fatal_error_block(&tpd_buf); -#ifdef ERTS_THR_HAVE_SIG_FUNCS +#ifdef ERTS_SYS_SUSPEND_SIGNAL /* * We suspend all scheduler threads so that we can dump some * data about the currently running processes and scheduler data. @@ -818,7 +818,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) #ifdef ERTS_SMP -#if defined(ERTS_THR_HAVE_SIG_FUNCS) +#ifdef ERTS_SYS_SUSPEND_SIGNAL /* We resume all schedulers so that we are in a known safe state when we write the rest of the crash dump */ diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index e729574ec7..1718b23688 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -2134,6 +2134,7 @@ erl_start(int argc, char **argv) init_break_handler(); if (replace_intr) erts_replace_intr(); + sys_init_suspend_handler(); #endif boot_argc = argc - i; /* Number of arguments to init */ -- cgit v1.2.3 From b022c5a175c528ebb66df129ddaa9f2d0b20a923 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 1 Sep 2016 14:24:55 +0200 Subject: Fix reclaim of literal areas --- erts/emulator/beam/beam_bif_load.c | 78 +++++++++++++++++++++++++++++--------- erts/emulator/beam/erl_alloc.types | 1 + erts/emulator/beam/global.h | 7 +++- 3 files changed, 66 insertions(+), 20 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 2413ccdba9..9e4176752a 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -53,7 +53,10 @@ static struct { Process *erts_code_purger = NULL; -ErtsLiteralArea *erts_copy_literal_area = NULL; +erts_smp_atomic_t erts_copy_literal_area__; +#define ERTS_SET_COPY_LITERAL_AREA(LA) \ + erts_smp_atomic_set_nob(&erts_copy_literal_area__, \ + (erts_aint_t) (LA)) #ifdef ERTS_NEW_PURGE_STRATEGY Process *erts_literal_area_collector = NULL; @@ -105,6 +108,8 @@ erts_beam_bif_load_init(void) release_literal_areas.first = NULL; release_literal_areas.last = NULL; #endif + erts_smp_atomic_init_nob(&erts_copy_literal_area__, + (erts_aint_t) NULL); init_purge_state(); } @@ -849,7 +854,7 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed Uint lit_bsize; ErlHeapFragment *hfrag; - la = erts_copy_literal_area; + la = ERTS_COPY_LITERAL_AREA(); if (!la) return am_ok; @@ -1438,16 +1443,33 @@ hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp, #ifdef ERTS_NEW_PURGE_STRATEGY +#ifdef ERTS_SMP + ErtsThrPrgrLaterOp later_literal_area_switch; -#ifdef ERTS_SMP +typedef struct { + ErtsThrPrgrLaterOp lop; + ErtsLiteralArea *la; +} ErtsLaterReleasLiteralArea; + +static void +later_release_literal_area(void *vlrlap) +{ + ErtsLaterReleasLiteralArea *lrlap; + lrlap = (ErtsLaterReleasLiteralArea *) vlrlap; + erts_release_literal_area(lrlap->la); + erts_free(ERTS_ALC_T_RELEASE_LAREA, vlrlap); +} + static void -complete_literal_area_switch(void *unused) +complete_literal_area_switch(void *literal_area) { Process *p = erts_literal_area_collector; erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); erts_resume(p, ERTS_PROC_LOCK_STATUS); erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + if (literal_area) + erts_release_literal_area((ErtsLiteralArea *) literal_area); } #endif @@ -1458,6 +1480,7 @@ BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0) #ifndef ERTS_NEW_PURGE_STRATEGY BIF_ERROR(BIF_P, EXC_NOTSUP); #else + ErtsLiteralArea *unused_la; ErtsLiteralAreaRef *la_ref; if (BIF_P != erts_literal_area_collector) @@ -1474,26 +1497,45 @@ BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0) erts_smp_mtx_unlock(&release_literal_areas.mtx); - if (erts_copy_literal_area) - erts_release_literal_area(erts_copy_literal_area); + unused_la = ERTS_COPY_LITERAL_AREA(); if (!la_ref) { - erts_copy_literal_area = NULL; + ERTS_SET_COPY_LITERAL_AREA(NULL); + if (unused_la) { +#ifdef ERTS_SMP + ErtsLaterReleasLiteralArea *lrlap; + lrlap = erts_alloc(ERTS_ALC_T_RELEASE_LAREA, + sizeof(ErtsLaterReleasLiteralArea)); + lrlap->la = unused_la; + erts_schedule_thr_prgr_later_cleanup_op( + later_release_literal_area, + (void *) lrlap, + &lrlap->lop, + (sizeof(ErtsLaterReleasLiteralArea) + + sizeof(ErtsLiteralArea) + + ((unused_la->end + - &unused_la->start[0]) + - 1)*(sizeof(Eterm)))); +#else + erts_release_literal_area(unused_la); +#endif + } BIF_RET(am_false); } - erts_copy_literal_area = la_ref->literal_area; + ERTS_SET_COPY_LITERAL_AREA(la_ref->literal_area); erts_free(ERTS_ALC_T_LITERAL_REF, la_ref); -#ifndef ERTS_SMP - BIF_RET(am_true); -#else +#ifdef ERTS_SMP erts_schedule_thr_prgr_later_op(complete_literal_area_switch, - NULL, + unused_la, &later_literal_area_switch); erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); ERTS_BIF_YIELD_RETURN(BIF_P, am_true); +#else + erts_release_literal_area(unused_la); + BIF_RET(am_true); #endif #endif /* ERTS_NEW_PURGE_STRATEGY */ @@ -1675,8 +1717,8 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) end = (BeamInstr *)((char *)code + modp->old.code_length); erts_fun_purge_prepare(code, end); #if !defined(ERTS_NEW_PURGE_STRATEGY) - ASSERT(!erts_copy_literal_area); - erts_copy_literal_area = modp->old.code_hdr->literal_area; + ASSERT(!ERTS_COPY_LITERAL_AREA()); + ERTS_SET_COPY_LITERAL_AREA(modp->old.code_hdr->literal_area); #endif } erts_runlock_old_code(code_ix); @@ -1718,8 +1760,8 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) erts_fun_purge_abort_prepare(purge_state.funs, purge_state.fe_ix); #if !defined(ERTS_NEW_PURGE_STRATEGY) - ASSERT(erts_copy_literal_area); - erts_copy_literal_area = NULL; + ASSERT(ERTS_COPY_LITERAL_AREA()); + ERTS_SET_COPY_LITERAL_AREA(NULL); #endif #ifndef ERTS_SMP erts_fun_purge_abort_finalize(purge_state.funs, purge_state.fe_ix); @@ -1831,8 +1873,8 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) #if !defined(ERTS_NEW_PURGE_STRATEGY) - ASSERT(erts_copy_literal_area == literals); - erts_copy_literal_area = NULL; + ASSERT(ERTS_COPY_LITERAL_AREA() == literals); + ERTS_SET_COPY_LITERAL_AREA(NULL); erts_release_literal_area(literals); #else /* ERTS_NEW_PURGE_STRATEGY */ diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 1b72a0cb59..d51086aa32 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -313,6 +313,7 @@ type RUNQ_BLNS LONG_LIVED SYSTEM run_queue_balancing type THR_PRGR_IDATA LONG_LIVED SYSTEM thr_prgr_internal_data type THR_PRGR_DATA LONG_LIVED SYSTEM thr_prgr_data type T_THR_PRGR_DATA SHORT_LIVED SYSTEM temp_thr_prgr_data +type RELEASE_LAREA SHORT_LIVED SYSTEM release_literal_area +endif # diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 00dfc421d5..76e6be5443 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1014,7 +1014,10 @@ typedef struct ErtsLiteralArea_ { #define ERTS_LITERAL_AREA_ALLOC_SIZE(N) \ (sizeof(ErtsLiteralArea) + sizeof(Eterm)*((N) - 1)) -extern ErtsLiteralArea *erts_copy_literal_area; +extern erts_smp_atomic_t erts_copy_literal_area__; +#define ERTS_COPY_LITERAL_AREA() \ + ((ErtsLiteralArea *) erts_smp_atomic_read_nob(&erts_copy_literal_area__)) + #ifdef ERTS_NEW_PURGE_STRATEGY extern Process *erts_literal_area_collector; #endif @@ -1102,7 +1105,7 @@ typedef struct { #define INITIALIZE_SHCOPY(info) \ do { \ - ErtsLiteralArea *larea__ = erts_copy_literal_area; \ + ErtsLiteralArea *larea__ = ERTS_COPY_LITERAL_AREA();\ info.queue_start = info.queue_default; \ info.bitstore_start = info.bitstore_default; \ info.shtable_start = info.shtable_default; \ -- cgit v1.2.3 From d5ba65f0aa25768c1af7a1639eb73298b029eaf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Tue, 23 Aug 2016 07:46:43 +0200 Subject: Slightly reduce code size of process_main() clang will generate smaller code if we avoid reassigning the reg and freg arrays. That can be easily arranged by passing them as arguments to process_main(). --- erts/emulator/beam/beam_emu.c | 10 ++++------ erts/emulator/beam/erl_init.c | 2 +- erts/emulator/beam/erl_process.c | 3 ++- erts/emulator/beam/global.h | 2 +- 4 files changed, 8 insertions(+), 9 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index c704323bb2..0ba06058a5 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1078,7 +1078,7 @@ static Eterm make_arglist(Process* c_p, Eterm* reg, int a); void init_emulator(void) { - process_main(); + process_main(0, 0); } /* @@ -1225,7 +1225,7 @@ init_emulator(void) * the instructions' C labels to the loader. * The second call starts execution of BEAM code. This call never returns. */ -void process_main(void) +void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) { static int init_done = 0; Process* c_p = NULL; @@ -1237,7 +1237,7 @@ void process_main(void) /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC, * in all other cases x0 is used. */ - register Eterm* reg REG_xregs = NULL; + register Eterm* reg REG_xregs = x_reg_array; /* * Top of heap (next free location); grows upwards. @@ -1264,7 +1264,7 @@ void process_main(void) * X registers and floating point registers are located in * scheduler specific data. */ - register FloatDef *freg; + register FloatDef *freg = f_reg_array; /* * For keeping the negative old value of 'reds' when call saving is active. @@ -1350,8 +1350,6 @@ void process_main(void) start_time_i = c_p->i; } - reg = erts_proc_sched_data(c_p)->x_reg_array; - freg = erts_proc_sched_data(c_p)->f_reg_array; ERL_BITS_RELOAD_STATEP(c_p); { int reds; diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 781bf024dd..2f68ece37b 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -2302,7 +2302,7 @@ erl_start(int argc, char **argv) #endif set_main_stack_size(); erts_sched_init_time_sup(esdp); - process_main(); + process_main(esdp->x_reg_array, esdp->f_reg_array); } #endif } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index bcbb9b3115..b577ae95b0 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -8169,7 +8169,8 @@ sched_thread_func(void *vesdp) ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL); #endif - process_main(); + process_main(esdp->x_reg_array, esdp->f_reg_array); + /* No schedulers should *ever* terminate */ erts_exit(ERTS_ABORT_EXIT, "Scheduler thread number %beu terminated\n", diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index d7dd6371b7..3030a095a8 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1200,7 +1200,7 @@ void print_pass_through(int, byte*, int); /* beam_emu.c */ int catchlevel(Process*); void init_emulator(void); -void process_main(void); +void process_main(Eterm* x_reg_array, FloatDef* f_reg_array); void erts_dirty_process_main(ErtsSchedulerData *); Eterm build_stacktrace(Process* c_p, Eterm exc); Eterm expand_error_value(Process* c_p, Uint freason, Eterm Value); -- cgit v1.2.3 From 18c52187b1714284416a8a1d618ce4db831d1a45 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 2 Sep 2016 15:05:26 +0200 Subject: Introduce literal sweep of native stack in new purge strategy --- erts/emulator/beam/beam_bif_load.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index ba79068859..dddcfbb77d 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -976,8 +976,13 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed } if (any_heap_ref_ptrs(c_p->stop, c_p->hend, literals, lit_bsize)) + goto literal_gc; + *redsp += 1; +#ifdef HIPE + if (nstack_any_heap_ref_ptrs(c_p, literals, lit_bsize)) goto literal_gc; *redsp += 1; +#endif if (any_heap_refs(c_p->heap, c_p->htop, literals, lit_bsize)) goto literal_gc; *redsp += 1; -- cgit v1.2.3 From 986d32a62b20c32338dac4dfd27c141c8f9be0fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Mon, 20 Jun 2016 13:25:04 +0200 Subject: Implement the new ceil/1 and floor/1 guard BIFs Implement as ceil/1 and floor/1 as new guard BIFs (essentially part of Erlang language). They are guard BIFs because trunc/1 is a guard BIF. It would be strange to have trunc/1 as a part of the language, but not ceil/1 and floor/1. --- erts/emulator/beam/bif.tab | 7 ++++ erts/emulator/beam/erl_bif_guard.c | 65 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 7a35c02934..67eae35dec 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -658,6 +658,13 @@ bif erlang:has_prepared_code_on_load/1 bif maps:take/2 +# +# New in 20.0 +# + +gcbif erlang:floor/1 +gcbif erlang:ceil/1 + # # Obsolete # diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c index b42d2dc28b..458315f293 100644 --- a/erts/emulator/beam/erl_bif_guard.c +++ b/erts/emulator/beam/erl_bif_guard.c @@ -141,6 +141,39 @@ BIF_RETTYPE trunc_1(BIF_ALIST_1) BIF_RET(res); } +BIF_RETTYPE floor_1(BIF_ALIST_1) +{ + Eterm res; + FloatDef f; + + if (is_not_float(BIF_ARG_1)) { + if (is_integer(BIF_ARG_1)) + BIF_RET(BIF_ARG_1); + BIF_ERROR(BIF_P, BADARG); + } + GET_DOUBLE(BIF_ARG_1, f); + res = double_to_integer(BIF_P, floor(f.fd)); + BIF_RET(res); +} + +BIF_RETTYPE ceil_1(BIF_ALIST_1) +{ + Eterm res; + FloatDef f; + + /* check arg */ + if (is_not_float(BIF_ARG_1)) { + if (is_integer(BIF_ARG_1)) + BIF_RET(BIF_ARG_1); + BIF_ERROR(BIF_P, BADARG); + } + /* get the float */ + GET_DOUBLE(BIF_ARG_1, f); + + res = double_to_integer(BIF_P, ceil(f.fd)); + BIF_RET(res); +} + BIF_RETTYPE round_1(BIF_ALIST_1) { Eterm res; @@ -621,6 +654,38 @@ Eterm erts_gc_trunc_1(Process* p, Eterm* reg, Uint live) reg, live); } +Eterm erts_gc_floor_1(Process* p, Eterm* reg, Uint live) +{ + Eterm arg; + FloatDef f; + + arg = reg[live]; + if (is_not_float(arg)) { + if (is_integer(arg)) { + return arg; + } + BIF_ERROR(p, BADARG); + } + GET_DOUBLE(arg, f); + return gc_double_to_integer(p, floor(f.fd), reg, live); +} + +Eterm erts_gc_ceil_1(Process* p, Eterm* reg, Uint live) +{ + Eterm arg; + FloatDef f; + + arg = reg[live]; + if (is_not_float(arg)) { + if (is_integer(arg)) { + return arg; + } + BIF_ERROR(p, BADARG); + } + GET_DOUBLE(arg, f); + return gc_double_to_integer(p, ceil(f.fd), reg, live); +} + static Eterm gc_double_to_integer(Process* p, double x, Eterm* reg, Uint live) { -- cgit v1.2.3 From 6d40cfd77f1d2f1e1403e4b41c0b53ae6499ea11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Mon, 27 Jun 2016 12:54:04 +0200 Subject: Add math:floor/1 and math:ceil/1 Add math:floor/1 and math:ceil/1 to avoid unnecessary conversions in floating point expressions. That is, instead of having to write float(floor(X)) as part of a floating point expressions, we can write simply math:floor(X). --- erts/emulator/beam/bif.tab | 2 ++ erts/emulator/beam/erl_math.c | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 67eae35dec..0bd80ea684 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -664,6 +664,8 @@ bif maps:take/2 gcbif erlang:floor/1 gcbif erlang:ceil/1 +bif math:floor/1 +bif math:ceil/1 # # Obsolete diff --git a/erts/emulator/beam/erl_math.c b/erts/emulator/beam/erl_math.c index fc0aaed18a..990fa63bd4 100644 --- a/erts/emulator/beam/erl_math.c +++ b/erts/emulator/beam/erl_math.c @@ -247,6 +247,12 @@ BIF_RETTYPE math_pow_2(BIF_ALIST_2) return math_call_2(BIF_P, pow, BIF_ARG_1, BIF_ARG_2); } +BIF_RETTYPE math_ceil_1(BIF_ALIST_1) +{ + return math_call_1(BIF_P, ceil, BIF_ARG_1); +} - - +BIF_RETTYPE math_floor_1(BIF_ALIST_1) +{ + return math_call_1(BIF_P, floor, BIF_ARG_1); +} -- cgit v1.2.3 From 02565089e3747a0f85125647b8ad2467e4c2b36d Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 5 Sep 2016 16:32:46 +0200 Subject: erts: Reject beam file with too few functions --- erts/emulator/beam/beam_load.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index b7e802775d..f63addb309 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -2635,6 +2635,10 @@ load_code(LoaderState* stp) * End of code found. */ case op_int_code_end: + if (function_number != stp->num_functions) { + LoadError2(stp, "too few functions (%u) in module (header said %u)", + function_number, stp->num_functions); + } stp->codev_size = codev_size; stp->ci = ci; stp->function = THE_NON_VALUE; -- cgit v1.2.3 From 1c969304d4df4ad22c0089100a071bcb1dca4275 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 5 Sep 2016 16:03:40 +0200 Subject: Fix thread calls to erl_drv_send_term()/erl_drv_output_term() --- erts/emulator/beam/io.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 7c6696405b..6c8bfa27d8 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -5681,16 +5681,20 @@ deliver_term_check_port(ErlDrvTermData port_id, Eterm *connected_p) ErtsThrPrgrDelayHandle dhndl = erts_thr_progress_unmanaged_delay(); #endif erts_aint32_t state; + int res = 1; Port *prt = erts_port_lookup_raw((Eterm) port_id); - if (!prt) - return -1; + if (!prt) { + res = -1; + goto done; + } state = erts_atomic32_read_nob(&prt->state); if (state & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP | ERTS_PORT_SFLG_CLOSING)) { if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) - return -1; + res = -1; else - return 0; + res = 0; + goto done; } if (connected_p) { #ifdef ERTS_SMP @@ -5699,22 +5703,25 @@ deliver_term_check_port(ErlDrvTermData port_id, Eterm *connected_p) #endif *connected_p = ERTS_PORT_GET_CONNECTED(prt); } + +done: + #ifdef ERTS_SMP if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) { + ERTS_SMP_LC_ASSERT(!prt || !erts_lc_is_port_locked(prt)); erts_thr_progress_unmanaged_continue(dhndl); ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); - } + } else #endif - ERTS_SMP_LC_ASSERT(dhndl == ERTS_THR_PRGR_DHANDLE_MANAGED - ? erts_lc_is_port_locked(prt) - : !erts_lc_is_port_locked(prt)); - return 1; + ERTS_SMP_LC_ASSERT(res != 1 || erts_lc_is_port_locked(prt)); + + return res; } int erl_drv_output_term(ErlDrvTermData port_id, ErlDrvTermData* data, int len) { /* May be called from arbitrary thread */ - Eterm connected; + Eterm connected = NIL; /* Shut up faulty warning... */ int res = deliver_term_check_port(port_id, &connected); if (res <= 0) return res; -- cgit v1.2.3 From 2e4812fe7082364ff5bb423b3365b0e6564e5db8 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 5 Sep 2016 19:18:08 +0200 Subject: Fix erts_thr_progress_unmanaged_delay() Thread progress leader update did not cache current unmanaged index when waiting for unmanaged threads. This caused erts_thr_progress_unmanaged_delay() to stop working until a new leader took over. --- erts/emulator/beam/erl_thr_progress.c | 1 + 1 file changed, 1 insertion(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c index 664c479eb6..83cdd56ead 100644 --- a/erts/emulator/beam/erl_thr_progress.c +++ b/erts/emulator/beam/erl_thr_progress.c @@ -699,6 +699,7 @@ leader_update(ErtsThrPrgrData *tpd) tpd->leader_state.chk_next_ix = no_managed; erts_atomic32_set_nob(&intrnl->misc.data.umrefc_ix.current, (erts_aint32_t) new_umrefc_ix); + tpd->leader_state.umrefc_ix.current = new_umrefc_ix; ETHR_MEMBAR(ETHR_StoreLoad); refc = erts_atomic_read_nob(&intrnl->umrefc[umrefc_ix].refc); ASSERT(refc >= 0); -- cgit v1.2.3 From 30c54173054973ef5ba74c864430c82f94292f99 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 7 Sep 2016 17:12:53 +0200 Subject: Fix restore of 'current' field when rescheduled NIF completes --- erts/emulator/beam/erl_nif.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 27392a5996..559b4017e7 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2268,7 +2268,7 @@ int enif_consume_timeslice(ErlNifEnv* env, int percent) * NIF exports need a few more items than the Export struct provides, * including the erl_module_nif* and a NIF function pointer, so the * NifExport below adds those. The Export member must be first in the - * struct. The saved_mfa, exception_thrown, saved_argc, rootset_extra, and + * struct. The saved_current, exception_thrown, saved_argc, rootset_extra, and * rootset members are used to track the MFA, any pending exception, and * arguments of the top NIF in case a chain of one or more * enif_schedule_nif() calls results in an exception, since in that case @@ -2282,7 +2282,7 @@ typedef struct { Export exp; struct erl_module_nif* m; NativeFunPtr fp; - Eterm saved_mfa[3]; + BeamInstr *saved_current; int exception_thrown; int saved_argc; int rootset_extra; @@ -2393,9 +2393,8 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec reg[i] = (Eterm) argv[i]; } if (need_save) { - ep->saved_mfa[0] = proc->current[0]; - ep->saved_mfa[1] = proc->current[1]; - ep->saved_mfa[2] = proc->current[2]; + ASSERT(proc->current); + ep->saved_current = proc->current; ep->saved_argc = argc; } proc->i = (BeamInstr*) ep->exp.addressv[0]; @@ -2418,21 +2417,21 @@ static void restore_nif_mfa(Process* proc, NifExport* ep, int exception) { int i; - Eterm* reg = erts_proc_sched_data(proc)->x_reg_array; ERTS_SMP_LC_ASSERT(!(proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)); ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(proc) & ERTS_PROC_LOCK_MAIN); - proc->current[0] = ep->saved_mfa[0]; - proc->current[1] = ep->saved_mfa[1]; - proc->current[2] = ep->saved_mfa[2]; - if (exception) + ASSERT(ep->saved_current != &ep->exp.code[0]); + proc->current = ep->saved_current; + ep->saved_current = NULL; + if (exception) { + Eterm* reg = erts_proc_sched_data(proc)->x_reg_array; for (i = 0; i < ep->saved_argc; i++) reg[i] = ep->rootset[i+1]; + } ep->saved_argc = 0; - ep->saved_mfa[0] = THE_NON_VALUE; } #ifdef ERTS_DIRTY_SCHEDULERS @@ -2507,6 +2506,7 @@ execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) */ ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); ASSERT(ep && fp); + ep->fp = NULL; erts_smp_atomic32_read_band_mb(&proc->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC | ERTS_PSFLG_DIRTY_IO_PROC)); @@ -2591,7 +2591,7 @@ schedule_dirty_nif(ErlNifEnv* env, int flags, int argc, const ERL_NIF_TERM argv[ } ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - need_save = (ep == NULL || is_non_value(ep->saved_mfa[0])); + need_save = (ep == NULL || !ep->saved_current); result = init_nif_sched_data(env, execute_dirty_nif, fp, need_save, argc, argv); if (scheduler <= 0) erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); @@ -2669,7 +2669,7 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, } ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - need_save = (ep == NULL || is_non_value(ep->saved_mfa[0])); + need_save = (ep == NULL || !ep->saved_current); if (flags) { #ifdef ERTS_DIRTY_SCHEDULERS -- cgit v1.2.3 From 809e70d0c8060f5d96584179b30950344ef51367 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 13 Sep 2016 17:00:11 +0200 Subject: Fix crash due to heap fragments left after GC Do not decode distribution messages as part of the GC operation. Distribution messages containing maps may generate heap fragments. --- erts/emulator/beam/erl_gc.c | 49 +++++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 28 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index d0d74bbf44..9e0919cc9c 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -531,9 +531,19 @@ young_gen_usage(Process *p) if (p->flags & F_ON_HEAP_MSGQ) { ErtsMessage *mp; - for (mp = p->msg.first; mp; mp = mp->next) + for (mp = p->msg.first; mp; mp = mp->next) { + /* + * We leave not yet decoded distribution messages + * as they are in the queue since it is not + * possible to determine a maximum size until + * actual decoding. However, we use their estimated + * size when calculating need, and by this making + * it more likely that they will fit on the heap + * when actually decoded. + */ if (mp->data.attached) hsz += erts_msg_attached_data_size(mp); + } } aheap = p->abandoned_heap; @@ -2247,37 +2257,22 @@ move_msgq_to_heap(Process *p) if (mp->data.attached) { ErlHeapFragment *bp; - ErtsHeapFactory factory; - - erts_factory_proc_prealloc_init(&factory, p, - erts_msg_attached_data_size(mp)); - - if (is_non_value(ERL_MESSAGE_TERM(mp))) { - if (mp->data.dist_ext) { - ASSERT(mp->data.dist_ext->heap_size >= 0); - if (is_not_nil(ERL_MESSAGE_TOKEN(mp))) { - bp = erts_dist_ext_trailer(mp->data.dist_ext); - ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp), - bp->used_size, - &factory.hp, - factory.off_heap); - erts_cleanup_offheap(&bp->off_heap); - } - ERL_MESSAGE_TERM(mp) = erts_decode_dist_ext(&factory, - mp->data.dist_ext); - erts_free_dist_ext_copy(mp->data.dist_ext); - mp->data.dist_ext = NULL; - } - } - else { + + /* + * We leave not yet decoded distribution messages + * as they are in the queue since it is not + * possible to determine a maximum size until + * actual decoding... + */ + if (is_value(ERL_MESSAGE_TERM(mp))) { bp = erts_message_to_heap_frag(mp); if (bp->next) - erts_move_multi_frags(&factory.hp, factory.off_heap, bp, + erts_move_multi_frags(&p->htop, &p->off_heap, bp, mp->m, ERL_MESSAGE_REF_ARRAY_SZ, 0); else - copy_one_frag(&factory.hp, factory.off_heap, bp, + copy_one_frag(&p->htop, &p->off_heap, bp, mp->m, ERL_MESSAGE_REF_ARRAY_SZ); if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) { @@ -2294,8 +2289,6 @@ move_msgq_to_heap(Process *p) mp = new_mp; } } - - erts_factory_close(&factory); } mpp = &(*mpp)->next; -- cgit v1.2.3 From c70ca686fe269db6079a2ca1c7e09cdfc0cfa903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Mon, 5 Sep 2016 16:16:23 +0200 Subject: Don't leak old code when loading a modules with an on_load function Normally, calling code:delete/1 before re-loading the code for a module is unnecessary but causes no problem. But there will be be problems if the new code has an on_load function. Code with an on_load function will always be loaded as old code to allowed it to be easily purged if the on_load function would fail. If the on_load function succeeds, the old and current code will be swapped. So in the scenario where code:delete/1 has been called explicitly, there is old code but no current code. Loading code with an on_load function will cause the reference to the old code to be overwritten. That will at best cause a memory leak, and at worst an emulator crash (especially if NIFs are involved). To avoid that situation, we will put the code with the on_load function in a special, third slot in Module. ERL-240 --- erts/emulator/beam/atom.names | 1 + erts/emulator/beam/beam_bif_load.c | 62 ++++++++++++++++++++++++++++++-------- erts/emulator/beam/beam_load.c | 15 +++++++-- erts/emulator/beam/erl_nif.c | 18 ++++++----- erts/emulator/beam/module.c | 3 ++ erts/emulator/beam/module.h | 1 + 6 files changed, 76 insertions(+), 24 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 9dae67cb2d..2b66bf6f4e 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -505,6 +505,7 @@ atom port_limit atom port_op atom positive atom prepare +atom prepare_on_load atom print atom priority atom private diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index dddcfbb77d..5969197168 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -53,6 +53,7 @@ static struct { ErlFunEntry *def_funs[10]; Uint fe_size; Uint fe_ix; + struct erl_module_instance saved_old; } purge_state; Process *erts_code_purger = NULL; @@ -105,6 +106,8 @@ init_purge_state(void) purge_state.fe_size = sizeof(purge_state.def_funs); purge_state.fe_size /= sizeof(purge_state.def_funs[0]); purge_state.fe_ix = 0; + + purge_state.saved_old.code_hdr = 0; } void @@ -739,10 +742,13 @@ BIF_RETTYPE call_on_load_function_1(BIF_ALIST_1) { Module* modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); - if (modp && modp->old.code_hdr) { - BIF_TRAP_CODE_PTR_0(BIF_P, modp->old.code_hdr->on_load_function_ptr); + if (!modp || !modp->on_load) { + BIF_ERROR(BIF_P, BADARG); } - else { + if (modp->on_load->code_hdr) { + BIF_TRAP_CODE_PTR_0(BIF_P, + modp->on_load->code_hdr->on_load_function_ptr); + } else { BIF_ERROR(BIF_P, BADARG); } } @@ -765,14 +771,14 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) code_ix = erts_active_code_ix(); modp = erts_get_module(BIF_ARG_1, code_ix); - if (!modp || !modp->old.code_hdr) { + if (!modp || !modp->on_load || !modp->on_load->code_hdr) { error: erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_release_code_write_permission(); BIF_ERROR(BIF_P, BADARG); } - if (modp->old.code_hdr->on_load_function_ptr == NULL) { + if (modp->on_load->code_hdr->on_load_function_ptr == NULL) { goto error; } if (BIF_ARG_2 != am_false && BIF_ARG_2 != am_true) { @@ -781,14 +787,17 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) if (BIF_ARG_2 == am_true) { int i; - struct erl_module_instance t; /* - * Swap old and new code. + * Make the code with the on_load function current. */ - t = modp->curr; - modp->curr = modp->old; - modp->old = t; + + if (modp->curr.code_hdr) { + modp->old = modp->curr; + } + modp->curr = *modp->on_load; + erts_free(ERTS_ALC_T_PREPARED_CODE, modp->on_load); + modp->on_load = 0; /* * The on_load function succeded. Fix up export entries. @@ -1731,7 +1740,8 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) switch (BIF_ARG_2) { - case am_prepare: { + case am_prepare: + case am_prepare_on_load: { /* * Prepare for purge by marking all fun * entries referring to the code to purge @@ -1756,7 +1766,22 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) /* * Any code to purge? */ - erts_rlock_old_code(code_ix); + + if (BIF_ARG_2 == am_prepare_on_load) { + erts_rwlock_old_code(code_ix); + } else { + erts_rlock_old_code(code_ix); + } + + if (BIF_ARG_2 == am_prepare_on_load) { + ASSERT(modp->on_load); + ASSERT(modp->on_load->code_hdr); + purge_state.saved_old = modp->old; + modp->old = *modp->on_load; + erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) modp->on_load); + modp->on_load = 0; + } + if (!modp->old.code_hdr) res = am_false; else { @@ -1774,7 +1799,12 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) ERTS_SET_COPY_LITERAL_AREA(modp->old.code_hdr->literal_area); #endif } - erts_runlock_old_code(code_ix); + + if (BIF_ARG_2 == am_prepare_on_load) { + erts_rwunlock_old_code(code_ix); + } else { + erts_runlock_old_code(code_ix); + } } #ifndef ERTS_SMP @@ -1911,8 +1941,14 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) modp->old.code_length = 0; modp->old.catches = BEAM_CATCHES_NIL; erts_remove_from_ranges(code); + ERTS_BIF_PREP_RET(ret, am_true); } + + if (purge_state.saved_old.code_hdr) { + modp->old = purge_state.saved_old; + purge_state.saved_old.code_hdr = 0; + } erts_rwunlock_old_code(code_ix); } if (is_blocking) { diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index f63addb309..0afdedf6c2 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -833,11 +833,20 @@ erts_finish_loading(Binary* magic, Process* c_p, size = stp->loaded_size; erts_total_code_size += size; - if (stp->on_load) { - inst_p = &mod_tab_p->old; - } else { + + if (!stp->on_load) { inst_p = &mod_tab_p->curr; + } else { + mod_tab_p->on_load = + (struct erl_module_instance *) + erts_alloc(ERTS_ALC_T_PREPARED_CODE, + sizeof(struct erl_module_instance)); + inst_p = mod_tab_p->on_load; + inst_p->nif = 0; + inst_p->num_breakpoints = 0; + inst_p->num_traced_exports = 0; } + inst_p->code_hdr = stp->hdr; inst_p->code_length = size; diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 559b4017e7..3a547982da 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -3203,18 +3203,20 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) if (init_func != NULL) handle = init_func; + this_mi = &module_p->curr; + prev_mi = &module_p->old; if (in_area(caller, module_p->old.code_hdr, module_p->old.code_length)) { - if (module_p->old.code_hdr->on_load_function_ptr) { - this_mi = &module_p->old; + ret = load_nif_error(BIF_P, "old_code", "Calling load_nif from old " + "module '%T' not allowed", mod_atom); + goto error; + } else if (module_p->on_load) { + ASSERT(module_p->on_load->code_hdr->on_load_function_ptr); + if (module_p->curr.code_hdr) { prev_mi = &module_p->curr; } else { - ret = load_nif_error(BIF_P, "old_code", "Calling load_nif from old " - "module '%T' not allowed", mod_atom); - goto error; + prev_mi = &module_p->old; } - } else { - this_mi = &module_p->curr; - prev_mi = &module_p->old; + this_mi = module_p->on_load; } if (init_func == NULL && diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index 3eb11f1693..4f36377450 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -85,6 +85,7 @@ static Module* module_alloc(Module* tmpl) obj->old.num_breakpoints = 0; obj->curr.num_traced_exports = 0; obj->old.num_traced_exports = 0; + obj->on_load = 0; return obj; } @@ -201,6 +202,7 @@ void module_start_staging(void) dst_mod->curr = src_mod->curr; dst_mod->old = src_mod->old; + dst_mod->on_load = src_mod->on_load; } /* @@ -214,6 +216,7 @@ void module_start_staging(void) dst_mod->curr = src_mod->curr; dst_mod->old = src_mod->old; + dst_mod->on_load = src_mod->on_load; } newsz = index_table_sz(dst); erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz)); diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 5a60bc90d9..1c1afc8461 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -39,6 +39,7 @@ typedef struct erl_module { struct erl_module_instance curr; struct erl_module_instance old; /* protected by "old_code" rwlock */ + struct erl_module_instance* on_load; } Module; Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix); -- cgit v1.2.3 From 57513239dc265373521bc4f04c154f17d5f0bae9 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 14 Sep 2016 12:58:27 +0200 Subject: Update vheap size when moving msgq to heap after GC --- erts/emulator/beam/erl_gc.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 9e0919cc9c..e2875cedb3 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -150,6 +150,7 @@ static void move_msgq_to_heap(Process *p); static int reached_max_heap_size(Process *p, Uint total_heap_size, Uint extra_heap_size, Uint extra_old_heap_size); static void init_gc_info(ErtsGCInfo *gcip); +static Uint64 next_vheap_size(Process* p, Uint64 vheap, Uint64 vheap_sz); #ifdef HARDDEBUG static void disallow_heap_frag_ref_in_heap(Process* p); @@ -748,6 +749,9 @@ do_major_collection: p->last_old_htop = p->old_htop; #endif + ASSERT(!p->mbuf); + ASSERT(!ERTS_IS_GC_DESIRED(p)); + return reds; } @@ -2250,7 +2254,9 @@ copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap, static void move_msgq_to_heap(Process *p) { + ErtsMessage **mpp = &p->msg.first; + Uint64 pre_oh = MSO(p).overhead; while (*mpp) { ErtsMessage *mp = *mpp; @@ -2293,6 +2299,11 @@ move_msgq_to_heap(Process *p) mpp = &(*mpp)->next; } + + if (pre_oh != MSO(p).overhead) { + /* Got new binaries; update vheap size... */ + BIN_VHEAP_SZ(p) = next_vheap_size(p, MSO(p).overhead, BIN_VHEAP_SZ(p)); + } } static Uint -- cgit v1.2.3 From 25ea6279b55db5af53040ec4ce9bf73b4aeeaa16 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 14 Sep 2016 13:20:53 +0200 Subject: Ensure we dont use an invalid live heap fragment pointer --- erts/emulator/beam/erl_gc.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index e2875cedb3..027ade45e5 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -388,6 +388,11 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end, return result; } + if (!p->mbuf) { + /* Must have GC:d in BIF call... invalidate live_hf_end */ + live_hf_end = ERTS_INVALID_HFRAG_PTR; + } + if (is_non_value(result)) { if (p->freason == TRAP) { #if HIPE -- cgit v1.2.3 From 4f8071d81aa0690caa3adc734d73a1cb004ad808 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 19 Sep 2016 14:34:57 +0200 Subject: erts: Remove deprecated nif 'reload' feature and instead let erlang:load_nif/2 return {error, {reload, _}} before even trying to load the library if a NIF library has already been successfully loaded for the calling module instance. --- erts/emulator/beam/erl_nif.c | 121 ++++++++++++------------------------------- 1 file changed, 33 insertions(+), 88 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index b53d3d4c19..7442e99cb3 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -3145,7 +3145,6 @@ static ErlNifFunc* next_func(ErlNifEntry* entry, int* incrp, ErlNifFunc* func) BIF_RETTYPE load_nif_2(BIF_ALIST_2) { static const char bad_lib[] = "bad_lib"; - static const char reload[] = "reload"; static const char upgrade[] = "upgrade"; char* lib_name = NULL; void* handle = NULL; @@ -3162,7 +3161,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) Eterm ret = am_ok; int veto; struct erl_module_nif* lib = NULL; - int reload_warning = 0; struct erl_module_instance* this_mi; struct erl_module_instance* prev_mi; @@ -3222,8 +3220,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) this_mi = module_p->on_load; } - if (init_func == NULL && - (err=erts_sys_ddll_open(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) { + if (this_mi->nif != NULL) { + ret = load_nif_error(BIF_P,"reload","NIF library already loaded" + " (reload disallowed since OTP 20)."); + } + else if (init_func == NULL && + (err=erts_sys_ddll_open(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) { const char slogan[] = "Failed to load NIF library"; if (strstr(errdesc.str, lib_name) != NULL) { ret = load_nif_error(BIF_P, "load_failed", "%s: '%s'", slogan, errdesc.str); @@ -3312,7 +3314,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) goto error; } - /* Call load, reload or upgrade: + /* Call load or upgrade: */ @@ -3324,82 +3326,33 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ASSERT(opened_rt_list == NULL); lib->mod = module_p; env.mod_nif = lib; - if (this_mi->nif != NULL) { /*************** Reload ******************/ - /* - * Repeated load_nif calls from same Erlang module instance ("reload") - * is deprecated and was only ment as a development feature not to - * be used in production systems. (See warning below) - */ - int k, old_incr = 0; - ErlNifFunc* old_func; - lib->priv_data = this_mi->nif->priv_data; - - ASSERT(this_mi->nif->entry != NULL); - if (entry->reload == NULL) { - ret = load_nif_error(BIF_P,reload,"Reload not supported by this NIF library."); - goto error; - } - /* Check that no NIF is removed */ - old_func = this_mi->nif->entry->funcs; - for (k=0; k < this_mi->nif->entry->num_of_funcs; k++) { - int incr = 0; - ErlNifFunc* f = entry->funcs; - for (i=0; i < entry->num_of_funcs; i++) { - if (old_func->arity == f->arity - && sys_strcmp(old_func->name, f->name) == 0) { - break; - } - f = next_func(entry, &incr, f); - } - if (i == entry->num_of_funcs) { - ret = load_nif_error(BIF_P,reload,"Reloaded library missing " - "function %T:%s/%u\r\n", mod_atom, - old_func->name, old_func->arity); - goto error; - } - old_func = next_func(this_mi->nif->entry, &old_incr, old_func); - } - erts_pre_nif(&env, BIF_P, lib, NULL); - veto = entry->reload(&env, &lib->priv_data, BIF_ARG_2); - erts_post_nif(&env); - if (veto) { - ret = load_nif_error(BIF_P, reload, "Library reload-call unsuccessful."); - } - else { - commit_opened_resource_types(lib); - this_mi->nif->entry = NULL; /* to prevent 'unload' callback */ - erts_unload_nif(this_mi->nif); - reload_warning = 1; - } + + lib->priv_data = NULL; + if (prev_mi->nif != NULL) { /**************** Upgrade ***************/ + void* prev_old_data = prev_mi->nif->priv_data; + if (entry->upgrade == NULL) { + ret = load_nif_error(BIF_P, upgrade, "Upgrade not supported by this NIF library."); + goto error; + } + erts_pre_nif(&env, BIF_P, lib, NULL); + veto = entry->upgrade(&env, &lib->priv_data, &prev_mi->nif->priv_data, BIF_ARG_2); + erts_post_nif(&env); + if (veto) { + prev_mi->nif->priv_data = prev_old_data; + ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful."); + } + else + commit_opened_resource_types(lib); } - else { - lib->priv_data = NULL; - if (prev_mi->nif != NULL) { /**************** Upgrade ***************/ - void* prev_old_data = prev_mi->nif->priv_data; - if (entry->upgrade == NULL) { - ret = load_nif_error(BIF_P, upgrade, "Upgrade not supported by this NIF library."); - goto error; - } - erts_pre_nif(&env, BIF_P, lib, NULL); - veto = entry->upgrade(&env, &lib->priv_data, &prev_mi->nif->priv_data, BIF_ARG_2); - erts_post_nif(&env); - if (veto) { - prev_mi->nif->priv_data = prev_old_data; - ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful."); - } - else - commit_opened_resource_types(lib); - } - else if (entry->load != NULL) { /********* Initial load ***********/ - erts_pre_nif(&env, BIF_P, lib, NULL); - veto = entry->load(&env, &lib->priv_data, BIF_ARG_2); - erts_post_nif(&env); - if (veto) { - ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful."); - } - else - commit_opened_resource_types(lib); - } + else if (entry->load != NULL) { /********* Initial load ***********/ + erts_pre_nif(&env, BIF_P, lib, NULL); + veto = entry->load(&env, &lib->priv_data, BIF_ARG_2); + erts_post_nif(&env); + if (veto) { + ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful."); + } + else + commit_opened_resource_types(lib); } if (ret == am_ok) { /* @@ -3458,14 +3411,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) erts_release_code_write_permission(); erts_free(ERTS_ALC_T_TMP, lib_name); - if (reload_warning) { - erts_dsprintf_buf_t* dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, - "Repeated calls to erlang:load_nif from module '%T'.\n\n" - "The NIF reload mechanism is deprecated and must not " - "be used in production systems.\n", mod_atom); - erts_send_warning_to_logger(BIF_P->group_leader, dsbufp); - } BIF_RET(ret); } -- cgit v1.2.3 From db12deff975c013a0ac02cfb49a339f8e6af5938 Mon Sep 17 00:00:00 2001 From: Luis Rascao Date: Sat, 10 Sep 2016 21:02:47 +0200 Subject: Allow for easier enabling of distribution debug Also update already deprecated calls. --- erts/emulator/beam/dist.c | 1 + 1 file changed, 1 insertion(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 09c83f1117..9bfeaa3379 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -25,6 +25,7 @@ /* define this to get a lot of debug output */ /* #define ERTS_DIST_MSG_DBG */ +/* #define ERTS_RAW_DIST_MSG_DBG */ #ifdef HAVE_CONFIG_H # include "config.h" -- cgit v1.2.3 From f1ffa5e90e5eecaac890e876760932d1bb1d9c86 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 19 Sep 2016 15:47:18 +0200 Subject: erts: Add ErtsSizeofMember macro (in case it matters) --- erts/emulator/beam/sys.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 2d0628f70e..16e6c33367 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -103,6 +103,8 @@ #define ErtsContainerStruct_(ptr, type, memberv) \ ((type *)((char *)(1 ? (ptr) : ((type *)0)->memberv) - offsetof(type, memberv))) +#define ErtsSizeofMember(type, member) sizeof(((type *)0)->member) + #if defined (__WIN32__) # include "erl_win_sys.h" #else -- cgit v1.2.3 From ff7eb12d002afbffacae5429bd9bb0819aa2d146 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 1 Jun 2016 16:19:04 +0200 Subject: erts: Remove unnecessary access of 'is_resizing' in tables without write_concurrency and remove it totally #ifndef ERTS_SMP --- erts/emulator/beam/erl_db_hash.c | 21 ++++++++++----------- erts/emulator/beam/erl_db_hash.h | 2 +- 2 files changed, 11 insertions(+), 12 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 12ae086b31..16d97f05ee 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -670,8 +670,8 @@ int db_create_hash(Process *p, DbTable *tbl) tb->nsegs = NSEG_1; tb->nslots = SEGSZ; - erts_smp_atomic_init_nob(&tb->is_resizing, 0); #ifdef ERTS_SMP + erts_smp_atomic_init_nob(&tb->is_resizing, 0); if (tb->common.type & DB_FINE_LOCKED) { erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; int i; @@ -2604,23 +2604,22 @@ static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, static ERTS_INLINE int begin_resizing(DbTableHash* tb) { +#ifdef ERTS_SMP if (DB_USING_FINE_LOCKING(tb)) - return !erts_smp_atomic_xchg_acqb(&tb->is_resizing, 1); - else { - if (erts_smp_atomic_read_nob(&tb->is_resizing)) - return 0; - erts_smp_atomic_set_nob(&tb->is_resizing, 1); - return 1; - } + return !erts_atomic_xchg_acqb(&tb->is_resizing, 1); + else + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock)); +#endif + return 1; } static ERTS_INLINE void done_resizing(DbTableHash* tb) { +#ifdef ERTS_SMP if (DB_USING_FINE_LOCKING(tb)) - erts_smp_atomic_set_relb(&tb->is_resizing, 0); - else - erts_smp_atomic_set_nob(&tb->is_resizing, 0); + erts_atomic_set_relb(&tb->is_resizing, 0); +#endif } /* Grow table with one new bucket. diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index e654363cd5..081ff8fafc 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -60,8 +60,8 @@ typedef struct db_table_hash { /* List of slots where elements have been deleted while table was fixed */ erts_smp_atomic_t fixdel; /* (FixedDeletion*) */ erts_smp_atomic_t nactive; /* Number of "active" slots */ - erts_smp_atomic_t is_resizing; /* grow/shrink in progress */ #ifdef ERTS_SMP + erts_smp_atomic_t is_resizing; /* grow/shrink in progress */ DbTableHashFineLocks* locks; #endif #ifdef VALGRIND -- cgit v1.2.3 From abc2f4fdae2d62b5d2843082dbb4437595973b38 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 23 Jun 2016 19:30:37 +0200 Subject: erts: Redesign ets with separate segment tables * Keep it simple(r) * To prepare for both dynamic sized segments and segtabs --- erts/emulator/beam/erl_db.h | 2 - erts/emulator/beam/erl_db_hash.c | 246 +++++++++++++++------------------------ erts/emulator/beam/erl_db_hash.h | 4 +- 3 files changed, 97 insertions(+), 155 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index 1d26c49652..2f3c4a8e1b 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -267,7 +267,5 @@ erts_db_free_nt(ErtsAlcType_t type, void *ptr, Uint size) #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ -#undef ERTS_DB_ALC_MEM_UPDATE_ - #endif /* #if defined(ERTS_WANT_DB_INTERNAL__) && !defined(ERTS_HAVE_DB_INTERNAL__) */ diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 16d97f05ee..82f03e458a 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -84,14 +84,6 @@ #include "erl_db_hash.h" -#ifdef MYDEBUG /* Will fail test case ets_SUITE:memory */ -# define IF_DEBUG(x) x -# define MY_ASSERT(x) ASSERT(x) -#else -# define IF_DEBUG(x) -# define MY_ASSERT(x) -#endif - /* * The following symbols can be manipulated to "tune" the linear hash array */ @@ -102,7 +94,7 @@ #define SEGSZ (1 << SEGSZ_EXP) #define SEGSZ_MASK (SEGSZ-1) -#define NSEG_1 2 /* Size of first segment table (must be at least 2) */ +#define NSEG_1 (ErtsSizeofMember(DbTableHash,first_segtab) / sizeof(struct segment*)) #define NSEG_2 256 /* Size of second segment table */ #define NSEG_INC 128 /* Number of segments to grow after that */ @@ -318,27 +310,21 @@ struct mp_info { /* A table segment */ struct segment { - HashDbTerm* buckets[SEGSZ]; -#ifdef MYDEBUG - int is_ext_segment; -#endif + HashDbTerm* buckets[1]; }; +#define SIZEOF_SEGMENT(N) \ + (offsetof(struct segment,buckets) + sizeof(HashDbTerm*)*(N)) -/* A segment that also contains a segment table */ -struct ext_segment { - struct segment s; /* The segment itself. Must be first */ - +/* An extended segment table */ +struct ext_segtab { + ErtsThrPrgrLaterOp lop; struct segment** prev_segtab; /* Used when table is shrinking */ - int nsegs; /* Size of segtab */ + int prev_nsegs; /* Size of prev_segtab */ + int nsegs; /* Size of this segtab */ struct segment* segtab[1]; /* The segment table */ }; -#define SIZEOF_EXTSEG(NSEGS) \ - (offsetof(struct ext_segment,segtab) + sizeof(struct segment*)*(NSEGS)) - -#if defined(DEBUG) || defined(VALGRIND) -# define EXTSEG(SEGTAB_PTR) \ - ((struct ext_segment*) (((char*)(SEGTAB_PTR)) - offsetof(struct ext_segment,segtab))) -#endif +#define SIZEOF_EXT_SEGTAB(NSEGS) \ + (offsetof(struct ext_segtab,segtab) + sizeof(struct segment*)*(NSEGS)) static ERTS_INLINE void SET_SEGTAB(DbTableHash* tb, @@ -348,41 +334,14 @@ static ERTS_INLINE void SET_SEGTAB(DbTableHash* tb, erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t) segtab); else erts_smp_atomic_set_nob(&tb->segtab, (erts_aint_t) segtab); -#ifdef VALGRIND - tb->top_ptr_to_segment_with_active_segtab = EXTSEG(segtab); -#endif } -/* How the table segments relate to each other: - - ext_segment: ext_segment: "plain" segment - #=================# #================# #=============# - | bucket[0] |<--+ +------->| bucket[256] | +->| bucket[512] | - | bucket[1] | | | | [257] | | | [513] | - : : | | : : | : : - | bucket[255] | | | | [511] | | | [767] | - |-----------------| | | |----------------| | #=============# - | prev_segtab=NULL| | | +--<---prev_segtab | | - | nsegs = 2 | | | | | nsegs = 256 | | -+->| segtab[0] -->-------+---|---|--<---segtab[0] |<-+ | -| | segtab[1] -->-----------+---|--<---segtab[1] | | | -| #=================# | | segtab[2] -->-----|--+ ext_segment: -| | : : | #================# -+----------------<---------------+ | segtab[255] ->----|----->| bucket[255*256]| - #================# | | | - | : : - | |----------------| - +----<---prev_segtab | - : : -*/ - /* ** Forward decl's (static functions) */ -static struct ext_segment* alloc_ext_seg(DbTableHash* tb, unsigned seg_ix, - struct segment** old_segtab); +static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix); static int alloc_seg(DbTableHash *tb); static int free_seg(DbTableHash *tb, int free_records); static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_smp_rwmtx_t** lck_ptr, @@ -666,9 +625,13 @@ int db_create_hash(Process *p, DbTable *tbl) erts_smp_atomic_init_nob(&tb->nactive, SEGSZ); erts_smp_atomic_init_nob(&tb->fixdel, (erts_aint_t)NULL); erts_smp_atomic_init_nob(&tb->segtab, (erts_aint_t)NULL); - SET_SEGTAB(tb, alloc_ext_seg(tb,0,NULL)->segtab); + SET_SEGTAB(tb, tb->first_segtab); tb->nsegs = NSEG_1; tb->nslots = SEGSZ; + tb->first_segtab[0] = (struct segment*) erts_db_alloc(ERTS_ALC_T_DB_SEG, + (DbTable *) tb, + SIZEOF_SEGMENT(SEGSZ)); + sys_memset(tb->first_segtab[0], 0, SIZEOF_SEGMENT(SEGSZ)); #ifdef ERTS_SMP erts_smp_atomic_init_nob(&tb->is_resizing, 0); @@ -2414,34 +2377,29 @@ static int analyze_pattern(DbTableHash *tb, Eterm pattern, return DB_ERROR_NONE; } -static struct ext_segment* alloc_ext_seg(DbTableHash* tb, unsigned seg_ix, - struct segment** old_segtab) +static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix) { - int nsegs; - struct ext_segment* eseg; + struct segment** old_segtab = SEGTAB(tb); + int nsegs = 0; + struct ext_segtab* est; + ASSERT(seg_ix >= NSEG_1); switch (seg_ix) { - case 0: nsegs = NSEG_1; break; - case 1: nsegs = NSEG_2; break; - default: nsegs = seg_ix + NSEG_INC; break; - } - eseg = (struct ext_segment*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, - (DbTable *) tb, - SIZEOF_EXTSEG(nsegs)); - ASSERT(eseg != NULL); - sys_memset(&eseg->s, 0, sizeof(struct segment)); - IF_DEBUG(eseg->s.is_ext_segment = 1); - eseg->prev_segtab = old_segtab; - eseg->nsegs = nsegs; - if (old_segtab) { - ASSERT(nsegs > tb->nsegs); - sys_memcpy(eseg->segtab, old_segtab, tb->nsegs*sizeof(struct segment*)); - } + case NSEG_1: nsegs = NSEG_2; break; + default: nsegs = seg_ix + NSEG_INC; break; + } + ASSERT(nsegs > tb->nsegs); + est = (struct ext_segtab*) erts_db_alloc(ERTS_ALC_T_DB_SEG, + (DbTable *) tb, + SIZEOF_EXT_SEGTAB(nsegs)); + est->nsegs = nsegs; + est->prev_segtab = old_segtab; + est->prev_nsegs = tb->nsegs; + sys_memcpy(est->segtab, old_segtab, tb->nsegs*sizeof(struct segment*)); #ifdef DEBUG - sys_memset(&eseg->segtab[seg_ix], 0, (nsegs-seg_ix)*sizeof(struct segment*)); + sys_memset(&est->segtab[seg_ix], 0, (nsegs-seg_ix)*sizeof(struct segment*)); #endif - eseg->segtab[seg_ix] = &eseg->s; - return eseg; + return est; } /* Extend table with one new segment @@ -2449,36 +2407,32 @@ static struct ext_segment* alloc_ext_seg(DbTableHash* tb, unsigned seg_ix, static int alloc_seg(DbTableHash *tb) { int seg_ix = tb->nslots >> SEGSZ_EXP; - - if (seg_ix+1 == tb->nsegs) { /* New segtab needed (extended segment) */ - struct segment** segtab = SEGTAB(tb); - struct ext_segment* seg = alloc_ext_seg(tb, seg_ix, segtab); - if (seg == NULL) return 0; - segtab[seg_ix] = &seg->s; - /* We don't use the new segtab until next call (see "shrink race") */ - } - else { /* Just a new plain segment */ - struct segment** segtab; - if (seg_ix == tb->nsegs) { /* Time to start use segtab from last call */ - struct ext_segment* eseg; - eseg = (struct ext_segment*) SEGTAB(tb)[seg_ix-1]; - MY_ASSERT(eseg!=NULL && eseg->s.is_ext_segment); - SET_SEGTAB(tb, eseg->segtab); - tb->nsegs = eseg->nsegs; - } - ASSERT(seg_ix < tb->nsegs); - segtab = SEGTAB(tb); - ASSERT(segtab[seg_ix] == NULL); - segtab[seg_ix] = (struct segment*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, - (DbTable *) tb, - sizeof(struct segment)); - if (segtab[seg_ix] == NULL) return 0; - sys_memset(segtab[seg_ix], 0, sizeof(struct segment)); - } + struct segment** segtab; + + if (seg_ix == tb->nsegs) { /* New segtab needed */ + struct ext_segtab* est = alloc_ext_segtab(tb, seg_ix); + SET_SEGTAB(tb, est->segtab); + tb->nsegs = est->nsegs; + } + ASSERT(seg_ix < tb->nsegs); + segtab = SEGTAB(tb); + segtab[seg_ix] = (struct segment*) erts_db_alloc(ERTS_ALC_T_DB_SEG, + (DbTable *) tb, + SIZEOF_SEGMENT(SEGSZ)); + sys_memset(segtab[seg_ix], 0, SIZEOF_SEGMENT(SEGSZ)); tb->nslots += SEGSZ; return 1; } +#ifdef ERTS_SMP +static void dealloc_ext_segtab(void* lop_data) +{ + struct ext_segtab* est = (struct ext_segtab*) lop_data; + + erts_free(ERTS_ALC_T_DB_SEG, est); +} +#endif + /* Shrink table by freeing the top segment ** free_records: 1=free any records in segment, 0=assume segment is empty */ @@ -2486,18 +2440,17 @@ static int free_seg(DbTableHash *tb, int free_records) { const int seg_ix = (tb->nslots >> SEGSZ_EXP) - 1; struct segment** const segtab = SEGTAB(tb); - struct ext_segment* const top = (struct ext_segment*) segtab[seg_ix]; - int bytes; + struct segment* const segp = segtab[seg_ix]; int nrecords = 0; - ASSERT(top != NULL); + ASSERT(segp != NULL); #ifndef DEBUG if (free_records) #endif { int i; for (i=0; is.buckets[i]; + HashDbTerm* p = segp->buckets[i]; while(p != 0) { HashDbTerm* nxt = p->next; ASSERT(free_records); /* segment not empty as assumed? */ @@ -2507,53 +2460,46 @@ static int free_seg(DbTableHash *tb, int free_records) } } } - - /* The "shrink race": - * We must avoid deallocating an extended segment while its segtab may - * still be used by other threads. - * The trick is to stop use a segtab one call earlier. That is, stop use - * a segtab when the segment above it is deallocated. When the segtab is - * later deallocated, it has not been used for a very long time. - * It is even theoretically safe as we have by then rehashed the entire - * segment, seizing *all* locks, so there cannot exist any retarded threads - * still hanging in BUCKET macro with an old segtab pointer. - * For this to work, we must of course allocate a new segtab one call - * earlier in alloc_seg() as well. And this is also the reason why - * the minimum size of the first segtab is 2 and not 1 (NSEG_1). - */ - if (seg_ix == tb->nsegs-1 || seg_ix==0) { /* Dealloc extended segment */ - MY_ASSERT(top->s.is_ext_segment); - ASSERT(segtab != top->segtab || seg_ix==0); - bytes = SIZEOF_EXTSEG(top->nsegs); - } - else { /* Dealloc plain segment */ - struct ext_segment* newtop = (struct ext_segment*) segtab[seg_ix-1]; - MY_ASSERT(!top->s.is_ext_segment); - - if (segtab == newtop->segtab) { /* New top segment is extended */ - MY_ASSERT(newtop->s.is_ext_segment); - if (newtop->prev_segtab != NULL) { - /* Time to use a smaller segtab */ - SET_SEGTAB(tb, newtop->prev_segtab); - tb->nsegs = seg_ix; - ASSERT(tb->nsegs == EXTSEG(SEGTAB(tb))->nsegs); - } - else { - ASSERT(NSEG_1 > 2 && seg_ix==1); - } - } - bytes = sizeof(struct segment); + if (seg_ix >= NSEG_1) { + struct ext_segtab* est = ErtsContainerStruct_(segtab,struct ext_segtab,segtab); + + if (seg_ix == est->prev_nsegs) { /* Dealloc extended segtab */ + ASSERT(est->prev_segtab != NULL); + SET_SEGTAB(tb, est->prev_segtab); + tb->nsegs = est->prev_nsegs; + +#ifdef ERTS_SMP + if (!tb->common.is_thread_safe) { + /* + * Table is doing a graceful shrink operation and we must avoid + * deallocating this segtab while it may still be read by other + * threads. Schedule deallocation with thread progress to make + * sure no lingering threads are still hanging in BUCKET macro + * with an old segtab pointer. + */ + Uint sz = SIZEOF_EXT_SEGTAB(est->nsegs); + ASSERT(sz == ERTS_ALC_DBG_BLK_SZ(est)); + ERTS_DB_ALC_MEM_UPDATE_(tb, sz, 0); + erts_schedule_thr_prgr_later_cleanup_op(dealloc_ext_segtab, + est, + &est->lop, + sz); + } + else +#endif + erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable*)tb, est, + SIZEOF_EXT_SEGTAB(est->nsegs)); + } + else { + + } } + erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb, segp, SIZEOF_SEGMENT(SEGSZ)); - erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb, - (void*)top, bytes); #ifdef DEBUG - if (seg_ix > 0) { - segtab[seg_ix] = NULL; - } else { - SET_SEGTAB(tb, NULL); - } + if (seg_ix < tb->nsegs) + SEGTAB(tb)[seg_ix] = NULL; #endif tb->nslots -= SEGSZ; ASSERT(tb->nslots >= 0); diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index 081ff8fafc..2d9b5e308a 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -51,6 +51,7 @@ typedef struct db_table_hash { DbTableCommon common; erts_smp_atomic_t segtab; /* The segment table (struct segment**) */ + struct segment* first_segtab[1]; erts_smp_atomic_t szm; /* current size mask. */ /* SMP: nslots and nsegs are protected by is_resizing or table write lock */ @@ -64,9 +65,6 @@ typedef struct db_table_hash { erts_smp_atomic_t is_resizing; /* grow/shrink in progress */ DbTableHashFineLocks* locks; #endif -#ifdef VALGRIND - struct ext_segment* top_ptr_to_segment_with_active_segtab; -#endif } DbTableHash; -- cgit v1.2.3 From 0d7a001039dbcab096397f27213b518113a9e5d0 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 24 Aug 2016 16:28:53 +0200 Subject: erts: Enable a smaller first hash segment for ets --- erts/emulator/beam/erl_db_hash.c | 67 ++++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 26 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 82f03e458a..2d24f438ca 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -89,10 +89,20 @@ */ #define CHAIN_LEN 6 /* Medium bucket chain len */ -/* Number of slots per segment */ -#define SEGSZ_EXP 8 -#define SEGSZ (1 << SEGSZ_EXP) -#define SEGSZ_MASK (SEGSZ-1) +/* +** We want the first mandatory segment to be small (to reduce minimal footprint) +** and larger extra segments (to reduce number of alloc/free calls). +*/ + +/* Number of slots in first segment */ +#define FIRST_SEGSZ_EXP 8 +#define FIRST_SEGSZ (1 << FIRST_SEGSZ_EXP) +#define FIRST_SEGSZ_MASK (FIRST_SEGSZ - 1) + +/* Number of slots per extra segment */ +#define EXT_SEGSZ_EXP 11 +#define EXT_SEGSZ (1 << EXT_SEGSZ_EXP) +#define EXT_SEGSZ_MASK (EXT_SEGSZ-1) #define NSEG_1 (ErtsSizeofMember(DbTableHash,first_segtab) / sizeof(struct segment*)) #define NSEG_2 256 /* Size of second segment table */ @@ -115,7 +125,9 @@ #define NACTIVE(tb) ((int)erts_smp_atomic_read_nob(&(tb)->nactive)) #define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems)) -#define BUCKET(tb, i) SEGTAB(tb)[(i) >> SEGSZ_EXP]->buckets[(i) & SEGSZ_MASK] +#define SLOT_IX_TO_SEG_IX(i) (((i)+(EXT_SEGSZ-FIRST_SEGSZ)) >> EXT_SEGSZ_EXP) + +#define BUCKET(tb, i) SEGTAB(tb)[SLOT_IX_TO_SEG_IX(i)]->buckets[(i) & EXT_SEGSZ_MASK] /* * When deleting a table, the number of records to delete. @@ -422,7 +434,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle); static ERTS_INLINE void try_shrink(DbTableHash* tb) { int nactive = NACTIVE(tb); - if (nactive > SEGSZ && NITEMS(tb) < (nactive * CHAIN_LEN) + if (nactive > FIRST_SEGSZ && NITEMS(tb) < (nactive * CHAIN_LEN) && !IS_FIXED(tb)) { shrink(tb, nactive); } @@ -621,17 +633,17 @@ int db_create_hash(Process *p, DbTable *tbl) { DbTableHash *tb = &tbl->hash; - erts_smp_atomic_init_nob(&tb->szm, SEGSZ_MASK); - erts_smp_atomic_init_nob(&tb->nactive, SEGSZ); + erts_smp_atomic_init_nob(&tb->szm, FIRST_SEGSZ_MASK); + erts_smp_atomic_init_nob(&tb->nactive, FIRST_SEGSZ); erts_smp_atomic_init_nob(&tb->fixdel, (erts_aint_t)NULL); erts_smp_atomic_init_nob(&tb->segtab, (erts_aint_t)NULL); SET_SEGTAB(tb, tb->first_segtab); tb->nsegs = NSEG_1; - tb->nslots = SEGSZ; + tb->nslots = FIRST_SEGSZ; tb->first_segtab[0] = (struct segment*) erts_db_alloc(ERTS_ALC_T_DB_SEG, (DbTable *) tb, - SIZEOF_SEGMENT(SEGSZ)); - sys_memset(tb->first_segtab[0], 0, SIZEOF_SEGMENT(SEGSZ)); + SIZEOF_SEGMENT(FIRST_SEGSZ)); + sys_memset(tb->first_segtab[0], 0, SIZEOF_SEGMENT(FIRST_SEGSZ)); #ifdef ERTS_SMP erts_smp_atomic_init_nob(&tb->is_resizing, 0); @@ -649,7 +661,7 @@ int db_create_hash(Process *p, DbTable *tbl) erts_smp_rwmtx_init_opt_x(&tb->locks->lck_vec[i].lck, &rwmtx_opt, "db_hash_slot", make_small(i)); } - /* This important property is needed to guarantee that the buckets + /* This important property is needed to guarantee the two buckets * involved in a grow/shrink operation it protected by the same lock: */ ASSERT(erts_smp_atomic_read_nob(&tb->nactive) % DB_HASH_LOCK_CNT == 0); @@ -2213,12 +2225,12 @@ static int db_free_table_continue_hash(DbTable *tbl) done /= 2; while(tb->nslots != 0) { - free_seg(tb, 1); + done += 1 + EXT_SEGSZ/64 + free_seg(tb, 1); /* * If we have done enough work, get out here. */ - if (++done >= (DELETE_RECORD_LIMIT / CHAIN_LEN / SEGSZ)) { + if (done >= DELETE_RECORD_LIMIT) { return 0; /* Not done */ } } @@ -2406,9 +2418,10 @@ static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix) */ static int alloc_seg(DbTableHash *tb) { - int seg_ix = tb->nslots >> SEGSZ_EXP; + int seg_ix = SLOT_IX_TO_SEG_IX(tb->nslots); struct segment** segtab; + ASSERT(seg_ix > 0); if (seg_ix == tb->nsegs) { /* New segtab needed */ struct ext_segtab* est = alloc_ext_segtab(tb, seg_ix); SET_SEGTAB(tb, est->segtab); @@ -2418,9 +2431,9 @@ static int alloc_seg(DbTableHash *tb) segtab = SEGTAB(tb); segtab[seg_ix] = (struct segment*) erts_db_alloc(ERTS_ALC_T_DB_SEG, (DbTable *) tb, - SIZEOF_SEGMENT(SEGSZ)); - sys_memset(segtab[seg_ix], 0, SIZEOF_SEGMENT(SEGSZ)); - tb->nslots += SEGSZ; + SIZEOF_SEGMENT(EXT_SEGSZ)); + sys_memset(segtab[seg_ix], 0, SIZEOF_SEGMENT(EXT_SEGSZ)); + tb->nslots += EXT_SEGSZ; return 1; } @@ -2438,9 +2451,10 @@ static void dealloc_ext_segtab(void* lop_data) */ static int free_seg(DbTableHash *tb, int free_records) { - const int seg_ix = (tb->nslots >> SEGSZ_EXP) - 1; + const int seg_ix = SLOT_IX_TO_SEG_IX(tb->nslots) - 1; struct segment** const segtab = SEGTAB(tb); struct segment* const segp = segtab[seg_ix]; + Uint seg_sz; int nrecords = 0; ASSERT(segp != NULL); @@ -2448,8 +2462,8 @@ static int free_seg(DbTableHash *tb, int free_records) if (free_records) #endif { - int i; - for (i=0; ibuckets[i]; while(p != 0) { HashDbTerm* nxt = p->next; @@ -2495,13 +2509,14 @@ static int free_seg(DbTableHash *tb, int free_records) } } - erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb, segp, SIZEOF_SEGMENT(SEGSZ)); + seg_sz = (seg_ix == 0) ? FIRST_SEGSZ : EXT_SEGSZ; + erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb, segp, SIZEOF_SEGMENT(seg_sz)); #ifdef DEBUG if (seg_ix < tb->nsegs) SEGTAB(tb)[seg_ix] = NULL; #endif - tb->nslots -= SEGSZ; + tb->nslots -= seg_sz; ASSERT(tb->nslots >= 0); return nrecords; } @@ -2589,7 +2604,7 @@ static void grow(DbTableHash* tb, int nactive) /* Ensure that the slot nactive exists */ if (nactive == tb->nslots) { /* Time to get a new segment */ - ASSERT((nactive & SEGSZ_MASK) == 0); + ASSERT(((nactive-FIRST_SEGSZ) & EXT_SEGSZ_MASK) == 0); if (!alloc_seg(tb)) goto abort; } ASSERT(nactive < tb->nslots); @@ -2668,7 +2683,7 @@ static void shrink(DbTableHash* tb, int nactive) int dst_ix = src_ix & low_szm; ASSERT(dst_ix < src_ix); - ASSERT(nactive > SEGSZ); + ASSERT(nactive > FIRST_SEGSZ); lck = WLOCK_HASH(tb, dst_ix); /* Double check for racing table fixers */ if (!IS_FIXED(tb)) { @@ -2697,7 +2712,7 @@ static void shrink(DbTableHash* tb, int nactive) } WUNLOCK_HASH(lck); - if (tb->nslots - src_ix >= SEGSZ) { + if (tb->nslots - src_ix >= EXT_SEGSZ) { free_seg(tb, 0); } } -- cgit v1.2.3 From 92c98a138638541a710f17f21073b568362502f8 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 24 Aug 2016 18:26:55 +0200 Subject: erts: Reduce ets hash load factor for faster lookup/insert/delete at the expense of about one word per object. --- erts/emulator/beam/erl_db_hash.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 2d24f438ca..1752ec5191 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -87,7 +87,8 @@ /* * The following symbols can be manipulated to "tune" the linear hash array */ -#define CHAIN_LEN 6 /* Medium bucket chain len */ +#define GROW_LIMIT(NACTIVE) ((NACTIVE)*1) +#define SHRINK_LIMIT(NACTIVE) ((NACTIVE) / 2) /* ** We want the first mandatory segment to be small (to reduce minimal footprint) @@ -434,7 +435,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle); static ERTS_INLINE void try_shrink(DbTableHash* tb) { int nactive = NACTIVE(tb); - if (nactive > FIRST_SEGSZ && NITEMS(tb) < (nactive * CHAIN_LEN) + if (nactive > FIRST_SEGSZ && NITEMS(tb) < SHRINK_LIMIT(nactive) && !IS_FIXED(tb)) { shrink(tb, nactive); } @@ -837,7 +838,7 @@ Lnew: WUNLOCK_HASH(lck); { int nactive = NACTIVE(tb); - if (nitems > nactive * (CHAIN_LEN+1) && !IS_FIXED(tb)) { + if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) { grow(tb, nactive); } } @@ -2891,7 +2892,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) WUNLOCK_HASH(lck); nactive = NACTIVE(tb); - if (nitems > nactive * (CHAIN_LEN + 1) && !IS_FIXED(tb)) { + if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) { grow(tb, nactive); } } else { -- cgit v1.2.3 From 25eb3fe353cb0f5c381107e43a865d3a312c8c25 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 30 Aug 2016 11:49:20 +0200 Subject: erts: Suppress failed ETS memory checks due to the grow/shrink hysteresis of the meta tables --- erts/emulator/beam/erl_bif_info.c | 8 ++++++++ erts/emulator/beam/erl_db.c | 30 +++++++++++++++++++++++++++ erts/emulator/beam/erl_db.h | 2 ++ erts/emulator/beam/erl_db_hash.c | 43 ++++++++++++++++++++++++++++++++++++++- erts/emulator/beam/erl_db_hash.h | 2 ++ 5 files changed, 84 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 3fb866733c..abf20a90e4 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -3547,6 +3547,10 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) size_t words = (sizeof(DbTable) + sizeof(Uint) - 1)/sizeof(Uint); BIF_RET(make_small((Uint) words)); } + else if (ERTS_IS_ATOM_STR("DbTable_meta", BIF_ARG_1)) { + /* Used by ets_SUITE (stdlib) */ + BIF_RET(erts_ets_get_meta_state(BIF_P)); + } else if (ERTS_IS_ATOM_STR("check_io_debug", BIF_ARG_1)) { /* Used by driver_SUITE (emulator) */ Uint sz, *szp; @@ -4280,6 +4284,10 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } BIF_RET(am_ok); } + else if (ERTS_IS_ATOM_STR("DbTable_meta", BIF_ARG_1)) { + /* Used by ets_SUITE (stdlib) */ + BIF_RET(erts_ets_restore_meta_state(BIF_P, BIF_ARG_2)); + } } BIF_ERROR(BIF_P, BADARG); diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index bad34211a5..df4e34511f 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -3978,6 +3978,36 @@ erts_ets_colliding_names(Process* p, Eterm name, Uint cnt) return list; } +/* + * For testing only + * Retreive meta table size state + */ +Eterm erts_ets_get_meta_state(Process* p) +{ + Eterm* hp = HAlloc(p, 3); + return TUPLE2(hp, + erts_ets_hash_get_memstate(p, &meta_pid_to_tab->hash), + erts_ets_hash_get_memstate(p, &meta_pid_to_fixed_tab->hash)); +} +/* + * For testing only + * Restore a previously retrieved meta table size state. + * We do this to suppress failed memory checks + * caused by the hysteresis of meta tables grow/shrink limits. + */ +Eterm erts_ets_restore_meta_state(Process* p, Eterm meta_state) +{ + Eterm* tv; + Eterm* hp; + if (!is_tuple_arity(meta_state, 2)) + return am_badarg; + + tv = tuple_val(meta_state); + hp = HAlloc(p, 3); + return TUPLE2(hp, + erts_ets_hash_restore_memstate(&meta_pid_to_tab->hash, tv[1]), + erts_ets_hash_restore_memstate(&meta_pid_to_fixed_tab->hash, tv[2])); +} #ifdef HARDDEBUG /* Here comes some debug functions */ diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index 2f3c4a8e1b..f7eb3dc45c 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -90,6 +90,8 @@ extern Export ets_select_continue_exp; extern erts_smp_atomic_t erts_ets_misc_mem_size; Eterm erts_ets_colliding_names(Process*, Eterm name, Uint cnt); +Eterm erts_ets_get_meta_state(Process* p); +Eterm erts_ets_restore_meta_state(Process* p, Eterm target_state); Uint erts_db_get_max_tabs(void); diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 1752ec5191..581b135233 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -2726,7 +2726,6 @@ static void shrink(DbTableHash* tb, int nactive) done_resizing(tb); } - /* Search a list of tuples for a matching key */ static HashDbTerm* search_list(DbTableHash* tb, Eterm key, @@ -2977,6 +2976,48 @@ void db_calc_stats_hash(DbTableHash* tb, DbHashStats* stats) stats->std_dev_expected = sqrt(stats->avg_chain_len * (1 - 1.0/NACTIVE(tb))); stats->kept_items = kept_items; } + +/* For testing only */ +Eterm erts_ets_hash_get_memstate(Process* p, DbTableHash* tb) +{ + Eterm seg_cnt; + while (!begin_resizing(tb)) + /*spinn*/; + + seg_cnt = make_small(SLOT_IX_TO_SEG_IX(tb->nslots)); + done_resizing(tb); + return seg_cnt; +} +/* For testing only */ +Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate) +{ + int seg_cnt, target; + int nactive; + + if (!is_small(memstate)) + return make_small(__LINE__); + + target = signed_val(memstate); + if (target < 1) + return make_small(__LINE__); + while (1) { + while (!begin_resizing(tb)) + /*spin*/; + seg_cnt = SLOT_IX_TO_SEG_IX(tb->nslots); + nactive = NACTIVE(tb); + done_resizing(tb); + + if (target == seg_cnt) + return am_ok; + if (IS_FIXED(tb)) + return make_small(__LINE__); + if (target < seg_cnt) + shrink(tb, nactive); + else + grow(tb, nactive); + } +} + #ifdef HARDDEBUG void db_check_table_hash(DbTable *tbl) diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index 2d9b5e308a..e209037878 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -107,5 +107,7 @@ typedef struct { }DbHashStats; void db_calc_stats_hash(DbTableHash* tb, DbHashStats*); +Eterm erts_ets_hash_get_memstate(Process*, DbTableHash* tb); +Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate); #endif /* _DB_HASH_H */ -- cgit v1.2.3 From fdc2f2b4a6f6314ae7c183dc4e39e19d05ac89e4 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 2 Sep 2016 20:04:47 +0200 Subject: erts: Fix ets_SUITE:memory by simply asking for the size of struct ext_segtab --- erts/emulator/beam/erl_bif_info.c | 4 +++- erts/emulator/beam/erl_db_hash.c | 5 +++++ erts/emulator/beam/erl_db_hash.h | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index abf20a90e4..13391b7c67 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -3545,7 +3545,9 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("DbTable_words", BIF_ARG_1)) { /* Used by ets_SUITE (stdlib) */ size_t words = (sizeof(DbTable) + sizeof(Uint) - 1)/sizeof(Uint); - BIF_RET(make_small((Uint) words)); + Eterm* hp = HAlloc(BIF_P ,3); + BIF_RET(TUPLE2(hp, make_small((Uint) words), + erts_ets_hash_sizeof_ext_segtab())); } else if (ERTS_IS_ATOM_STR("DbTable_meta", BIF_ARG_1)) { /* Used by ets_SUITE (stdlib) */ diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 581b135233..3f7e14d15d 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -2977,6 +2977,11 @@ void db_calc_stats_hash(DbTableHash* tb, DbHashStats* stats) stats->kept_items = kept_items; } +/* For testing only */ +Eterm erts_ets_hash_sizeof_ext_segtab(void) +{ + return make_small(((SIZEOF_EXT_SEGTAB(0)-1) / sizeof(UWord)) + 1); +} /* For testing only */ Eterm erts_ets_hash_get_memstate(Process* p, DbTableHash* tb) { diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index e209037878..11bd6aa32a 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -107,6 +107,7 @@ typedef struct { }DbHashStats; void db_calc_stats_hash(DbTableHash* tb, DbHashStats*); +Eterm erts_ets_hash_sizeof_ext_segtab(void); Eterm erts_ets_hash_get_memstate(Process*, DbTableHash* tb); Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate); -- cgit v1.2.3 From 1573603177f5ba641390c67b0515defb2786ecfb Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 6 Sep 2016 17:42:30 +0200 Subject: erts: Tweak ets grow/shrink to keep up at contention --- erts/emulator/beam/erl_db_hash.c | 287 +++++++++++++++++++++------------------ erts/emulator/beam/erl_db_hash.h | 8 +- 2 files changed, 161 insertions(+), 134 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 3f7e14d15d..698e4c301b 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -194,6 +194,7 @@ static ERTS_INLINE int add_fixed_deletion(DbTableHash* tb, int ix, #ifdef ERTS_SMP # define DB_HASH_LOCK_MASK (DB_HASH_LOCK_CNT-1) # define GET_LOCK(tb,hval) (&(tb)->locks->lck_vec[(hval) & DB_HASH_LOCK_MASK].lck) +# define GET_LOCK_MAYBE(tb,hval) ((tb)->common.is_thread_safe ? NULL : GET_LOCK(tb,hval)) /* Fine grained read lock */ static ERTS_INLINE erts_smp_rwmtx_t* RLOCK_HASH(DbTableHash* tb, HashValue hval) @@ -330,7 +331,9 @@ struct segment { /* An extended segment table */ struct ext_segtab { +#ifdef ERTS_SMP ErtsThrPrgrLaterOp lop; +#endif struct segment** prev_segtab; /* Used when table is shrinking */ int prev_nsegs; /* Size of prev_segtab */ int nsegs; /* Size of this segtab */ @@ -355,14 +358,14 @@ static ERTS_INLINE void SET_SEGTAB(DbTableHash* tb, ** Forward decl's (static functions) */ static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix); -static int alloc_seg(DbTableHash *tb); +static void alloc_seg(DbTableHash *tb); static int free_seg(DbTableHash *tb, int free_records); static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_smp_rwmtx_t** lck_ptr, HashDbTerm *list); static HashDbTerm* search_list(DbTableHash* tb, Eterm key, HashValue hval, HashDbTerm *list); -static void shrink(DbTableHash* tb, int nactive); -static void grow(DbTableHash* tb, int nactive); +static void shrink(DbTableHash* tb, int nitems); +static void grow(DbTableHash* tb, int nitems); static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, Uint sz, DbTableHash*); static int analyze_pattern(DbTableHash *tb, Eterm pattern, @@ -435,9 +438,10 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle); static ERTS_INLINE void try_shrink(DbTableHash* tb) { int nactive = NACTIVE(tb); - if (nactive > FIRST_SEGSZ && NITEMS(tb) < SHRINK_LIMIT(nactive) + int nitems = NITEMS(tb); + if (nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive) && !IS_FIXED(tb)) { - shrink(tb, nactive); + shrink(tb, nitems); } } @@ -839,7 +843,7 @@ Lnew: { int nactive = NACTIVE(tb); if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) { - grow(tb, nactive); + grow(tb, nitems); } } CHECK_TABLES(); @@ -2417,7 +2421,7 @@ static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix) /* Extend table with one new segment */ -static int alloc_seg(DbTableHash *tb) +static void alloc_seg(DbTableHash *tb) { int seg_ix = SLOT_IX_TO_SEG_IX(tb->nslots); struct segment** segtab; @@ -2435,7 +2439,6 @@ static int alloc_seg(DbTableHash *tb) SIZEOF_SEGMENT(EXT_SEGSZ)); sys_memset(segtab[seg_ix], 0, SIZEOF_SEGMENT(EXT_SEGSZ)); tb->nslots += EXT_SEGSZ; - return 1; } #ifdef ERTS_SMP @@ -2506,9 +2509,6 @@ static int free_seg(DbTableHash *tb, int free_records) erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable*)tb, est, SIZEOF_EXT_SEGTAB(est->nsegs)); } - else { - - } } seg_sz = (seg_ix == 0) ? FIRST_SEGSZ : EXT_SEGSZ; erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb, segp, SIZEOF_SEGMENT(seg_sz)); @@ -2584,85 +2584,93 @@ done_resizing(DbTableHash* tb) #endif } -/* Grow table with one new bucket. +/* Grow table with one or more new buckets. ** Allocate new segment if needed. */ -static void grow(DbTableHash* tb, int nactive) +static void grow(DbTableHash* tb, int nitems) { HashDbTerm** pnext; HashDbTerm** to_pnext; HashDbTerm* p; erts_smp_rwmtx_t* lck; - int from_ix; + int nactive; + int from_ix, to_ix; int szm; + int loop_limit = 5; - if (!begin_resizing(tb)) - return; /* already in progress */ - if (NACTIVE(tb) != nactive) { - goto abort; /* already done (race) */ - } + do { + if (!begin_resizing(tb)) + return; /* already in progress */ + nactive = NACTIVE(tb); + if (nitems <= GROW_LIMIT(nactive)) { + goto abort; /* already done (race) */ + } - /* Ensure that the slot nactive exists */ - if (nactive == tb->nslots) { - /* Time to get a new segment */ - ASSERT(((nactive-FIRST_SEGSZ) & EXT_SEGSZ_MASK) == 0); - if (!alloc_seg(tb)) goto abort; - } - ASSERT(nactive < tb->nslots); + /* Ensure that the slot nactive exists */ + if (nactive == tb->nslots) { + /* Time to get a new segment */ + ASSERT(((nactive-FIRST_SEGSZ) & EXT_SEGSZ_MASK) == 0); + alloc_seg(tb); + } + ASSERT(nactive < tb->nslots); - szm = erts_smp_atomic_read_nob(&tb->szm); - if (nactive <= szm) { - from_ix = nactive & (szm >> 1); - } else { - ASSERT(nactive == szm+1); - from_ix = 0; - szm = (szm<<1) | 1; - } + szm = erts_smp_atomic_read_nob(&tb->szm); + if (nactive <= szm) { + from_ix = nactive & (szm >> 1); + } else { + ASSERT(nactive == szm+1); + from_ix = 0; + szm = (szm<<1) | 1; + } + to_ix = nactive; + + lck = WLOCK_HASH(tb, from_ix); + ERTS_SMP_ASSERT(lck == GET_LOCK_MAYBE(tb,to_ix)); + /* Now a final double check (with the from_ix lock held) + * that we did not get raced by a table fixer. + */ + if (IS_FIXED(tb)) { + WUNLOCK_HASH(lck); + goto abort; + } + erts_smp_atomic_set_nob(&tb->nactive, ++nactive); + if (from_ix == 0) { + if (DB_USING_FINE_LOCKING(tb)) + erts_smp_atomic_set_relb(&tb->szm, szm); + else + erts_smp_atomic_set_nob(&tb->szm, szm); + } + done_resizing(tb); - lck = WLOCK_HASH(tb, from_ix); - /* Now a final double check (with the from_ix lock held) - * that we did not get raced by a table fixer. - */ - if (IS_FIXED(tb)) { - WUNLOCK_HASH(lck); - goto abort; - } - erts_smp_atomic_inc_nob(&tb->nactive); - if (from_ix == 0) { - if (DB_USING_FINE_LOCKING(tb)) - erts_smp_atomic_set_relb(&tb->szm, szm); - else - erts_smp_atomic_set_nob(&tb->szm, szm); - } - done_resizing(tb); + /* Finally, let's split the bucket. We try to do it in a smart way + to keep link order and avoid unnecessary updates of next-pointers */ + pnext = &BUCKET(tb, from_ix); + p = *pnext; + to_pnext = &BUCKET(tb, to_ix); + while (p != NULL) { + if (p->hvalue == INVALID_HASH) { /* rare but possible with fine locking */ + *pnext = p->next; + free_term(tb, p); + p = *pnext; + } + else { + int ix = p->hvalue & szm; + if (ix != from_ix) { + ASSERT(ix == (from_ix ^ ((szm+1)>>1))); + *to_pnext = p; + /* Swap "from" and "to": */ + from_ix = ix; + to_pnext = pnext; + } + pnext = &p->next; + p = *pnext; + } + } + *to_pnext = NULL; + WUNLOCK_HASH(lck); - /* Finally, let's split the bucket. We try to do it in a smart way - to keep link order and avoid unnecessary updates of next-pointers */ - pnext = &BUCKET(tb, from_ix); - p = *pnext; - to_pnext = &BUCKET(tb, nactive); - while (p != NULL) { - if (p->hvalue == INVALID_HASH) { /* rare but possible with fine locking */ - *pnext = p->next; - free_term(tb, p); - p = *pnext; - } - else { - int ix = p->hvalue & szm; - if (ix != from_ix) { - ASSERT(ix == (from_ix ^ ((szm+1)>>1))); - *to_pnext = p; - /* Swap "from" and "to": */ - from_ix = ix; - to_pnext = pnext; - } - pnext = &p->next; - p = *pnext; - } - } - *to_pnext = NULL; + }while (--loop_limit && nitems > GROW_LIMIT(nactive)); - WUNLOCK_HASH(lck); return; abort: @@ -2673,56 +2681,75 @@ abort: /* Shrink table by joining top bucket. ** Remove top segment if it gets empty. */ -static void shrink(DbTableHash* tb, int nactive) -{ - if (!begin_resizing(tb)) - return; /* already in progress */ - if (NACTIVE(tb) == nactive) { - erts_smp_rwmtx_t* lck; - int src_ix = nactive - 1; - int low_szm = erts_smp_atomic_read_nob(&tb->szm) >> 1; - int dst_ix = src_ix & low_szm; - - ASSERT(dst_ix < src_ix); - ASSERT(nactive > FIRST_SEGSZ); - lck = WLOCK_HASH(tb, dst_ix); - /* Double check for racing table fixers */ - if (!IS_FIXED(tb)) { - HashDbTerm** src_bp = &BUCKET(tb, src_ix); - HashDbTerm** dst_bp = &BUCKET(tb, dst_ix); - HashDbTerm** bp = src_bp; - - /* Q: Why join lists by appending "dst" at the end of "src"? - A: Must step through "src" anyway to purge pseudo deleted. */ - while(*bp != NULL) { - if ((*bp)->hvalue == INVALID_HASH) { - HashDbTerm* deleted = *bp; - *bp = deleted->next; - free_term(tb, deleted); - } else { - bp = &(*bp)->next; - } - } - *bp = *dst_bp; - *dst_bp = *src_bp; - *src_bp = NULL; - - erts_smp_atomic_set_nob(&tb->nactive, src_ix); - if (dst_ix == 0) { - erts_smp_atomic_set_relb(&tb->szm, low_szm); - } - WUNLOCK_HASH(lck); - - if (tb->nslots - src_ix >= EXT_SEGSZ) { - free_seg(tb, 0); - } - } - else { - WUNLOCK_HASH(lck); - } +static void shrink(DbTableHash* tb, int nitems) +{ + HashDbTerm** src_bp; + HashDbTerm** dst_bp; + HashDbTerm** bp; + erts_smp_rwmtx_t* lck; + int src_ix, dst_ix, low_szm; + int nactive; + int loop_limit = 5; - } - /*else already done */ + do { + if (!begin_resizing(tb)) + return; /* already in progress */ + nactive = NACTIVE(tb); + if (!(nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive))) { + goto abort; /* already done (race) */ + } + src_ix = nactive - 1; + low_szm = erts_smp_atomic_read_nob(&tb->szm) >> 1; + dst_ix = src_ix & low_szm; + + ASSERT(dst_ix < src_ix); + ASSERT(nactive > FIRST_SEGSZ); + lck = WLOCK_HASH(tb, dst_ix); + ERTS_SMP_ASSERT(lck == GET_LOCK_MAYBE(tb,src_ix)); + /* Double check for racing table fixers */ + if (IS_FIXED(tb)) { + WUNLOCK_HASH(lck); + goto abort; + } + + src_bp = &BUCKET(tb, src_ix); + dst_bp = &BUCKET(tb, dst_ix); + bp = src_bp; + + /* + * We join lists by appending "dst" at the end of "src" + * as we must step through "src" anyway to purge pseudo deleted. + */ + while(*bp != NULL) { + if ((*bp)->hvalue == INVALID_HASH) { + HashDbTerm* deleted = *bp; + *bp = deleted->next; + free_term(tb, deleted); + } else { + bp = &(*bp)->next; + } + } + *bp = *dst_bp; + *dst_bp = *src_bp; + *src_bp = NULL; + + nactive = src_ix; + erts_smp_atomic_set_nob(&tb->nactive, nactive); + if (dst_ix == 0) { + erts_smp_atomic_set_relb(&tb->szm, low_szm); + } + WUNLOCK_HASH(lck); + + if (tb->nslots - src_ix >= EXT_SEGSZ) { + free_seg(tb, 0); + } + done_resizing(tb); + + } while (--loop_limit + && nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive)); + return; + +abort: done_resizing(tb); } @@ -2892,7 +2919,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) nactive = NACTIVE(tb); if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) { - grow(tb, nactive); + grow(tb, nitems); } } else { WUNLOCK_HASH(lck); @@ -2997,7 +3024,6 @@ Eterm erts_ets_hash_get_memstate(Process* p, DbTableHash* tb) Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate) { int seg_cnt, target; - int nactive; if (!is_small(memstate)) return make_small(__LINE__); @@ -3009,7 +3035,6 @@ Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate) while (!begin_resizing(tb)) /*spin*/; seg_cnt = SLOT_IX_TO_SEG_IX(tb->nslots); - nactive = NACTIVE(tb); done_resizing(tb); if (target == seg_cnt) @@ -3017,9 +3042,9 @@ Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate) if (IS_FIXED(tb)) return make_small(__LINE__); if (target < seg_cnt) - shrink(tb, nactive); + shrink(tb, 0); else - grow(tb, nactive); + grow(tb, INT_MAX); } } @@ -3031,7 +3056,7 @@ void db_check_table_hash(DbTable *tbl) HashDbTerm* list; int j; - for (j = 0; j < tb->nactive; j++) { + for (j = 0; j < NACTIVE(tb); j++) { if ((list = BUCKET(tb,j)) != 0) { while (list != 0) { if (!is_tuple(make_tuple(list->dbterm.tpl))) { diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index 11bd6aa32a..6d25c73549 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -50,17 +50,19 @@ typedef struct db_table_hash_fine_locks { typedef struct db_table_hash { DbTableCommon common; + /* SMP: szm and nactive are write-protected by is_resizing or table write lock */ + erts_smp_atomic_t szm; /* current size mask. */ + erts_smp_atomic_t nactive; /* Number of "active" slots */ + erts_smp_atomic_t segtab; /* The segment table (struct segment**) */ struct segment* first_segtab[1]; - erts_smp_atomic_t szm; /* current size mask. */ - + /* SMP: nslots and nsegs are protected by is_resizing or table write lock */ int nslots; /* Total number of slots */ int nsegs; /* Size of segment table */ /* List of slots where elements have been deleted while table was fixed */ erts_smp_atomic_t fixdel; /* (FixedDeletion*) */ - erts_smp_atomic_t nactive; /* Number of "active" slots */ #ifdef ERTS_SMP erts_smp_atomic_t is_resizing; /* grow/shrink in progress */ DbTableHashFineLocks* locks; -- cgit v1.2.3 From d100187bde4f3f555b4f3cd9561ec8c67a528895 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 13 Sep 2016 18:28:09 +0200 Subject: erts: Unify reduction count for ets:select to be per object as the other select-variants and not per table slot. --- erts/emulator/beam/erl_db_hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 698e4c301b..40df1c5356 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -1494,6 +1494,7 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl, match_list = CONS(hp, match_res, match_list); ++got; } + --num_left; } current = current->next; } @@ -1511,7 +1512,6 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl, } } else { /* Key is variable */ - --num_left; if ((slot_ix=next_slot(tb,slot_ix,&lck)) == 0) { slot_ix = -1; -- cgit v1.2.3 From ca7578aca806a6eb3e816c918120db78f485f5cb Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 26 Sep 2016 15:05:21 +0200 Subject: Fix initialization dependencies between time, lock-check and lcnt --- erts/emulator/beam/erl_init.c | 3 --- erts/emulator/beam/erl_lock_count.c | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index d9c3b0dcf4..611a98a30f 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -762,9 +762,6 @@ early_init(int *argc, char **argv) /* erts_thr_progress_pre_init(); #endif -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_init(); -#endif #ifdef ERTS_SMP erts_smp_atomic32_init_nob(&erts_writing_erl_crash_dump, 0L); erts_tsd_key_create(&erts_is_crash_dumping_key,"erts_is_crash_dumping_key"); diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index bd00480ba2..7eba2e34e9 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -274,11 +274,11 @@ void erts_lcnt_init() { lcnt_unlock(); - /* set start timer and zero statistics */ - erts_lcnt_clear_counters(); } void erts_lcnt_late_init() { + /* set start timer and zero statistics */ + erts_lcnt_clear_counters(); erts_thr_install_exit_handler(erts_lcnt_thread_exit_handler); } -- cgit v1.2.3 From 23ecba248ade6e4f1a6ed58c6636598643916019 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 29 Sep 2016 16:43:44 +0200 Subject: Fix minor soft purge bug A process calling a fun from a module currently being soft-purged could race with failure of the soft purge. When this happened the call triggered loading of the module which erroneously would load new code for the module in the case no new generation was currently loaded. --- erts/emulator/beam/erl_fun.c | 1 - 1 file changed, 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index c639ba623f..5258d83a18 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -236,7 +236,6 @@ erts_fun_purge_abort_prepare(ErlFunEntry **funs, Uint no) ErlFunEntry *fe = funs[ix]; if (fe->address == unloaded_fun) fe->address = fe->pend_purge_address; - fe->pend_purge_address = NULL; } } -- cgit v1.2.3 From b001b56e061454f3175d9d9a99e2efb945a696b9 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Tue, 4 Oct 2016 10:24:56 +0200 Subject: erts: Do tracer liveness check on current tracer This fixes a fault introduced in 19.0 where an invalid tracer would block setting of a new tracer on a process. --- erts/emulator/beam/erl_bif_trace.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 66e5146da0..96275eb228 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -512,7 +512,7 @@ start_trace(Process *c_p, ErtsTracer tracer, && !ERTS_TRACER_COMPARE(ERTS_TRACER(port), tracer)) { /* This tracee is already being traced, and not by the * tracer to be */ - if (erts_is_tracer_enabled(tracer, common)) { + if (erts_is_tracer_enabled(ERTS_TRACER(port), common)) { /* The tracer is still in use */ return 1; } @@ -715,8 +715,8 @@ Eterm erts_internal_trace_3(BIF_ALIST_3) Process* tracee_p = erts_pix2proc(i); if (! tracee_p) continue; - start_trace(p, tracer, &tracee_p->common, on, mask); - matches++; + if (!start_trace(p, tracer, &tracee_p->common, on, mask)) + matches++; } } if (ports || mods) { @@ -730,8 +730,8 @@ Eterm erts_internal_trace_3(BIF_ALIST_3) state = erts_atomic32_read_nob(&tracee_port->state); if (state & ERTS_PORT_SFLGS_DEAD) continue; - start_trace(p, tracer, &tracee_port->common, on, mask); - matches++; + if (!start_trace(p, tracer, &tracee_port->common, on, mask)) + matches++; } } } -- cgit v1.2.3 From ee6e1bbfab12d5b1106fab2745fc73f7ad7b473e Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 4 Oct 2016 16:30:17 +0200 Subject: erts: Print error code from failed NIF load/upgrade/reload in Text part of error tuple, like {error, {load, "Library load-call unsuccessful (606)}} --- erts/emulator/beam/erl_nif.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 3a547982da..9a873857b9 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -3360,7 +3360,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) veto = entry->reload(&env, &lib->priv_data, BIF_ARG_2); erts_post_nif(&env); if (veto) { - ret = load_nif_error(BIF_P, reload, "Library reload-call unsuccessful."); + ret = load_nif_error(BIF_P, reload, "Library reload-call unsuccessful (%d).", veto); } else { commit_opened_resource_types(lib); @@ -3382,7 +3382,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) erts_post_nif(&env); if (veto) { prev_mi->nif->priv_data = prev_old_data; - ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful."); + ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful (%d).", veto); } else commit_opened_resource_types(lib); @@ -3392,7 +3392,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) veto = entry->load(&env, &lib->priv_data, BIF_ARG_2); erts_post_nif(&env); if (veto) { - ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful."); + ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful (%d).", veto); } else commit_opened_resource_types(lib); -- cgit v1.2.3 From 65737b70b9b71d53147e4155f8974b0fad01e1f6 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Sun, 8 May 2016 15:54:33 +0200 Subject: erts: Refactor module_start_staging with a copy_module() function. --- erts/emulator/beam/module.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index 4f36377450..d4f6e17c56 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -181,6 +181,13 @@ static ErtsCodeIndex dbg_load_code_ix = 0; static int entries_at_start_staging = 0; +static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod) +{ + dst_mod->curr = src_mod->curr; + dst_mod->old = src_mod->old; + dst_mod->on_load = src_mod->on_load; +} + void module_start_staging(void) { IndexTable* src = &module_tables[erts_active_code_ix()]; @@ -199,10 +206,7 @@ void module_start_staging(void) src_mod = (Module*) erts_index_lookup(src, i); dst_mod = (Module*) erts_index_lookup(dst, i); ASSERT(src_mod->module == dst_mod->module); - - dst_mod->curr = src_mod->curr; - dst_mod->old = src_mod->old; - dst_mod->on_load = src_mod->on_load; + copy_module(dst_mod, src_mod); } /* @@ -213,10 +217,7 @@ void module_start_staging(void) src_mod = (Module*) erts_index_lookup(src, i); dst_mod = (Module*) index_put_entry(dst, src_mod); ASSERT(dst_mod != src_mod); - - dst_mod->curr = src_mod->curr; - dst_mod->old = src_mod->old; - dst_mod->on_load = src_mod->on_load; + copy_module(dst_mod, src_mod); } newsz = index_table_sz(dst); erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz)); -- cgit v1.2.3 From 099c60de4033d7b397d4b3fb47f183b52fcba855 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 15 Sep 2016 21:21:10 +0200 Subject: erts: Improve hipe load/upgrade/purge machinery A step toward better integration of hipe load and purge Highlights: * code_server no longer needs to call hipe_unified_loader:post_beam_load/1 Instead new internal function hipe_redirect_to_module() is called by loading BIFs to patch native call sites if needed. * hipe_purge_module() is called by erts_internal:purge_module/2 to purge any native code. * struct hipe_mfa_info redesigned and only used for exported functions that are called from or implemented by native code. A list of native call sites (struct hipe_ref) are kept for each hipe_mfa_info. * struct hipe_sdesc used by hipe_find_mfa_from_ra() to build native stack traces. --- erts/emulator/beam/beam_bif_load.c | 91 ++++++++++++++++++++++++------------ erts/emulator/beam/beam_emu.c | 1 + erts/emulator/beam/beam_load.c | 94 +++++++++++++++++++++++++++++++------- erts/emulator/beam/beam_load.h | 20 ++++++++ erts/emulator/beam/export.c | 8 ++++ erts/emulator/beam/module.c | 69 +++++++++++++++++++++------- erts/emulator/beam/module.h | 15 ++++++ 7 files changed, 237 insertions(+), 61 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 153fa205ed..237513095a 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -36,6 +36,12 @@ #include "erl_nif.h" #include "erl_bits.h" #include "erl_thr_progress.h" +#ifdef HIPE +# include "hipe_bif0.h" +# define IF_HIPE(X) (X) +#else +# define IF_HIPE(X) (0) +#endif #ifdef HIPE # include "hipe_stack.h" @@ -169,6 +175,11 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) if (res == BIF_ARG_1) { erts_end_staging_code_ix(); erts_commit_staging_code_ix(); +#ifdef HIPE + if (!modp) + modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); + hipe_redirect_to_module(modp); +#endif } else { erts_abort_staging_code_ix(); @@ -241,7 +252,7 @@ struct m { Uint exception; }; -static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int); +static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int, int); #ifdef ERTS_SMP static void smp_code_ix_commiter(void*); @@ -377,8 +388,9 @@ finish_loading_1(BIF_ALIST_1) for (i = 0; i < n; i++) { if (p[i].modp->curr.num_breakpoints > 0 || p[i].modp->curr.num_traced_exports > 0 || - erts_is_default_trace_enabled()) { - /* tracing involved, fallback with thread blocking */ + erts_is_default_trace_enabled() || + IF_HIPE(hipe_need_blocking(p[i].modp))) { + /* tracing or hipe need thread blocking */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; @@ -436,32 +448,36 @@ finish_loading_1(BIF_ALIST_1) } done: - return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n); + return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n, 1); } static Eterm staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, - struct m* loaded, int nloaded) + struct m* mods, int nmods, int free_mods) { #ifdef ERTS_SMP if (is_blocking || !commit) #endif { if (commit) { + int i; erts_end_staging_code_ix(); erts_commit_staging_code_ix(); - if (loaded) { - int i; - for (i=0; i < nloaded; i++) { - set_default_trace_pattern(loaded[i].module); + + for (i=0; i < nmods; i++) { + if (mods[i].modp->curr.code_hdr) { + set_default_trace_pattern(mods[i].module); } + #ifdef HIPE + hipe_redirect_to_module(mods[i].modp); + #endif } } else { erts_abort_staging_code_ix(); } - if (loaded) { - erts_free(ERTS_ALC_T_LOADER_TMP, loaded); + if (free_mods) { + erts_free(ERTS_ALC_T_LOADER_TMP, mods); } if (is_blocking) { erts_smp_thr_progress_unblock(); @@ -474,8 +490,8 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, else { ASSERT(is_value(res)); - if (loaded) { - erts_free(ERTS_ALC_T_LOADER_TMP, loaded); + if (free_mods) { + erts_free(ERTS_ALC_T_LOADER_TMP, mods); } erts_end_staging_code_ix(); /* @@ -653,8 +669,9 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) } else { if (modp->curr.num_breakpoints > 0 || - modp->curr.num_traced_exports > 0) { - /* we have tracing, retry single threaded */ + modp->curr.num_traced_exports > 0 || + IF_HIPE(hipe_need_blocking(modp))) { + /* tracing or hipe need to go single threaded */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; @@ -668,7 +685,14 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) success = 1; } } - return staging_epilogue(BIF_P, success, res, is_blocking, NULL, 0); + { + struct m mod; + Eterm retval; + mod.module = BIF_ARG_1; + mod.modp = modp; + retval = staging_epilogue(BIF_P, success, res, is_blocking, &mod, 1, 0); + return retval; + } } BIF_RETTYPE module_loaded_1(BIF_ALIST_1) @@ -809,6 +833,9 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) } modp->curr.code_hdr->on_load_function_ptr = NULL; set_default_trace_pattern(BIF_ARG_1); + #ifdef HIPE + hipe_redirect_to_module(modp); + #endif } else if (BIF_ARG_2 == am_false) { int i; @@ -1619,15 +1646,18 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) /* * Unload any NIF library */ - if (modp->old.nif != NULL) { + if (modp->old.nif != NULL + || IF_HIPE(hipe_purge_need_blocking(modp))) { /* ToDo: Do unload nif without blocking */ erts_rwunlock_old_code(code_ix); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; erts_rwlock_old_code(code_ix); - erts_unload_nif(modp->old.nif); - modp->old.nif = NULL; + if (modp->old.nif) { + erts_unload_nif(modp->old.nif); + modp->old.nif = NULL; + } } /* @@ -1646,7 +1676,9 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) modp->old.code_length = 0; modp->old.catches = BEAM_CATCHES_NIL; erts_remove_from_ranges(code); - +#ifdef HIPE + hipe_purge_module(modp); +#endif ERTS_BIF_PREP_RET(ret, am_true); } @@ -1719,6 +1751,8 @@ delete_code(Module* modp) (BeamInstr) BeamOp(op_i_generic_breakpoint)) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(modp->curr.num_traced_exports > 0); + DBG_TRACE_MFA(ep->code[0],ep->code[1],ep->code[2], + "export trace cleared, code_ix=%d", code_ix); erts_clear_export_break(modp, ep->code+3); } else ASSERT(ep->code[3] == (BeamInstr) em_call_error_handler @@ -1727,17 +1761,16 @@ delete_code(Module* modp) ep->addressv[code_ix] = ep->code+3; ep->code[3] = (BeamInstr) em_call_error_handler; ep->code[4] = 0; + DBG_TRACE_MFA(ep->code[0],ep->code[1],ep->code[2], + "export invalidation, code_ix=%d", code_ix); } } ASSERT(modp->curr.num_breakpoints == 0); ASSERT(modp->curr.num_traced_exports == 0); + DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "delete_code old.first_hipe_ref=%p", modp->curr.first_hipe_ref); modp->old = modp->curr; - modp->curr.code_hdr = NULL; - modp->curr.code_length = 0; - modp->curr.catches = BEAM_CATCHES_NIL; - modp->curr.nif = NULL; - + erts_module_instance_init(&modp->curr); } @@ -1751,9 +1784,11 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module) * if not, delete old code; error if old code already exists. */ - if (modp->curr.code_hdr && modp->old.code_hdr) { - return am_not_purged; - } else if (!modp->old.code_hdr) { /* Make the current version old. */ + if (modp->curr.code_hdr) { + if (modp->old.code_hdr) { + return am_not_purged; + } + /* Make the current version old. */ delete_code(modp); } return NIL; diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 0ba06058a5..ccb0f786ec 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -6105,6 +6105,7 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) Uint sz; int i; + DBG_TRACE_MFA(fi[0], fi[1], fi[2], "call_error_handler"); /* * Search for the error_handler module. */ diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index d69b18e22f..9b206f9a23 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -482,7 +482,8 @@ static void free_literal_fragment(ErlHeapFragment*); static void loader_state_dtor(Binary* magic); static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, - BeamCodeHeader* code, Uint size); + BeamCodeHeader* code, Uint size, + void* hipe_code_start, UWord hipe_code_size); static int init_iff_file(LoaderState* stp, byte* code, Uint size); static int scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory); @@ -842,9 +843,7 @@ erts_finish_loading(Binary* magic, Process* c_p, erts_alloc(ERTS_ALC_T_PREPARED_CODE, sizeof(struct erl_module_instance)); inst_p = mod_tab_p->on_load; - inst_p->nif = 0; - inst_p->num_breakpoints = 0; - inst_p->num_traced_exports = 0; + erts_module_instance_init(inst_p); } inst_p->code_hdr = stp->hdr; @@ -1094,7 +1093,8 @@ loader_state_dtor(Binary* magic) static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, - BeamCodeHeader* code_hdr, Uint size) + BeamCodeHeader* code_hdr, Uint size, + void* hipe_code_start, UWord hipe_code_size) { Module* modp; Eterm retval; @@ -1117,6 +1117,17 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, modp->curr.code_hdr = code_hdr; modp->curr.code_length = size; modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */ +#if defined(HIPE) + DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code new_hipe_refs = %p", modp->new_hipe_refs); + modp->curr.first_hipe_ref = modp->new_hipe_refs; + modp->curr.first_hipe_sdesc = modp->new_hipe_sdesc; + modp->curr.hipe_code_start = hipe_code_start; + modp->new_hipe_refs = NULL; + modp->new_hipe_sdesc = NULL; +# ifdef DEBUG + modp->curr.hipe_code_size = hipe_code_size; +# endif +#endif /* * Update ranges (used for finding a function from a PC value). @@ -4773,9 +4784,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) } ep = erts_export_put(stp->module, stp->export[i].function, stp->export[i].arity); - if (!on_load) { - ep->addressv[erts_staging_code_ix()] = address; - } else { + if (on_load) { /* * on_load: Don't make any of the exported functions * callable yet. Keep any function in the current @@ -4783,6 +4792,8 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) */ ep->code[4] = (BeamInstr) address; } + else + ep->addressv[erts_staging_code_ix()] = address; } /* @@ -5989,17 +6000,12 @@ code_module_md5_1(BIF_ALIST_1) static BeamInstr* make_stub(BeamInstr* fp, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode) { + DBG_TRACE_MFA(mod,func,arity,"make beam stub at %p", &fp[5]); fp[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI); fp[1] = native; fp[2] = mod; fp[3] = func; fp[4] = arity; -#ifdef HIPE - if (native) { - fp[5] = BeamOpCode(op_move_return_n); - hipe_mfa_save_orig_beam_op(mod, func, arity, fp+5); - } -#endif fp[5] = OpCode; return fp + WORDS_PER_FUNCTION; } @@ -6087,6 +6093,8 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) if (stp->export[i].function == function && stp->export[i].arity == arity) { Export* ep = erts_export_put(mod, function, arity); ep->addressv[erts_staging_code_ix()] = fp+5; + DBG_TRACE_MFA(mod,function,arity,"set beam stub at %p in export at %p (code_ix=%d)", + fp+5, ep, erts_staging_code_ix()); return; } } @@ -6282,6 +6290,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) byte* temp_alloc = NULL; byte* bytes; Uint size; + UWord hipe_code_start = NULL, hipe_code_size = 0; /* * Must initialize stp->lambdas here because the error handling code @@ -6297,7 +6306,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) goto error; } tp = tuple_val(Info); - if (tp[0] != make_arityval(3)) { + if (tp[0] != make_arityval(5)) { goto error; } Funcs = tp[1]; @@ -6314,6 +6323,15 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) } size = binary_size(Beam); +#ifdef HIPE + if (!term_to_Uint(tp[4], &hipe_code_start)) + goto error; +# ifdef DEBUG + if (!term_to_Uint(tp[5], &hipe_code_size)) + goto error; +# endif +#endif + /* * Scan the Beam binary and read the interesting sections. */ @@ -6476,7 +6494,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) */ rval = stub_insert_new_code(p, 0, p->group_leader, Mod, - code_hdr, code_size); + code_hdr, code_size, + (void*)hipe_code_start, hipe_code_size); if (rval != NIL) { goto error; } @@ -6517,3 +6536,46 @@ static int safe_mul(UWord a, UWord b, UWord* resp) } } +#ifdef ENABLE_DBG_TRACE_MFA + +#define MFA_MAX 10 +Eterm dbg_trace_m[MFA_MAX]; +Eterm dbg_trace_f[MFA_MAX]; +Uint dbg_trace_a[MFA_MAX]; +unsigned int dbg_trace_ix = 0; + +void dbg_set_traced_mfa(const char* m, const char* f, Uint a) +{ + unsigned i = dbg_trace_ix++; + ASSERT(i < MFA_MAX); + dbg_trace_m[i] = am_atom_put(m, strlen(m)); + dbg_trace_f[i] = am_atom_put(f, strlen(f)); + dbg_trace_a[i] = a; +} + +int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a) +{ + unsigned int i; + for (i = 0; i < dbg_trace_ix; ++i) { + if (m == dbg_trace_m[i] && + (!f || (f == dbg_trace_f[i] && a == dbg_trace_a[i]))) { + + return i+1; + } + } + return 0; +} + +void dbg_vtrace_mfa(unsigned ix, const char* format, ...) +{ + va_list arglist; + va_start(arglist, format); + ASSERT(--ix < MFA_MAX); + erts_fprintf(stderr, "MFA TRACE %T:%T/%u: ", + dbg_trace_m[ix], dbg_trace_f[ix], (int)dbg_trace_a[ix]); + + erts_vfprintf(stderr, format, arglist); + va_end(arglist); +} + +#endif /* ENABLE_DBG_TRACE_MFA */ diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 6be4031822..5f0f34fcdd 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -151,4 +151,24 @@ struct BeamCodeLineTab_ { #define LOC_FILE(Loc) ((Loc) >> 24) #define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1)) +#ifdef DEBUG +# define ENABLE_DBG_TRACE_MFA +#endif + +#ifdef ENABLE_DBG_TRACE_MFA + +void dbg_set_traced_mfa(const char* m, const char* f, Uint a); +int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a); +void dbg_vtrace_mfa(unsigned ix, const char* format, ...); +#define DBG_TRACE_MFA(M,F,A,FMT, ...) do {\ + unsigned ix;\ + if ((ix=dbg_is_traced_mfa(M,F,A))) \ + dbg_vtrace_mfa(ix, FMT"\n", ##__VA_ARGS__);\ + }while(0) + +#else +# define dbg_set_traced_mfa(M,F,A) +# define DBG_TRACE_MFA(M,F,A,FMT, ...) +#endif /* ENABLE_DBG_TRACE_MFA */ + #endif /* _BEAM_LOAD_H */ diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 2a19211987..9da7fae1dc 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -145,6 +145,9 @@ export_alloc(struct export_entry* tmpl_e) blob->entryv[ix].ep = &blob->exp; } ix = 0; + + DBG_TRACE_MFA(obj->code[0], obj->code[1], obj->code[2], + "export allocation at %p", obj); } else { /* Existing entry in another table, use free entry in blob */ blob = entry_to_blob(tmpl_e); @@ -163,9 +166,14 @@ export_free(struct export_entry* obj) obj->slot.index = -1; for (i=0; i < ERTS_NUM_CODE_IX; i++) { if (blob->entryv[i].slot.index >= 0) { + DBG_TRACE_MFA(blob->exp.code[0], blob->exp.code[1], blob->exp.code[2], + "export entry slot %u freed for %p", + (obj - blob->entryv), &blob->exp); return; } } + DBG_TRACE_MFA(blob->exp.code[0], blob->exp.code[1], blob->exp.code[2], + "export blob deallocation at %p", &blob->exp); erts_free(ERTS_ALC_T_EXPORT, blob); erts_smp_atomic_add_nob(&total_entries_bytes, -sizeof(*blob)); } diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index d4f6e17c56..93ce8fcf9f 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -26,6 +26,7 @@ #include "erl_vm.h" #include "global.h" #include "module.h" +#include "beam_catches.h" #ifdef DEBUG # define IF_DEBUG(x) x @@ -67,6 +68,23 @@ static int module_cmp(Module* tmpl, Module* obj) return tmpl->module != obj->module; } +void erts_module_instance_init(struct erl_module_instance* modi) +{ + modi->code_hdr = 0; + modi->code_length = 0; + modi->catches = BEAM_CATCHES_NIL; + modi->nif = NULL; + modi->num_breakpoints = 0; + modi->num_traced_exports = 0; +#ifdef HIPE + modi->first_hipe_ref = NULL; + modi->first_hipe_sdesc = NULL; + modi->hipe_code_start = NULL; +# ifdef DEBUG + modi->hipe_code_size = 0; +# endif +#endif +} static Module* module_alloc(Module* tmpl) { @@ -74,18 +92,16 @@ static Module* module_alloc(Module* tmpl) erts_smp_atomic_add_nob(&tot_module_bytes, sizeof(Module)); obj->module = tmpl->module; - obj->curr.code_hdr = 0; - obj->old.code_hdr = 0; - obj->curr.code_length = 0; - obj->old.code_length = 0; obj->slot.index = -1; - obj->curr.nif = NULL; - obj->old.nif = NULL; - obj->curr.num_breakpoints = 0; - obj->old.num_breakpoints = 0; - obj->curr.num_traced_exports = 0; - obj->old.num_traced_exports = 0; + erts_module_instance_init(&obj->curr); + erts_module_instance_init(&obj->old); obj->on_load = 0; +#ifdef HIPE + obj->first_hipe_mfa = NULL; + obj->new_hipe_refs = NULL; + obj->new_hipe_sdesc = NULL; +#endif + DBG_TRACE_MFA(make_atom(obj->module), 0, 0, "module_alloc"); return obj; } @@ -139,19 +155,14 @@ erts_get_module(Eterm mod, ErtsCodeIndex code_ix) } } -Module* -erts_put_module(Eterm mod) + +static Module* put_module(Eterm mod, IndexTable* mod_tab) { Module e; - IndexTable* mod_tab; int oldsz, newsz; Module* res; ASSERT(is_atom(mod)); - ERTS_SMP_LC_ASSERT(erts_initialized == 0 - || erts_has_code_write_permission()); - - mod_tab = &module_tables[erts_staging_code_ix()]; e.module = atom_val(mod); oldsz = index_table_sz(mod_tab); res = (Module*) index_put_entry(mod_tab, (void*) &e); @@ -160,6 +171,24 @@ erts_put_module(Eterm mod) return res; } +Module* +erts_put_module(Eterm mod) +{ + ERTS_SMP_LC_ASSERT(erts_initialized == 0 + || erts_has_code_write_permission()); + + return put_module(mod, &module_tables[erts_staging_code_ix()]); +} + +Module* +erts_put_active_module(Eterm mod) +{ + ASSERT(is_atom(mod)); + //SVERK Why not? ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + return put_module(mod, &module_tables[erts_active_code_ix()]); +} + Module *module_code(int i, ErtsCodeIndex code_ix) { return (Module*) erts_index_lookup(&module_tables[code_ix], i); @@ -186,6 +215,11 @@ static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod) dst_mod->curr = src_mod->curr; dst_mod->old = src_mod->old; dst_mod->on_load = src_mod->on_load; +#ifdef HIPE + dst_mod->first_hipe_mfa = src_mod->first_hipe_mfa; + dst_mod->new_hipe_refs = src_mod->new_hipe_refs; + dst_mod->new_hipe_sdesc = src_mod->new_hipe_sdesc; +#endif } void module_start_staging(void) @@ -217,6 +251,7 @@ void module_start_staging(void) src_mod = (Module*) erts_index_lookup(src, i); dst_mod = (Module*) index_put_entry(dst, src_mod); ASSERT(dst_mod != src_mod); + copy_module(dst_mod, src_mod); } newsz = index_table_sz(dst); diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 1c1afc8461..694583597b 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -30,6 +30,14 @@ struct erl_module_instance { struct erl_module_nif* nif; int num_breakpoints; int num_traced_exports; +#ifdef HIPE + struct hipe_ref* first_hipe_ref; /* all external hipe calls from this module */ + struct hipe_sdesc* first_hipe_sdesc; /* all stack descriptors for this module */ + void* hipe_code_start; +# ifdef DEBUG + UWord hipe_code_size; +# endif +#endif }; typedef struct erl_module { @@ -40,10 +48,17 @@ typedef struct erl_module { struct erl_module_instance curr; struct erl_module_instance old; /* protected by "old_code" rwlock */ struct erl_module_instance* on_load; +#ifdef HIPE + struct hipe_mfa_info* first_hipe_mfa; + struct hipe_ref* new_hipe_refs; + struct hipe_sdesc* new_hipe_sdesc; +#endif } Module; +void erts_module_instance_init(struct erl_module_instance* modi); Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix); Module* erts_put_module(Eterm mod); +Module* erts_put_active_module(Eterm mod); /* only while blocked */ void init_module_table(void); void module_start_staging(void); -- cgit v1.2.3 From c5a826b3d3a80726473fd9c2a7bea58b4363a993 Mon Sep 17 00:00:00 2001 From: Victor Ren Date: Thu, 6 Oct 2016 18:27:07 +0800 Subject: Use atom value as hash value in process dictionary In the origin implementation, the hash value of atom term is retrieved from the atom table. Reading the atom table is expensive since it is in memory and leads to more cache missing. The size of a process dictionary is usually small. The atom value (the index) is unique and can be hash value for it. Using the atom value directly should be more efficient. --- erts/emulator/beam/beam_load.c | 2 +- erts/emulator/beam/erl_process_dict.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index d69b18e22f..8aecf01bfd 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -4345,7 +4345,7 @@ hash_internal_genop_arg(LoaderState* stp, GenOpArg Key, Uint32* hx) { switch (Key.type) { case TAG_a: - *hx = atom_tab(atom_val(Key.val))->slot.bucket.hvalue; + *hx = atom_val(Key.val); return 1; case TAG_i: *hx = Key.val; diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c index d8c2eaba94..e05d81158d 100644 --- a/erts/emulator/beam/erl_process_dict.c +++ b/erts/emulator/beam/erl_process_dict.c @@ -56,7 +56,7 @@ #define MAKE_HASH(Term) \ ((is_small(Term)) ? unsigned_val(Term) : \ ((is_atom(Term)) ? \ - (atom_tab(atom_val(Term))->slot.bucket.hvalue) : \ + atom_val(Term) : \ make_internal_hash(Term))) #define PD_SZ2BYTES(Sz) (sizeof(ProcDict) + ((Sz) - 1)*sizeof(Eterm)) -- cgit v1.2.3 From b1070efa22f18e19bc7c55fd69e0796f48abf256 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 10 Oct 2016 15:05:26 +0200 Subject: erts: Refactor process dict hash pre-calculation with new function erts_pd_make_hx() --- erts/emulator/beam/beam_load.c | 20 ++++++++++++-------- erts/emulator/beam/erl_process_dict.c | 5 +++++ erts/emulator/beam/erl_process_dict.h | 1 + 3 files changed, 18 insertions(+), 8 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 8aecf01bfd..165328f992 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -39,6 +39,7 @@ #include "erl_binary.h" #include "erl_zlib.h" #include "erl_map.h" +#include "erl_process_dict.h" #ifdef HIPE #include "hipe_bif0.h" @@ -4343,22 +4344,25 @@ gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src, static int hash_internal_genop_arg(LoaderState* stp, GenOpArg Key, Uint32* hx) { + Eterm key_term; switch (Key.type) { case TAG_a: - *hx = atom_val(Key.val); - return 1; + key_term = Key.val; + break; case TAG_i: - *hx = Key.val; - return 1; + key_term = make_small(Key.val); + break; case TAG_n: - *hx = make_internal_hash(NIL); - return 1; + key_term = NIL; + break; case TAG_q: - *hx = make_internal_hash(stp->literals[Key.val].term); - return 1; + key_term = stp->literals[Key.val].term; + break; default: return 0; } + *hx = erts_pd_make_hx(key_term); + return 1; } diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c index e05d81158d..d443fff22c 100644 --- a/erts/emulator/beam/erl_process_dict.c +++ b/erts/emulator/beam/erl_process_dict.c @@ -408,6 +408,11 @@ static void pd_hash_erase_all(Process *p) } } +Uint32 erts_pd_make_hx(Eterm key) +{ + return MAKE_HASH(key); +} + Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id) { unsigned int hval; diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h index 387562058c..db5e1c7442 100644 --- a/erts/emulator/beam/erl_process_dict.h +++ b/erts/emulator/beam/erl_process_dict.h @@ -43,6 +43,7 @@ void erts_deep_dictionary_dump(int to, void *to_arg, Eterm erts_dictionary_copy(struct process *p, ProcDict *pd); Eterm erts_pd_hash_get(struct process *p, Eterm id); +Uint32 erts_pd_make_hx(Eterm key); Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id); #endif -- cgit v1.2.3 From 855b3a9be724ffd3c9f7e311cf9d810099fa36ef Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Mon, 27 Jun 2016 20:18:22 +0200 Subject: erts: Refactor out func_info into struct This commit adds two new structs to be used to represent erlang code in erts. ErtsCodeInfo is used to describe the i_func_info header that is part of all Export entries and the prelude of each function. This replaces all the BeamInstr * that were previously used to point to these locations. After this change the code should never use BeamInstr * with offsets to figure out different parts of the func_info header. ErtsCodeMFA is a struct that is used to descripe a MFA in code. It is used within ErtsCodeInfo and also in Process->current. All function that previously took Eterm * or BeamInstr * to identify a MFA now use the ErtsCodeMFA or ErtsCodeInfo where appropriate. The code has been tested to work when adding a new field to the ErtsCodeInfo struct, but some updates are needed in ops.tab to make it work. --- erts/emulator/beam/beam_bif_load.c | 46 ++--- erts/emulator/beam/beam_bp.c | 330 +++++++++++++++++----------------- erts/emulator/beam/beam_bp.h | 46 ++--- erts/emulator/beam/beam_debug.c | 87 ++++----- erts/emulator/beam/beam_emu.c | 309 ++++++++++++++++--------------- erts/emulator/beam/beam_load.c | 153 ++++++++-------- erts/emulator/beam/beam_load.h | 10 +- erts/emulator/beam/beam_ranges.c | 18 +- erts/emulator/beam/bif.c | 12 +- erts/emulator/beam/bif.h | 16 +- erts/emulator/beam/break.c | 18 +- erts/emulator/beam/code_ix.h | 76 +++++++- erts/emulator/beam/erl_bif_info.c | 39 ++-- erts/emulator/beam/erl_bif_op.c | 2 +- erts/emulator/beam/erl_bif_trace.c | 226 +++++++++++------------ erts/emulator/beam/erl_db_util.c | 8 +- erts/emulator/beam/erl_fun.c | 4 +- erts/emulator/beam/erl_nif.c | 80 +++++---- erts/emulator/beam/erl_printf_term.c | 6 +- erts/emulator/beam/erl_process.c | 21 +-- erts/emulator/beam/erl_process.h | 15 +- erts/emulator/beam/erl_process_dump.c | 7 +- erts/emulator/beam/erl_trace.c | 62 ++++--- erts/emulator/beam/erl_trace.h | 11 +- erts/emulator/beam/erl_vm.h | 7 + erts/emulator/beam/error.h | 4 +- erts/emulator/beam/export.c | 39 ++-- erts/emulator/beam/export.h | 18 +- erts/emulator/beam/external.c | 17 +- erts/emulator/beam/global.h | 14 +- erts/emulator/beam/utils.c | 26 +-- 31 files changed, 927 insertions(+), 800 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 153fa205ed..f584b486ed 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -792,19 +792,19 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) */ for (i = 0; i < export_list_size(code_ix); i++) { Export *ep = export_list(i,code_ix); - if (ep == NULL || ep->code[0] != BIF_ARG_1) { + if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { continue; } - if (ep->code[4] != 0) { - ep->addressv[code_ix] = (void *) ep->code[4]; - ep->code[4] = 0; + if (ep->code[1] != 0) { + ep->addressv[code_ix] = (void *) ep->code[1]; + ep->code[1] = 0; } else { - if (ep->addressv[code_ix] == ep->code+3 && - ep->code[3] == (BeamInstr) em_apply_bif) { + if (ep->addressv[code_ix] == ep->code && + ep->code[0] == (BeamInstr) em_apply_bif) { continue; } - ep->addressv[code_ix] = ep->code+3; - ep->code[3] = (BeamInstr) em_call_error_handler; + ep->addressv[code_ix] = ep->code; + ep->code[0] = (BeamInstr) em_call_error_handler; } } modp->curr.code_hdr->on_load_function_ptr = NULL; @@ -819,13 +819,13 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) for (i = 0; i < export_list_size(code_ix); i++) { Export *ep = export_list(i,code_ix); - if (ep == NULL || ep->code[0] != BIF_ARG_1) { + if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { continue; } - if (ep->code[3] == (BeamInstr) em_apply_bif) { + if (ep->code[0] == (BeamInstr) em_apply_bif) { continue; } - ep->code[4] = 0; + ep->code[1] = 0; } } erts_smp_thr_progress_unblock(); @@ -849,9 +849,9 @@ set_default_trace_pattern(Eterm module) &trace_pattern_flags, &meta_tracer); if (trace_pattern_is_on) { - Eterm mfa[1]; - mfa[0] = module; - (void) erts_set_trace_pattern(0, mfa, 1, + ErtsCodeMFA mfa; + mfa.module = module; + (void) erts_set_trace_pattern(0, &mfa, 1, match_spec, meta_match_spec, 1, trace_pattern_flags, @@ -1710,23 +1710,23 @@ delete_code(Module* modp) for (i = 0; i < export_list_size(code_ix); i++) { Export *ep = export_list(i, code_ix); - if (ep != NULL && (ep->code[0] == module)) { - if (ep->addressv[code_ix] == ep->code+3) { - if (ep->code[3] == (BeamInstr) em_apply_bif) { + if (ep != NULL && (ep->info.mfa.module == module)) { + if (ep->addressv[code_ix] == ep->code) { + if (ep->code[0] == (BeamInstr) em_apply_bif) { continue; } - else if (ep->code[3] == + else if (ep->code[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(modp->curr.num_traced_exports > 0); - erts_clear_export_break(modp, ep->code+3); + erts_clear_export_break(modp, &ep->info); } - else ASSERT(ep->code[3] == (BeamInstr) em_call_error_handler + else ASSERT(ep->code[0] == (BeamInstr) em_call_error_handler || !erts_initialized); } - ep->addressv[code_ix] = ep->code+3; - ep->code[3] = (BeamInstr) em_call_error_handler; - ep->code[4] = 0; + ep->addressv[code_ix] = ep->code; + ep->code[0] = (BeamInstr) em_call_error_handler; + ep->code[1] = 0; } } diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 920c8b1ed0..2cdf66f441 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -92,27 +92,27 @@ get_mtime(Process *c_p) /* ** Helpers */ -static ErtsTracer do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, +static ErtsTracer do_call_trace(Process* c_p, ErtsCodeInfo *info, Eterm* reg, int local, Binary* ms, ErtsTracer tracer); static void set_break(BpFunctions* f, Binary *match_spec, Uint break_flags, enum erts_break_op count_op, ErtsTracer tracer); -static void set_function_break(BeamInstr *pc, +static void set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags, enum erts_break_op count_op, ErtsTracer tracer); static void clear_break(BpFunctions* f, Uint break_flags); -static int clear_function_break(BeamInstr *pc, Uint break_flags); +static int clear_function_break(ErtsCodeInfo *ci, Uint break_flags); -static BpDataTime* get_time_break(BeamInstr *pc); -static GenericBpData* check_break(BeamInstr *pc, Uint break_flags); +static BpDataTime* get_time_break(ErtsCodeInfo *ci); +static GenericBpData* check_break(ErtsCodeInfo *ci, Uint break_flags); -static void bp_meta_unref(BpMetaTracer* bmt); -static void bp_count_unref(BpCount* bcp); -static void bp_time_unref(BpDataTime* bdt); -static void consolidate_bp_data(Module* modp, BeamInstr* pc, int local); -static void uninstall_breakpoint(BeamInstr* pc); +static void bp_meta_unref(BpMetaTracer *bmt); +static void bp_count_unref(BpCount *bcp); +static void bp_time_unref(BpDataTime *bdt); +static void consolidate_bp_data(Module *modp, ErtsCodeInfo *ci, int local); +static void uninstall_breakpoint(ErtsCodeInfo *ci); /* bp_hash */ #define BP_TIME_ADD(pi0, pi1) \ @@ -139,7 +139,7 @@ erts_bp_init(void) { void -erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified) +erts_bp_match_functions(BpFunctions* f, ErtsCodeMFA *mfa, int specified) { ErtsCodeIndex code_ix = erts_active_code_ix(); Uint max_funcs = 0; @@ -164,41 +164,44 @@ erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified) i = 0; for (current = 0; current < num_modules; current++) { BeamCodeHeader* code_hdr = module[current]->curr.code_hdr; - BeamInstr* code; + ErtsCodeInfo* ci; Uint num_functions = (Uint)(UWord) code_hdr->num_functions; Uint fi; if (specified > 0) { - if (mfa[0] != make_atom(module[current]->module)) { + if (mfa->module != make_atom(module[current]->module)) { /* Wrong module name */ continue; } } for (fi = 0; fi < num_functions; fi++) { - BeamInstr* pc; - int wi; - code = code_hdr->functions[fi]; - ASSERT(code[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - pc = code+5; - if (erts_is_native_break(pc)) { + ci = code_hdr->functions[fi]; + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + if (erts_is_native_break(ci)) { continue; } - if (is_nil(code[3])) { /* Ignore BIF stub */ + if (is_nil(ci->mfa.module)) { /* Ignore BIF stub */ continue; } - for (wi = 0; - wi < specified && (Eterm) code[2+wi] == mfa[wi]; - wi++) { - /* Empty loop body */ - } - if (wi == specified) { - /* Store match */ - f->matching[i].pc = pc; - f->matching[i].mod = module[current]; - i++; - } + switch (specified) { + case 3: + if (ci->mfa.arity != mfa->arity) + continue; + case 2: + if (ci->mfa.function != mfa->function) + continue; + case 1: + if (ci->mfa.module != mfa->module) + continue; + case 0: + break; + } + /* Store match */ + f->matching[i].ci = ci; + f->matching[i].mod = module[current]; + i++; } } f->matched = i; @@ -206,7 +209,7 @@ erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified) } void -erts_bp_match_export(BpFunctions* f, Eterm mfa[3], int specified) +erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified) { ErtsCodeIndex code_ix = erts_active_code_ix(); int i; @@ -218,27 +221,36 @@ erts_bp_match_export(BpFunctions* f, Eterm mfa[3], int specified) for (i = 0; i < num_exps; i++) { Export* ep = export_list(i, code_ix); BeamInstr* pc; - int j; - for (j = 0; j < specified && mfa[j] == ep->code[j]; j++) { - /* Empty loop body */ - } - if (j < specified) { - continue; - } - pc = ep->code+3; + switch (specified) { + case 3: + if (mfa->arity != ep->info.mfa.arity) + continue; + case 2: + if (mfa->function != ep->info.mfa.function) + continue; + case 1: + if (mfa->module != ep->info.mfa.module) + continue; + case 0: + break; + default: + ASSERT(0); + } + + pc = ep->code; if (ep->addressv[code_ix] == pc) { if ((*pc == (BeamInstr) em_apply_bif || *pc == (BeamInstr) em_call_error_handler)) { continue; } ASSERT(*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)); - } else if (erts_is_native_break(ep->addressv[code_ix])) { + } else if (erts_is_native_break(erts_code_to_codeinfo(ep->addressv[code_ix]))) { continue; } - f->matching[ne].pc = pc; - f->matching[ne].mod = erts_get_module(ep->code[0], code_ix); + f->matching[ne].ci = &ep->info; + f->matching[ne].mod = erts_get_module(ep->info.mfa.module, code_ix); ne++; } @@ -264,7 +276,7 @@ erts_consolidate_bp_data(BpFunctions* f, int local) ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); for (i = 0; i < n; i++) { - consolidate_bp_data(fs[i].mod, fs[i].pc, local); + consolidate_bp_data(fs[i].mod, fs[i].ci, local); } } @@ -276,14 +288,14 @@ erts_consolidate_bif_bp_data(void) ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); for (i = 0; i < BIF_SIZE; i++) { Export *ep = bif_export[i]; - consolidate_bp_data(0, ep->code+3, 0); + consolidate_bp_data(0, &ep->info, 0); } } static void -consolidate_bp_data(Module* modp, BeamInstr* pc, int local) +consolidate_bp_data(Module* modp, ErtsCodeInfo *ci, int local) { - GenericBp* g = (GenericBp *) pc[-4]; + GenericBp* g = (GenericBp *) ci->native; GenericBpData* src; GenericBpData* dst; Uint flags; @@ -329,9 +341,10 @@ consolidate_bp_data(Module* modp, BeamInstr* pc, int local) } ASSERT(modp->curr.num_breakpoints >= 0); ASSERT(modp->curr.num_traced_exports >= 0); - ASSERT(*pc != (BeamInstr) BeamOp(op_i_generic_breakpoint)); + ASSERT(*erts_codeinfo_to_code(ci) != + (BeamInstr) BeamOp(op_i_generic_breakpoint)); } - pc[-4] = 0; + ci->native = 0; Free(g); return; } @@ -380,8 +393,9 @@ erts_install_breakpoints(BpFunctions* f) BeamInstr br = (BeamInstr) BeamOp(op_i_generic_breakpoint); for (i = 0; i < n; i++) { - BeamInstr* pc = f->matching[i].pc; - GenericBp* g = (GenericBp *) pc[-4]; + ErtsCodeInfo* ci = f->matching[i].ci; + BeamInstr *pc = erts_codeinfo_to_code(ci); + GenericBp* g = (GenericBp *) ci->native; if (*pc != br && g) { Module* modp = f->matching[i].mod; @@ -413,16 +427,16 @@ erts_uninstall_breakpoints(BpFunctions* f) Uint n = f->matched; for (i = 0; i < n; i++) { - BeamInstr* pc = f->matching[i].pc; - uninstall_breakpoint(pc); + uninstall_breakpoint(f->matching[i].ci); } } static void -uninstall_breakpoint(BeamInstr* pc) +uninstall_breakpoint(ErtsCodeInfo *ci) { + BeamInstr *pc = erts_codeinfo_to_code(ci); if (*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { - GenericBp* g = (GenericBp *) pc[-4]; + GenericBp* g = (GenericBp *) ci->native; if (g->data[erts_active_bp_ix()].flags == 0) { /* * The following write is not protected by any lock. We @@ -449,30 +463,30 @@ erts_set_mtrace_break(BpFunctions* f, Binary *match_spec, ErtsTracer tracer) } void -erts_set_call_trace_bif(BeamInstr *pc, Binary *match_spec, int local) +erts_set_call_trace_bif(ErtsCodeInfo *ci, Binary *match_spec, int local) { Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE; - set_function_break(pc, match_spec, flags, 0, erts_tracer_nil); + set_function_break(ci, match_spec, flags, 0, erts_tracer_nil); } void -erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, ErtsTracer tracer) +erts_set_mtrace_bif(ErtsCodeInfo *ci, Binary *match_spec, ErtsTracer tracer) { - set_function_break(pc, match_spec, ERTS_BPF_META_TRACE, 0, tracer); + set_function_break(ci, match_spec, ERTS_BPF_META_TRACE, 0, tracer); } void -erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op count_op) +erts_set_time_trace_bif(ErtsCodeInfo *ci, enum erts_break_op count_op) { - set_function_break(pc, NULL, + set_function_break(ci, NULL, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE, count_op, erts_tracer_nil); } void -erts_clear_time_trace_bif(BeamInstr *pc) { - clear_function_break(pc, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE); +erts_clear_time_trace_bif(ErtsCodeInfo *ci) { + clear_function_break(ci, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE); } void @@ -501,14 +515,14 @@ erts_clear_trace_break(BpFunctions* f) } void -erts_clear_call_trace_bif(BeamInstr *pc, int local) +erts_clear_call_trace_bif(ErtsCodeInfo *ci, int local) { - GenericBp* g = (GenericBp *) pc[-4]; + GenericBp* g = (GenericBp *) ci->native; if (g) { Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE; if (g->data[erts_staging_bp_ix()].flags & flags) { - clear_function_break(pc, flags); + clear_function_break(ci, flags); } } } @@ -520,9 +534,9 @@ erts_clear_mtrace_break(BpFunctions* f) } void -erts_clear_mtrace_bif(BeamInstr *pc) +erts_clear_mtrace_bif(ErtsCodeInfo *ci) { - clear_function_break(pc, ERTS_BPF_META_TRACE); + clear_function_break(ci, ERTS_BPF_META_TRACE); } void @@ -564,52 +578,48 @@ erts_clear_module_break(Module *modp) { } n = (Uint)(UWord) code_hdr->num_functions; for (i = 0; i < n; ++i) { - BeamInstr* pc; - - pc = code_hdr->functions[i] + 5; - if (erts_is_native_break(pc)) { + ErtsCodeInfo *ci = code_hdr->functions[i]; + if (erts_is_native_break(ci)) continue; - } - clear_function_break(pc, ERTS_BPF_ALL); + clear_function_break(ci, ERTS_BPF_ALL); } erts_commit_staged_bp(); for (i = 0; i < n; ++i) { - BeamInstr* pc; - - pc = code_hdr->functions[i] + 5; - if (erts_is_native_break(pc)) { + ErtsCodeInfo *ci = code_hdr->functions[i]; + if (erts_is_native_break(ci)) continue; - } - uninstall_breakpoint(pc); - consolidate_bp_data(modp, pc, 1); - ASSERT(pc[-4] == 0); + uninstall_breakpoint(ci); + consolidate_bp_data(modp, ci, 1); + ASSERT(ci->native == 0); } return n; } void -erts_clear_export_break(Module* modp, BeamInstr* pc) +erts_clear_export_break(Module* modp, ErtsCodeInfo *ci) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - clear_function_break(pc, ERTS_BPF_ALL); + clear_function_break(ci, ERTS_BPF_ALL); erts_commit_staged_bp(); - *pc = (BeamInstr) 0; - consolidate_bp_data(modp, pc, 0); - ASSERT(pc[-4] == 0); + *erts_codeinfo_to_code(ci) = (BeamInstr) 0; + consolidate_bp_data(modp, ci, 0); + ASSERT(ci->native == 0); } BeamInstr -erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) +erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg) { GenericBp* g; GenericBpData* bp; Uint bp_flags; ErtsBpIndex ix = erts_active_bp_ix(); - g = (GenericBp *) I[-4]; + ASSERT(info->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + + g = (GenericBp *) info->native; bp = &g->data[ix]; bp_flags = bp->flags; ASSERT((bp_flags & ~ERTS_BPF_ALL) == 0); @@ -628,9 +638,9 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) if (bp_flags & ERTS_BPF_LOCAL_TRACE) { ASSERT((bp_flags & ERTS_BPF_GLOBAL_TRACE) == 0); - (void) do_call_trace(c_p, I, reg, 1, bp->local_ms, erts_tracer_true); + (void) do_call_trace(c_p, info, reg, 1, bp->local_ms, erts_tracer_true); } else if (bp_flags & ERTS_BPF_GLOBAL_TRACE) { - (void) do_call_trace(c_p, I, reg, 0, bp->local_ms, erts_tracer_true); + (void) do_call_trace(c_p, info, reg, 0, bp->local_ms, erts_tracer_true); } if (bp_flags & ERTS_BPF_META_TRACE) { @@ -638,7 +648,8 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) old_tracer = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer); - new_tracer = do_call_trace(c_p, I, reg, 1, bp->meta_ms, old_tracer); + new_tracer = do_call_trace(c_p, info, reg, 1, bp->meta_ms, old_tracer); + if (!ERTS_TRACER_COMPARE(new_tracer, old_tracer)) { if (old_tracer == erts_smp_atomic_cmpxchg_acqb( &bp->meta_tracer->tracer, @@ -657,7 +668,7 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE) { Eterm w; - erts_trace_time_call(c_p, I, bp->time); + erts_trace_time_call(c_p, info, bp->time); w = (BeamInstr) *c_p->cp; if (! (w == (BeamInstr) BeamOp(op_i_return_time_trace) || w == (BeamInstr) BeamOp(op_return_trace) || @@ -665,7 +676,7 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) Eterm* E = c_p->stop; ASSERT(c_p->htop <= E && E <= c_p->hend); if (E - 2 < c_p->htop) { - (void) erts_garbage_collect(c_p, 2, reg, I[-1]); + (void) erts_garbage_collect(c_p, 2, reg, info->mfa.arity); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); } E = c_p->stop; @@ -673,7 +684,7 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) ASSERT(c_p->htop <= E && E <= c_p->hend); E -= 2; - E[0] = make_cp(I); + E[0] = make_cp(erts_codeinfo_to_code(info)); E[1] = make_cp(c_p->cp); /* original return address */ c_p->cp = beam_return_time_trace; c_p->stop = E; @@ -701,9 +712,9 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) Export* ep = bif_export[bif_index]; Uint32 flags = 0, flags_meta = 0; ErtsTracer meta_tracer = erts_tracer_nil; - int applying = (I == &(ep->code[3])); /* Yup, the apply code for a bif - * is actually in the - * export entry */ + int applying = (I == ep->code); /* Yup, the apply code for a bif + * is actually in the + * export entry */ BeamInstr *cp = p->cp; GenericBp* g; GenericBpData* bp = NULL; @@ -711,7 +722,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); - g = (GenericBp *) ep->fake_op_func_info_for_hipe[1]; + g = (GenericBp *) ep->info.native; if (g) { bp = &g->data[erts_active_bp_ix()]; bp_flags = bp->flags; @@ -727,7 +738,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) if (bp_flags & (ERTS_BPF_LOCAL_TRACE|ERTS_BPF_GLOBAL_TRACE) && IS_TRACED_FL(p, F_TRACE_CALLS)) { int local = !!(bp_flags & ERTS_BPF_LOCAL_TRACE); - flags = erts_call_trace(p, ep->code, bp->local_ms, args, + flags = erts_call_trace(p, &ep->info, bp->local_ms, args, local, &ERTS_TRACER(p)); } if (bp_flags & ERTS_BPF_META_TRACE) { @@ -735,7 +746,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) meta_tracer = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer); old_tracer = meta_tracer; - flags_meta = erts_call_trace(p, ep->code, bp->meta_ms, args, + flags_meta = erts_call_trace(p, &ep->info, bp->meta_ms, args, 0, &meta_tracer); if (!ERTS_TRACER_COMPARE(old_tracer, meta_tracer)) { @@ -753,8 +764,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) } if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE && IS_TRACED_FL(p, F_TRACE_CALLS)) { - BeamInstr *pc = (BeamInstr *)ep->code+3; - erts_trace_time_call(p, pc, bp->time); + erts_trace_time_call(p, &ep->info, bp->time); } /* Restore original continuation pointer (if changed). */ @@ -817,11 +827,11 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) class = exception_tag[GET_EXC_CLASS(reason)]; if (flags_meta & MATCH_SET_EXCEPTION_TRACE) { - erts_trace_exception(p, ep->code, class, value, + erts_trace_exception(p, &ep->info.mfa, class, value, &meta_tracer); } if (flags & MATCH_SET_EXCEPTION_TRACE) { - erts_trace_exception(p, ep->code, class, value, + erts_trace_exception(p, &ep->info.mfa, class, value, &ERTS_TRACER(p)); } if ((flags & MATCH_SET_RETURN_TO_TRACE) && p->catches > 0) { @@ -852,11 +862,11 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) } } else { if (flags_meta & MATCH_SET_RX_TRACE) { - erts_trace_return(p, ep->code, result, &meta_tracer); + erts_trace_return(p, &ep->info.mfa, result, &meta_tracer); } /* MATCH_SET_RETURN_TO_TRACE cannot occur if(meta) */ if (flags & MATCH_SET_RX_TRACE) { - erts_trace_return(p, ep->code, result, &ERTS_TRACER(p)); + erts_trace_return(p, &ep->info.mfa, result, &ERTS_TRACER(p)); } if (flags & MATCH_SET_RETURN_TO_TRACE && IS_TRACED_FL(p, F_TRACE_RETURN_TO)) { @@ -875,7 +885,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) } static ErtsTracer -do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, +do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg, int local, Binary* ms, ErtsTracer tracer) { Eterm* cpp; @@ -916,7 +926,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, ASSERT(is_CP(*cpp)); } ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - flags = erts_call_trace(c_p, I-3, ms, reg, local, &tracer); + flags = erts_call_trace(c_p, info, ms, reg, local, &tracer); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); if (cpp) { c_p->cp = cp_save; @@ -932,7 +942,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, if (need) { ASSERT(c_p->htop <= E && E <= c_p->hend); if (E - need < c_p->htop) { - (void) erts_garbage_collect(c_p, need, reg, I[-1]); + (void) erts_garbage_collect(c_p, need, reg, info->mfa.arity); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); E = c_p->stop; } @@ -948,12 +958,12 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, E -= 3; c_p->stop = E; ASSERT(c_p->htop <= E && E <= c_p->hend); - ASSERT(is_CP((Eterm) (UWord) (I - 3))); + ASSERT(is_CP((Eterm) (UWord) (&info->mfa.module))); ASSERT(IS_TRACER_VALID(tracer)); E[2] = make_cp(c_p->cp); E[1] = copy_object(tracer, c_p); - E[0] = make_cp(I - 3); /* We ARE at the beginning of an - instruction, + E[0] = make_cp(&info->mfa.module); + /* We ARE at the beginning of an instruction, the funcinfo is above i. */ c_p->cp = (flags & MATCH_SET_EXCEPTION_TRACE) ? beam_exception_trace : beam_return_trace; @@ -966,7 +976,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, } void -erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) +erts_trace_time_call(Process* c_p, ErtsCodeInfo *info, BpDataTime* bdt) { ErtsMonotonicTime time; process_breakpoint_time_t *pbt = NULL; @@ -995,14 +1005,14 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) pbt = Alloc(sizeof(process_breakpoint_time_t)); (void) ERTS_PROC_SET_CALL_TIME(c_p, pbt); } else { - ASSERT(pbt->pc); + ASSERT(pbt->ci); /* add time to previous code */ sitem.time = time - pbt->time; sitem.pid = c_p->common.id; sitem.count = 0; /* previous breakpoint */ - pbdt = get_time_break(pbt->pc); + pbdt = get_time_break(pbt->ci); /* if null then the breakpoint was removed */ if (pbdt) { @@ -1039,12 +1049,12 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) BP_TIME_ADD(item, &sitem); } - pbt->pc = I; + pbt->ci = info; pbt->time = time; } void -erts_trace_time_return(Process *p, BeamInstr *pc) +erts_trace_time_return(Process *p, ErtsCodeInfo *ci) { ErtsMonotonicTime time; process_breakpoint_time_t *pbt = NULL; @@ -1074,14 +1084,14 @@ erts_trace_time_return(Process *p, BeamInstr *pc) /* might have been removed due to * trace_pattern(false) */ - ASSERT(pbt->pc); + ASSERT(pbt->ci); sitem.time = time - pbt->time; sitem.pid = p->common.id; sitem.count = 0; /* previous breakpoint */ - pbdt = get_time_break(pbt->pc); + pbdt = get_time_break(pbt->ci); /* beware, the trace_pattern might have been removed */ if (pbdt) { @@ -1098,16 +1108,16 @@ erts_trace_time_return(Process *p, BeamInstr *pc) } } - pbt->pc = pc; + pbt->ci = ci; pbt->time = time; } } int -erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local) +erts_is_trace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, int local) { Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE; - GenericBpData* bp = check_break(pc, flags); + GenericBpData* bp = check_break(ci, flags); if (bp) { if (match_spec_ret) { @@ -1119,10 +1129,10 @@ erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local) } int -erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, +erts_is_mtrace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, ErtsTracer *tracer_ret) { - GenericBpData* bp = check_break(pc, ERTS_BPF_META_TRACE); + GenericBpData* bp = check_break(ci, ERTS_BPF_META_TRACE); if (bp) { if (match_spec_ret) { @@ -1137,20 +1147,20 @@ erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, } int -erts_is_native_break(BeamInstr *pc) { +erts_is_native_break(ErtsCodeInfo *ci) { #ifdef HIPE - ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - return pc[0] == (BeamInstr) BeamOp(op_hipe_trap_call) - || pc[0] == (BeamInstr) BeamOp(op_hipe_trap_call_closure); + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + return erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call) + || erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call_closure); #else return 0; #endif } int -erts_is_count_break(BeamInstr *pc, Uint *count_ret) +erts_is_count_break(ErtsCodeInfo *ci, Uint *count_ret) { - GenericBpData* bp = check_break(pc, ERTS_BPF_COUNT); + GenericBpData* bp = check_break(ci, ERTS_BPF_COUNT); if (bp) { if (count_ret) { @@ -1161,13 +1171,13 @@ erts_is_count_break(BeamInstr *pc, Uint *count_ret) return 0; } -int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) { +int erts_is_time_break(Process *p, ErtsCodeInfo *ci, Eterm *retval) { Uint i, ix; bp_time_hash_t hash; Uint size; Eterm *hp, t; bp_data_time_item_t *item = NULL; - BpDataTime *bdt = get_time_break(pc); + BpDataTime *bdt = get_time_break(ci); if (bdt) { if (retval) { @@ -1221,26 +1231,25 @@ int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) { } -BeamInstr * -erts_find_local_func(Eterm mfa[3]) { +ErtsCodeInfo * +erts_find_local_func(ErtsCodeMFA *mfa) { Module *modp; BeamCodeHeader* code_hdr; - BeamInstr* code_ptr; + ErtsCodeInfo* ci; Uint i,n; - if ((modp = erts_get_module(mfa[0], erts_active_code_ix())) == NULL) + if ((modp = erts_get_module(mfa->module, erts_active_code_ix())) == NULL) return NULL; if ((code_hdr = modp->curr.code_hdr) == NULL) return NULL; n = (BeamInstr) code_hdr->num_functions; for (i = 0; i < n; ++i) { - code_ptr = code_hdr->functions[i]; - ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == code_ptr[0]); - ASSERT(mfa[0] == ((Eterm) code_ptr[2]) || - is_nil((Eterm) code_ptr[2])); - if (mfa[1] == ((Eterm) code_ptr[3]) && - ((BeamInstr) mfa[2]) == code_ptr[4]) { - return code_ptr + 5; + ci = code_hdr->functions[i]; + ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == ci->op); + ASSERT(mfa->module == ci->mfa.module || is_nil(ci->mfa.module)); + if (mfa->function == ci->mfa.function && + mfa->arity == ci->mfa.arity) { + return ci; } } return NULL; @@ -1369,7 +1378,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) { * the previous breakpoint. */ - pbdt = get_time_break(pbt->pc); + pbdt = get_time_break(pbt->ci); if (pbdt) { sitem.time = get_mtime(p) - pbt->time; sitem.pid = p->common.id; @@ -1417,14 +1426,14 @@ set_break(BpFunctions* f, Binary *match_spec, Uint break_flags, n = f->matched; for (i = 0; i < n; i++) { - BeamInstr* pc = f->matching[i].pc; - set_function_break(pc, match_spec, break_flags, + set_function_break(f->matching[i].ci, + match_spec, break_flags, count_op, tracer); } } static void -set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, +set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags, enum erts_break_op count_op, ErtsTracer tracer) { GenericBp* g; @@ -1433,7 +1442,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, ErtsBpIndex ix = erts_staging_bp_ix(); ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); - g = (GenericBp *) pc[-4]; + g = (GenericBp *) ci->native; if (g == 0) { int i; if (count_op == ERTS_BREAK_RESTART || count_op == ERTS_BREAK_PAUSE) { @@ -1441,11 +1450,11 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, return; } g = Alloc(sizeof(GenericBp)); - g->orig_instr = *pc; + g->orig_instr = *erts_codeinfo_to_code(ci); for (i = 0; i < ERTS_NUM_BP_IX; i++) { g->data[i].flags = 0; } - pc[-4] = (BeamInstr) g; + ci->native = (BeamInstr) g; } bp = &g->data[ix]; @@ -1537,13 +1546,12 @@ clear_break(BpFunctions* f, Uint break_flags) n = f->matched; for (i = 0; i < n; i++) { - BeamInstr* pc = f->matching[i].pc; - clear_function_break(pc, break_flags); + clear_function_break(f->matching[i].ci, break_flags); } } static int -clear_function_break(BeamInstr *pc, Uint break_flags) +clear_function_break(ErtsCodeInfo *ci, Uint break_flags) { GenericBp* g; GenericBpData* bp; @@ -1552,7 +1560,7 @@ clear_function_break(BeamInstr *pc, Uint break_flags) ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); - if ((g = (GenericBp *) pc[-4]) == 0) { + if ((g = (GenericBp *) ci->native) == 0) { return 1; } @@ -1638,19 +1646,19 @@ bp_time_unref(BpDataTime* bdt) } static BpDataTime* -get_time_break(BeamInstr *pc) +get_time_break(ErtsCodeInfo *ci) { - GenericBpData* bp = check_break(pc, ERTS_BPF_TIME_TRACE); + GenericBpData* bp = check_break(ci, ERTS_BPF_TIME_TRACE); return bp ? bp->time : 0; } static GenericBpData* -check_break(BeamInstr *pc, Uint break_flags) +check_break(ErtsCodeInfo *ci, Uint break_flags) { - GenericBp* g = (GenericBp *) pc[-4]; + GenericBp* g = (GenericBp *) ci->native; - ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - if (erts_is_native_break(pc)) { + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + if (erts_is_native_break(ci)) { return 0; } if (g) { diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 541af77211..d32a6bcba5 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -46,7 +46,7 @@ typedef struct bp_data_time { /* Call time */ typedef struct { ErtsMonotonicTime time; - BeamInstr *pc; + ErtsCodeInfo *ci; } process_breakpoint_time_t; /* used within psd */ typedef struct { @@ -95,7 +95,7 @@ enum erts_break_op{ typedef Uint32 ErtsBpIndex; typedef struct { - BeamInstr* pc; + ErtsCodeInfo *ci; Module* mod; } BpFunction; @@ -116,8 +116,8 @@ void erts_commit_staged_bp(void); ERTS_GLB_INLINE ErtsBpIndex erts_active_bp_ix(void); ERTS_GLB_INLINE ErtsBpIndex erts_staging_bp_ix(void); -void erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified); -void erts_bp_match_export(BpFunctions* f, Eterm mfa[3], int specified); +void erts_bp_match_functions(BpFunctions* f, ErtsCodeMFA *mfa, int specified); +void erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified); void erts_bp_free_matched_functions(BpFunctions* f); void erts_install_breakpoints(BpFunctions* f); @@ -128,15 +128,15 @@ void erts_consolidate_bif_bp_data(void); void erts_set_trace_break(BpFunctions *f, Binary *match_spec); void erts_clear_trace_break(BpFunctions *f); -void erts_set_call_trace_bif(BeamInstr *pc, Binary *match_spec, int local); -void erts_clear_call_trace_bif(BeamInstr *pc, int local); +void erts_set_call_trace_bif(ErtsCodeInfo *ci, Binary *match_spec, int local); +void erts_clear_call_trace_bif(ErtsCodeInfo *ci, int local); void erts_set_mtrace_break(BpFunctions *f, Binary *match_spec, ErtsTracer tracer); void erts_clear_mtrace_break(BpFunctions *f); -void erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, +void erts_set_mtrace_bif(ErtsCodeInfo *ci, Binary *match_spec, ErtsTracer tracer); -void erts_clear_mtrace_bif(BeamInstr *pc); +void erts_clear_mtrace_bif(ErtsCodeInfo *ci); void erts_set_debug_break(BpFunctions *f); void erts_clear_debug_break(BpFunctions *f); @@ -146,32 +146,32 @@ void erts_clear_count_break(BpFunctions *f); void erts_clear_all_breaks(BpFunctions* f); int erts_clear_module_break(Module *modp); -void erts_clear_export_break(Module *modp, BeamInstr* pc); +void erts_clear_export_break(Module *modp, ErtsCodeInfo* ci); -BeamInstr erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg); -BeamInstr erts_trace_break(Process *p, BeamInstr *pc, Eterm *args, +BeamInstr erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *ci, Eterm* reg); +BeamInstr erts_trace_break(Process *p, ErtsCodeInfo *ci, Eterm *args, Uint32 *ret_flags, ErtsTracer *tracer); -int erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local); -int erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, +int erts_is_trace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, int local); +int erts_is_mtrace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, ErtsTracer *tracer_ret); -int erts_is_mtrace_bif(BeamInstr *pc, Binary **match_spec_ret, +int erts_is_mtrace_bif(ErtsCodeInfo *ci, Binary **match_spec_ret, ErtsTracer *tracer_ret); -int erts_is_native_break(BeamInstr *pc); -int erts_is_count_break(BeamInstr *pc, Uint *count_ret); -int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *call_time); +int erts_is_native_break(ErtsCodeInfo *ci); +int erts_is_count_break(ErtsCodeInfo *ci, Uint *count_ret); +int erts_is_time_break(Process *p, ErtsCodeInfo *ci, Eterm *call_time); -void erts_trace_time_call(Process* c_p, BeamInstr* pc, BpDataTime* bdt); -void erts_trace_time_return(Process* c_p, BeamInstr* pc); +void erts_trace_time_call(Process* c_p, ErtsCodeInfo *ci, BpDataTime* bdt); +void erts_trace_time_return(Process* c_p, ErtsCodeInfo *ci); void erts_schedule_time_break(Process *p, Uint out); void erts_set_time_break(BpFunctions *f, enum erts_break_op); void erts_clear_time_break(BpFunctions *f); -int erts_is_time_trace_bif(Process *p, BeamInstr *pc, Eterm *call_time); -void erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op); -void erts_clear_time_trace_bif(BeamInstr *pc); +int erts_is_time_trace_bif(Process *p, ErtsCodeInfo *ci, Eterm *call_time); +void erts_set_time_trace_bif(ErtsCodeInfo *ci, enum erts_break_op); +void erts_clear_time_trace_bif(ErtsCodeInfo *ci); -BeamInstr *erts_find_local_func(Eterm mfa[3]); +ErtsCodeInfo *erts_find_local_func(ErtsCodeMFA *mfa); #if ERTS_GLB_INLINE_INCL_FUNC_DEF diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 8f47ed4d9d..8d4be4667b 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -116,7 +116,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) Eterm MFA = BIF_ARG_1; Eterm boolean = BIF_ARG_2; Eterm* tp; - Eterm mfa[3]; + ErtsCodeMFA mfa; int i; int specified = 0; Eterm res; @@ -132,23 +132,24 @@ erts_debug_breakpoint_2(BIF_ALIST_2) if (*tp != make_arityval(3)) { goto error; } - mfa[0] = tp[1]; - mfa[1] = tp[2]; - mfa[2] = tp[3]; - if (!is_atom(mfa[0]) || !is_atom(mfa[1]) || - (!is_small(mfa[2]) && mfa[2] != am_Underscore)) { + if (!is_atom(tp[1]) || !is_atom(tp[2]) || + (!is_small(tp[3]) && tp[3] != am_Underscore)) { goto error; } - for (i = 0; i < 3 && mfa[i] != am_Underscore; i++, specified++) { + for (i = 0; i < 3 && tp[i+1] != am_Underscore; i++, specified++) { /* Empty loop body */ } for (i = specified; i < 3; i++) { - if (mfa[i] != am_Underscore) { + if (tp[i+1] != am_Underscore) { goto error; } } - if (is_small(mfa[2])) { - mfa[2] = signed_val(mfa[2]); + + mfa.module = tp[1]; + mfa.function = tp[2]; + + if (is_small(tp[3])) { + mfa.arity = signed_val(tp[3]); } if (!erts_try_seize_code_write_permission(BIF_P)) { @@ -158,7 +159,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); - erts_bp_match_functions(&f, mfa, specified); + erts_bp_match_functions(&f, &mfa, specified); if (boolean == am_true) { erts_set_debug_break(&f); erts_install_breakpoints(&f); @@ -242,17 +243,17 @@ erts_debug_disassemble_1(BIF_ALIST_1) Eterm* tp; Eterm bin; Eterm mfa; - BeamInstr* funcinfo = NULL; /* Initialized to eliminate warning. */ + ErtsCodeInfo *ci = NULL; BeamCodeHeader* code_hdr; - BeamInstr* code_ptr = NULL; /* Initialized to eliminate warning. */ + BeamInstr *code_ptr; BeamInstr instr; BeamInstr uaddr; Uint hsz; int i; if (term_to_UWord(addr, &uaddr)) { - code_ptr = (BeamInstr *) uaddr; - if ((funcinfo = find_function_from_pc(code_ptr)) == NULL) { + BeamInstr *pc = (BeamInstr *) uaddr; + if ((ci = find_function_from_pc(pc)) == NULL) { BIF_RET(am_false); } } else if (is_tuple(addr)) { @@ -283,24 +284,22 @@ erts_debug_disassemble_1(BIF_ALIST_1) * such as erts_debug:apply/4. Then search for it in the module. */ if ((ep = erts_find_function(mod, name, arity, code_ix)) != NULL) { - /* XXX: add "&& ep->address != ep->code+3" condition? + /* XXX: add "&& ep->address != ep->code" condition? * Consider a traced function. - * Its ep will have ep->address == ep->code+3. + * Its ep will have ep->address == ep->code. * erts_find_function() will return the non-NULL ep. * Below we'll try to derive a code_ptr from ep->address. * But this code_ptr will point to the start of the Export, * not the function's func_info instruction. BOOM !? */ - code_ptr = ((BeamInstr *) ep->addressv[code_ix]) - 5; - funcinfo = code_ptr+2; + ci = erts_code_to_codeinfo(ep->addressv[code_ix]); } else if (modp == NULL || (code_hdr = modp->curr.code_hdr) == NULL) { BIF_RET(am_undef); } else { n = code_hdr->num_functions; for (i = 0; i < n; i++) { - code_ptr = code_hdr->functions[i]; - if (code_ptr[3] == name && code_ptr[4] == arity) { - funcinfo = code_ptr+2; + ci = code_hdr->functions[i]; + if (ci->mfa.function == name && ci->mfa.arity == arity) { break; } } @@ -312,6 +311,8 @@ erts_debug_disassemble_1(BIF_ALIST_1) goto error; } + + code_ptr = erts_codeinfo_to_code(ci); dsbufp = erts_create_tmp_dsbuf(0); erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr); instr = (BeamInstr) code_ptr[0]; @@ -333,9 +334,10 @@ erts_debug_disassemble_1(BIF_ALIST_1) (void) erts_bld_uword(NULL, &hsz, (BeamInstr) code_ptr); hp = HAlloc(p, hsz); addr = erts_bld_uword(&hp, NULL, (BeamInstr) code_ptr); - ASSERT(is_atom(funcinfo[0]) || funcinfo[0] == NIL); - ASSERT(is_atom(funcinfo[1]) || funcinfo[1] == NIL); - mfa = TUPLE3(hp, (Eterm) funcinfo[0], (Eterm) funcinfo[1], make_small((Eterm) funcinfo[2])); + ASSERT(is_atom(ci->mfa.module) || is_nil(ci->mfa.module)); + ASSERT(is_atom(ci->mfa.function) || is_nil(ci->mfa.function == NIL)); + mfa = TUPLE3(hp, ci->mfa.module, ci->mfa.function, + make_small(ci->mfa.arity)); hp += 4; return TUPLE3(hp, addr, bin, mfa); } @@ -347,11 +349,12 @@ dbg_bt(Process* p, Eterm* sp) while (sp < stack) { if (is_CP(*sp)) { - BeamInstr* addr = find_function_from_pc(cp_val(*sp)); - if (addr) + ErtsCodeInfo* ci = find_function_from_pc(cp_val(*sp)); + if (ci) erts_fprintf(stderr, HEXF ": %T:%T/%bpu\n", - addr, (Eterm) addr[0], (Eterm) addr[1], addr[2]); + &ci->mfa.module, ci->mfa.module, + ci->mfa.function, ci->mfa.arity); } sp++; } @@ -360,17 +363,17 @@ dbg_bt(Process* p, Eterm* sp) void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg) { - BeamInstr* f = find_function_from_pc(addr); + ErtsCodeInfo* ci = find_function_from_pc(addr); - if (f == NULL) { + if (ci == NULL) { erts_fprintf(stderr, "???\n"); } else { int arity; int i; - addr = f; - arity = addr[2]; - erts_fprintf(stderr, HEXF ": %T:%T(", addr, (Eterm) addr[0], (Eterm) addr[1]); + arity = ci->mfa.arity; + erts_fprintf(stderr, HEXF ": %T:%T(", addr, + ci->mfa.module, ci->mfa.function); for (i = 0; i < arity; i++) erts_fprintf(stderr, i ? ", %T" : "%T", i ? reg[i] : x0); erts_fprintf(stderr, ")\n"); @@ -546,22 +549,24 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) break; case 'f': /* Destination label */ { - BeamInstr* f = find_function_from_pc((BeamInstr *)*ap); - if (f+3 != (BeamInstr *) *ap) { + ErtsCodeInfo* ci = find_function_from_pc((BeamInstr *)*ap); + if (erts_codeinfo_to_code(ci) != (BeamInstr *) *ap) { erts_print(to, to_arg, "f(" HEXF ")", *ap); } else { - erts_print(to, to_arg, "%T:%T/%bpu", (Eterm) f[0], (Eterm) f[1], f[2]); + erts_print(to, to_arg, "%T:%T/%bpu", ci->mfa.module, + ci->mfa.function, ci->mfa.arity); } ap++; } break; case 'p': /* Pointer (to label) */ { - BeamInstr* f = find_function_from_pc((BeamInstr *)*ap); - if (f+3 != (BeamInstr *) *ap) { + ErtsCodeInfo* ci = find_function_from_pc((BeamInstr *)*ap); + if (erts_codeinfo_to_code(ci) != (BeamInstr *) *ap) { erts_print(to, to_arg, "p(" HEXF ")", *ap); } else { - erts_print(to, to_arg, "%T:%T/%bpu", (Eterm) f[0], (Eterm) f[1], f[2]); + erts_print(to, to_arg, "%T:%T/%bpu", ci->mfa.module, + ci->mfa.function, ci->mfa.arity); } ap++; } @@ -574,7 +579,9 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) { Export* ex = (Export *) *ap; erts_print(to, to_arg, - "%T:%T/%bpu", (Eterm) ex->code[0], (Eterm) ex->code[1], ex->code[2]); + "%T:%T/%bpu", (Eterm) ex->info.mfa.module, + (Eterm) ex->info.mfa.function, + ex->info.mfa.arity); ap++; } break; diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 0ba06058a5..4ff177f215 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -116,10 +116,10 @@ do { \ #define MAX(x, y) (((x) > (y)) ? (x) : (y)) #endif -#define GET_BIF_MODULE(p) ((Eterm) (((Export *) p)->code[0])) -#define GET_BIF_FUNCTION(p) ((Eterm) (((Export *) p)->code[1])) -#define GET_BIF_ARITY(p) ((Eterm) (((Export *) p)->code[2])) -#define GET_BIF_ADDRESS(p) ((BifFunction) (((Export *) p)->code[4])) +#define GET_BIF_MODULE(p) (p->info.mfa.module) +#define GET_BIF_FUNCTION(p) (p->info.mfa.function) +#define GET_BIF_ARITY(p) (p->info.mfa.arity) +#define GET_BIF_ADDRESS(p) ((BifFunction) (p->code[1])) #define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm)))) @@ -1045,7 +1045,7 @@ void** beam_ops; static BifFunction translate_gc_bif(void* gcf) NOINLINE; static BeamInstr* handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) NOINLINE; -static BeamInstr* call_error_handler(Process* p, BeamInstr* ip, +static BeamInstr* call_error_handler(Process* p, ErtsCodeMFA* mfa, Eterm* reg, Eterm func) NOINLINE; static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity) NOINLINE; static BeamInstr* apply(Process* p, Eterm module, Eterm function, @@ -1106,98 +1106,91 @@ init_emulator(void) #ifdef USE_VM_CALL_PROBES -#define DTRACE_LOCAL_CALL(p, m, f, a) \ +#define DTRACE_LOCAL_CALL(p, mfa) \ if (DTRACE_ENABLED(local_function_entry)) { \ DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ int depth = STACK_START(p) - STACK_TOP(p); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE3(local_function_entry, process_name, mfa, depth); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE3(local_function_entry, process_name, mfa_buf, depth); \ } -#define DTRACE_GLOBAL_CALL(p, m, f, a) \ +#define DTRACE_GLOBAL_CALL(p, mfa) \ if (DTRACE_ENABLED(global_function_entry)) { \ DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ int depth = STACK_START(p) - STACK_TOP(p); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE3(global_function_entry, process_name, mfa, depth); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE3(global_function_entry, process_name, mfa_buf, depth); \ } -#define DTRACE_RETURN(p, m, f, a) \ +#define DTRACE_RETURN(p, mfa) \ if (DTRACE_ENABLED(function_return)) { \ DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ int depth = STACK_START(p) - STACK_TOP(p); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE3(function_return, process_name, mfa, depth); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE3(function_return, process_name, mfa_buf, depth); \ } -#define DTRACE_BIF_ENTRY(p, m, f, a) \ - if (DTRACE_ENABLED(bif_entry)) { \ - DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE2(bif_entry, process_name, mfa); \ +#define DTRACE_BIF_ENTRY(p, mfa) \ + if (DTRACE_ENABLED(bif_entry)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE2(bif_entry, process_name, mfa_buf); \ } -#define DTRACE_BIF_RETURN(p, m, f, a) \ - if (DTRACE_ENABLED(bif_return)) { \ - DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE2(bif_return, process_name, mfa); \ +#define DTRACE_BIF_RETURN(p, mfa) \ + if (DTRACE_ENABLED(bif_return)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE2(bif_return, process_name, mfa_buf); \ } -#define DTRACE_NIF_ENTRY(p, m, f, a) \ - if (DTRACE_ENABLED(nif_entry)) { \ - DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE2(nif_entry, process_name, mfa); \ +#define DTRACE_NIF_ENTRY(p, mfa) \ + if (DTRACE_ENABLED(nif_entry)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE2(nif_entry, process_name, mfa_buf); \ } -#define DTRACE_NIF_RETURN(p, m, f, a) \ - if (DTRACE_ENABLED(nif_return)) { \ - DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE2(nif_return, process_name, mfa); \ +#define DTRACE_NIF_RETURN(p, mfa) \ + if (DTRACE_ENABLED(nif_return)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_mfa, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE2(nif_return, process_name, mfa_buf); \ } #define DTRACE_GLOBAL_CALL_FROM_EXPORT(p,e) \ do { \ if (DTRACE_ENABLED(global_function_entry)) { \ BeamInstr* fp = (BeamInstr *) (((Export *) (e))->addressv[erts_active_code_ix()]); \ - DTRACE_GLOBAL_CALL((p), (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); \ + DTRACE_GLOBAL_CALL((p), erts_code_to_codemfa(fp)); \ } \ } while(0) #define DTRACE_RETURN_FROM_PC(p) \ do { \ - BeamInstr* fp; \ - if (DTRACE_ENABLED(function_return) && (fp = find_function_from_pc((p)->cp))) { \ - DTRACE_RETURN((p), (Eterm)fp[0], (Eterm)fp[1], (Uint)fp[2]); \ + ErtsCodeInfo* ci; \ + if (DTRACE_ENABLED(function_return) && (ci = find_function_from_pc((p)->cp))) { \ + DTRACE_RETURN((p), &ci->mfa); \ } \ } while(0) #else /* USE_VM_PROBES */ -#define DTRACE_LOCAL_CALL(p, m, f, a) do {} while (0) -#define DTRACE_GLOBAL_CALL(p, m, f, a) do {} while (0) +#define DTRACE_LOCAL_CALL(p, mfa) do {} while (0) +#define DTRACE_GLOBAL_CALL(p, mfa) do {} while (0) #define DTRACE_GLOBAL_CALL_FROM_EXPORT(p, e) do {} while (0) -#define DTRACE_RETURN(p, m, f, a) do {} while (0) +#define DTRACE_RETURN(p, mfa) do {} while (0) #define DTRACE_RETURN_FROM_PC(p) do {} while (0) -#define DTRACE_BIF_ENTRY(p, m, f, a) do {} while (0) -#define DTRACE_BIF_RETURN(p, m, f, a) do {} while (0) -#define DTRACE_NIF_ENTRY(p, m, f, a) do {} while (0) -#define DTRACE_NIF_RETURN(p, m, f, a) do {} while (0) +#define DTRACE_BIF_ENTRY(p, mfa) do {} while (0) +#define DTRACE_BIF_RETURN(p, mfa) do {} while (0) +#define DTRACE_NIF_ENTRY(p, mfa) do {} while (0) +#define DTRACE_NIF_RETURN(p, mfa) do {} while (0) #endif /* USE_VM_PROBES */ #ifdef DEBUG @@ -1324,8 +1317,8 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) if (start_time != 0) { Sint64 diff = erts_timestamp_millis() - start_time; if (diff > 0 && (Uint) diff > erts_system_monitor_long_schedule) { - BeamInstr *inptr = find_function_from_pc(start_time_i); - BeamInstr *outptr = find_function_from_pc(c_p->i); + ErtsCodeInfo *inptr = find_function_from_pc(start_time_i); + ErtsCodeInfo *outptr = find_function_from_pc(c_p->i); monitor_long_schedule_proc(c_p,inptr,outptr,(Uint) diff); } } @@ -1583,7 +1576,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) /* FALL THROUGH */ OpCase(i_call_only_f): { SET_I((BeamInstr *) Arg(0)); - DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); Dispatch(); } @@ -1595,7 +1588,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) RESTORE_CP(E); E = ADD_BYTE_OFFSET(E, Arg(1)); SET_I((BeamInstr *) Arg(0)); - DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); Dispatch(); } @@ -1607,7 +1600,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) OpCase(i_call_f): { SET_CP(c_p, I+2); SET_I((BeamInstr *) Arg(0)); - DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); Dispatch(); } @@ -2834,25 +2827,27 @@ do { \ Eterm result; BeamInstr *next; ErlHeapFragment *live_hf_end; + Export *export = (Export*)Arg(0); if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) { /* If we have run out of reductions, we do a context switch before calling the bif */ - c_p->arity = ((Export *)Arg(0))->code[2]; - c_p->current = ((Export *)Arg(0))->code; + c_p->arity = GET_BIF_ARITY(export); + c_p->current = &export->info.mfa; goto context_switch3; } - ERTS_MSACC_SET_BIF_STATE_CACHED_X(GET_BIF_MODULE(Arg(0)), GET_BIF_ADDRESS(Arg(0))); + ERTS_MSACC_SET_BIF_STATE_CACHED_X( + GET_BIF_MODULE(export), GET_BIF_ADDRESS(export)); - bf = GET_BIF_ADDRESS(Arg(0)); + bf = GET_BIF_ADDRESS(export); PRE_BIF_SWAPOUT(c_p); ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS - 1; if (FCALLS <= 0) { - save_calls(c_p, (Export *) Arg(0)); + save_calls(c_p, export); } PreFetch(1, next); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -2866,7 +2861,7 @@ do { \ ERTS_HOLE_CHECK(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); if (ERTS_IS_GC_DESIRED(c_p)) { - Uint arity = ((Export *)Arg(0))->code[2]; + Uint arity = GET_BIF_ARITY(export); result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, result, reg, arity); E = c_p->stop; } @@ -3359,14 +3354,15 @@ do { \ * called from I[-3], I[-2], and I[-1] respectively. */ context_switch_fun: - c_p->arity = I[-1] + 1; + /* Add one for the environment of the fun */ + c_p->arity = erts_code_to_codemfa(I)->arity + 1; goto context_switch2; context_switch: - c_p->arity = I[-1]; + c_p->arity = erts_code_to_codemfa(I)->arity; - context_switch2: /* Entry for fun calls. */ - c_p->current = I-3; /* Pointer to Mod, Func, Arity */ + context_switch2: /* Entry for fun calls. */ + c_p->current = erts_code_to_codemfa(I); context_switch3: @@ -3507,7 +3503,8 @@ do { \ * code[4]: Not used */ HEAVY_SWAPOUT; - I = call_error_handler(c_p, I-3, reg, am_undefined_function); + I = call_error_handler(c_p, erts_code_to_codemfa(I), + reg, am_undefined_function); HEAVY_SWAPIN; if (I) { Goto(*I); @@ -3544,9 +3541,12 @@ do { \ * I[1]: Function pointer to NIF function * I[2]: Pointer to erl_module_nif * I[3]: Function pointer to dirty NIF + * + * This layout is determined by the NifExport struct */ BifFunction vbf; ErlHeapFragment *live_hf_end; + ErtsCodeMFA *codemfa; if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) { /* If we have run out of reductions, we do a context @@ -3556,12 +3556,16 @@ do { \ ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); - DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); - c_p->current = I-3; /* current and vbf set to please handle_error */ + codemfa = erts_code_to_codemfa(I); + + c_p->current = codemfa; /* current and vbf set to please handle_error */ + + DTRACE_NIF_ENTRY(c_p, codemfa); + SWAPOUT; c_p->fcalls = FCALLS - 1; PROCESS_MAIN_CHK_LOCKS(c_p); - bif_nif_arity = I[-1]; + bif_nif_arity = codemfa->arity; ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -3588,19 +3592,19 @@ do { \ ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } - DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + DTRACE_NIF_RETURN(c_p, codemfa); goto apply_bif_or_nif_epilogue; OpCase(apply_bif): /* - * At this point, I points to the code[3] in the export entry for + * At this point, I points to the code[0] in the export entry for * the BIF: * - * code[0]: Module - * code[1]: Function - * code[2]: Arity - * code[3]: &&apply_bif - * code[4]: Function pointer to BIF function + * code[-3]: Module + * code[-2]: Function + * code[-1]: Arity + * code[0]: &&apply_bif + * code[1]: Function pointer to BIF function */ if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) { @@ -3609,21 +3613,25 @@ do { \ goto context_switch; } - ERTS_MSACC_SET_BIF_STATE_CACHED_X((Eterm)I[-3], (BifFunction)Arg(0)); + codemfa = erts_code_to_codemfa(I); + + ERTS_MSACC_SET_BIF_STATE_CACHED_X(codemfa->module, (BifFunction)Arg(0)); + - c_p->current = I-3; /* In case we apply process_info/1,2 or load_nif/1 */ + /* In case we apply process_info/1,2 or load_nif/1 */ + c_p->current = codemfa; c_p->i = I; /* In case we apply check_process_code/2. */ c_p->arity = 0; /* To allow garbage collection on ourselves * (check_process_code/2). */ - DTRACE_BIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + DTRACE_BIF_ENTRY(c_p, codemfa); SWAPOUT; ERTS_DBG_CHK_REDS(c_p, FCALLS - 1); c_p->fcalls = FCALLS - 1; vbf = (BifFunction) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); - bif_nif_arity = I[-1]; + bif_nif_arity = codemfa->arity; ASSERT(bif_nif_arity <= 4); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -3645,7 +3653,7 @@ do { \ if (ERTS_MSACC_IS_ENABLED_CACHED_X()) ERTS_MSACC_UPDATE_CACHE_X(); ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); - DTRACE_BIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + DTRACE_BIF_RETURN(c_p, codemfa); apply_bif_or_nif_epilogue: ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); @@ -3712,8 +3720,9 @@ do { \ goto find_func_info; OpCase(i_func_info_IaaI): { + ErtsCodeInfo *ci = (ErtsCodeInfo*)I; c_p->freason = EXC_FUNCTION_CLAUSE; - c_p->current = I + 2; + c_p->current = &ci->mfa; goto handle_error; } @@ -4704,11 +4713,11 @@ do { \ */ OpCase(return_trace): { - BeamInstr* code = (BeamInstr *) (UWord) E[0]; + ErtsCodeMFA* mfa = (ErtsCodeMFA *)(E[0]); SWAPOUT; /* Needed for shared heap */ ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - erts_trace_return(c_p, code, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */); + erts_trace_return(c_p, mfa, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); SWAPIN; c_p->cp = NULL; @@ -4719,9 +4728,8 @@ do { \ OpCase(i_generic_breakpoint): { BeamInstr real_I; - ASSERT(I[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); HEAVY_SWAPOUT; - real_I = erts_generic_breakpoint(c_p, I, reg); + real_I = erts_generic_breakpoint(c_p, erts_code_to_codeinfo(I), reg); HEAVY_SWAPIN; ASSERT(VALID_INSTR(real_I)); Goto(real_I); @@ -4730,7 +4738,7 @@ do { \ OpCase(i_return_time_trace): { BeamInstr *pc = (BeamInstr *) (UWord) E[0]; SWAPOUT; - erts_trace_time_return(c_p, pc); + erts_trace_time_return(c_p, erts_code_to_codeinfo(pc)); SWAPIN; c_p->cp = NULL; SET_I((BeamInstr *) cp_val(E[1])); @@ -4915,16 +4923,18 @@ do { \ * I[ 0]: &&lb_hipe_trap_call * ... remainder of original BEAM code */ - ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.u.ncallee = (void(*)(void)) I[-4]; + ErtsCodeInfo *ci = erts_code_to_codeinfo(I); + ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); + c_p->hipe.u.ncallee = (void(*)(void)) ci->native; ++hipe_trap_count; - HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (I[-1] << 8)); + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (ci->mfa.arity << 8)); } OpCase(hipe_trap_call_closure): { - ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.u.ncallee = (void(*)(void)) I[-4]; + ErtsCodeInfo *ci = erts_code_to_codeinfo(I); + ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); + c_p->hipe.u.ncallee = (void(*)(void)) ci->native; ++hipe_trap_count; - HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (I[-1] << 8)); + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (ci->mfa.arity << 8)); } OpCase(hipe_trap_return): { HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RETURN); @@ -4993,8 +5003,9 @@ do { \ * I[ 0]: &&lb_hipe_call_count * ... remainder of original BEAM code */ - struct hipe_call_count *hcc = (struct hipe_call_count*)I[-4]; - ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); + ErtsCodeInfo *ci = erts_code_to_codeinfo(I); + struct hipe_call_count *hcc = (struct hipe_call_count*)ci->native; + ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); ASSERT(hcc != NULL); ASSERT(VALID_INSTR(hcc->opcode)); ++(hcc->count); @@ -5063,7 +5074,7 @@ do { \ OpCase(i_debug_breakpoint): { HEAVY_SWAPOUT; - I = call_error_handler(c_p, I-3, reg, am_breakpoint); + I = call_error_handler(c_p, erts_code_to_codemfa(I), reg, am_breakpoint); HEAVY_SWAPIN; if (I) { Goto(*I); @@ -5136,10 +5147,10 @@ do { \ bif_table[i].name, bif_table[i].arity); bif_export[i] = ep; - ep->code[3] = (BeamInstr) OpCode(apply_bif); - ep->code[4] = (BeamInstr) bif_table[i].f; + ep->code[0] = (BeamInstr) OpCode(apply_bif); + ep->code[1] = (BeamInstr) bif_table[i].f; /* XXX: set func info for bifs */ - ep->fake_op_func_info_for_hipe[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI); + ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI); } return; @@ -5219,8 +5230,8 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) goto do_dirty_schedule; context_switch: - c_p->arity = I[-1]; - c_p->current = I-3; /* Pointer to Mod, Func, Arity */ + c_p->current = erts_code_to_codemfa(I); /* Pointer to Mod, Func, Arity */ + c_p->arity = c_p->current->arity; { int reds_used; @@ -5378,16 +5389,22 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) * I[1]: Function pointer to NIF function * I[2]: Pointer to erl_module_nif * I[3]: Function pointer to dirty NIF + * + * This layout is determined by the NifExport struct */ BifFunction vbf; + ErtsCodeMFA *codemfa; ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); - DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); - c_p->current = I-3; /* current and vbf set to please handle_error */ + codemfa = erts_code_to_codemfa(I); + + DTRACE_NIF_ENTRY(c_p, codemfa); + /* current and vbf set to please handle_error */ + c_p->current = codemfa; SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); - arity = I[-1]; + arity = codemfa->arity; ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -5422,7 +5439,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } - DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + DTRACE_NIF_RETURN(c_p, codemfa); ERTS_HOLE_CHECK(c_p); SWAPIN; I = c_p->i; @@ -5602,7 +5619,8 @@ next_catch(Process* c_p, Eterm *reg) { /* Can not follow cp here - code may be unloaded */ BeamInstr *cpp = c_p->cp; if (cpp == beam_exception_trace) { - erts_trace_exception(c_p, cp_val(ptr[0]), + ErtsCodeMFA *mfa = (ErtsCodeMFA*)cp_val(ptr[0]); + erts_trace_exception(c_p, mfa, reg[1], reg[2], ERTS_TRACER_FROM_ETERM(ptr+1)); /* Skip return_trace parameters */ @@ -5630,7 +5648,8 @@ next_catch(Process* c_p, Eterm *reg) { if (is_catch(*ptr) && active_catches) goto found_catch; } if (cp_val(*prev) == beam_exception_trace) { - erts_trace_exception(c_p, cp_val(ptr[0]), + ErtsCodeMFA *mfa = (ErtsCodeMFA*)cp_val(ptr[0]); + erts_trace_exception(c_p, mfa, reg[1], reg[2], ERTS_TRACER_FROM_ETERM(ptr+1)); } @@ -5836,7 +5855,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, for (i = 0; i < BIF_SIZE; i++) { if (bf == bif_table[i].f || bf == bif_table[i].traced) { Export *ep = bif_export[i]; - s->current = ep->code; + s->current = &ep->info.mfa; a = bif_table[i].arity; break; } @@ -5850,7 +5869,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, */ ASSERT(c_p->current); s->current = c_p->current; - a = s->current[2]; + a = s->current->arity; } /* Save first stack entry */ ASSERT(pc); @@ -5875,7 +5894,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, (GET_EXC_INDEX(EXC_FUNCTION_CLAUSE)) ) { int a; ASSERT(s->current); - a = s->current[2]; + a = s->current->arity; args = make_arglist(c_p, reg, a); /* Overwrite CAR(c_p->ftrace) */ /* Save first stack entry */ ASSERT(c_p->cp); @@ -6046,7 +6065,7 @@ build_stacktrace(Process* c_p, Eterm exc) { erts_lookup_function_info(&fi, s->pc, 1); } else if (GET_EXC_INDEX(s->freason) == GET_EXC_INDEX(EXC_FUNCTION_CLAUSE)) { - erts_lookup_function_info(&fi, s->current, 1); + erts_lookup_function_info(&fi, erts_codemfa_to_code(s->current), 1); } else { erts_set_current_function(&fi, s->current); } @@ -6055,8 +6074,8 @@ build_stacktrace(Process* c_p, Eterm exc) { * If fi.current is still NULL, default to the initial function * (e.g. spawn_link(erlang, abs, [1])). */ - if (fi.current == NULL) { - erts_set_current_function(&fi, c_p->u.initial); + if (fi.ci == NULL) { + erts_set_current_function(&fi, &c_p->u.initial); args = am_true; /* Just in case */ } else { args = get_args_from_exc(exc); @@ -6072,7 +6091,7 @@ build_stacktrace(Process* c_p, Eterm exc) { heap_size = fi.needed + 2; for (i = 0; i < depth; i++) { erts_lookup_function_info(stkp, s->trace[i], 1); - if (stkp->current) { + if (stkp->ci) { heap_size += stkp->needed + 2; stkp++; } @@ -6096,7 +6115,7 @@ build_stacktrace(Process* c_p, Eterm exc) { } static BeamInstr* -call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) +call_error_handler(Process* p, ErtsCodeMFA* mfa, Eterm* reg, Eterm func) { Eterm* hp; Export* ep; @@ -6111,7 +6130,7 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) ep = erts_find_function(erts_proc_get_error_handler(p), func, 3, erts_active_code_ix()); if (ep == NULL) { /* No error handler */ - p->current = fi; + p->current = mfa; p->freason = EXC_UNDEF; return 0; } @@ -6120,7 +6139,7 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) * Create a list with all arguments in the x registers. */ - arity = fi[2]; + arity = mfa->arity; sz = 2 * arity; if (HeapWordsLeft(p) < sz) { erts_garbage_collect(p, sz, reg, arity); @@ -6136,8 +6155,8 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) /* * Set up registers for call to error_handler:/3. */ - reg[0] = fi[0]; - reg[1] = fi[1]; + reg[0] = mfa->module; + reg[1] = mfa->function; reg[2] = args; return ep->addressv[erts_active_code_ix()]; } @@ -6433,7 +6452,7 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); - c_p->current = bif_export[BIF_hibernate_3]->code; + c_p->current = &bif_export[BIF_hibernate_3]->info.mfa; c_p->flags |= F_HIBERNATE_SCHED; /* Needed also when woken! */ return 1; } @@ -6456,21 +6475,15 @@ call_fun(Process* p, /* Current process. */ if (is_fun_header(hdr)) { ErlFunThing* funp = (ErlFunThing *) fun_val(fun); - ErlFunEntry* fe; - BeamInstr* code_ptr; + ErlFunEntry* fe = funp->fe; + BeamInstr* code_ptr = fe->address; Eterm* var_ptr; - int actual_arity; - unsigned num_free; - - fe = funp->fe; - num_free = funp->num_free; - code_ptr = fe->address; - actual_arity = (int) code_ptr[-1]; + unsigned num_free = funp->num_free; + ErtsCodeMFA *mfa = erts_code_to_codemfa(code_ptr); + int actual_arity = mfa->arity; if (actual_arity == arity+num_free) { - DTRACE_LOCAL_CALL(p, (Eterm)code_ptr[-3], - (Eterm)code_ptr[-2], - code_ptr[-1]); + DTRACE_LOCAL_CALL(p, mfa); if (num_free == 0) { return code_ptr; } else { @@ -6575,10 +6588,10 @@ call_fun(Process* p, /* Current process. */ int actual_arity; ep = *((Export **) (export_val(fun) + 1)); - actual_arity = (int) ep->code[2]; + actual_arity = ep->info.mfa.arity; if (arity == actual_arity) { - DTRACE_GLOBAL_CALL(p, ep->code[0], ep->code[1], (Uint)ep->code[2]); + DTRACE_GLOBAL_CALL(p, &ep->info.mfa); return ep->addressv[erts_active_code_ix()]; } else { /* @@ -7199,15 +7212,15 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity) return 1; } - e.code[0] = Mod; - e.code[1] = Name; - e.code[2] = arity; + e.info.mfa.module = Mod; + e.info.mfa.function = Name; + e.info.mfa.arity = arity; if ((ep = export_get(&e)) == NULL) { return 0; } - return ep->addressv[erts_active_code_ix()] == ep->code+3 - && (ep->code[3] == (BeamInstr) em_apply_bif); + return ep->addressv[erts_active_code_ix()] == ep->code + && (ep->code[0] == (BeamInstr) em_apply_bif); } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index d69b18e22f..c822cd8606 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -806,21 +806,21 @@ erts_finish_loading(Binary* magic, Process* c_p, for (i = 0; i < export_list_size(code_ix); i++) { Export *ep = export_list(i, code_ix); - if (ep == NULL || ep->code[0] != module) { + if (ep == NULL || ep->info.mfa.module != module) { continue; } - if (ep->addressv[code_ix] == ep->code+3) { - if (ep->code[3] == (BeamInstr) em_apply_bif) { + if (ep->addressv[code_ix] == ep->code) { + if (ep->code[0] == (BeamInstr) em_apply_bif) { continue; - } else if (ep->code[3] == + } else if (ep->code[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(mod_tab_p->curr.num_traced_exports > 0); - erts_clear_export_break(mod_tab_p, ep->code+3); - ep->addressv[code_ix] = (BeamInstr *) ep->code[4]; - ep->code[4] = 0; + erts_clear_export_break(mod_tab_p, &ep->info); + ep->addressv[code_ix] = (BeamInstr *) ep->code[1]; + ep->code[1] = 0; } - ASSERT(ep->code[4] == 0); + ASSERT(ep->code[1] == 0); } } ASSERT(mod_tab_p->curr.num_breakpoints == 0); @@ -1420,8 +1420,8 @@ load_import_table(LoaderState* stp) * the BIF function. */ if ((e = erts_active_export_entry(mod, func, arity)) != NULL) { - if (e->code[3] == (BeamInstr) em_apply_bif) { - stp->import[i].bf = (BifFunction) e->code[4]; + if (e->code[0] == (BeamInstr) em_apply_bif) { + stp->import[i].bf = (BifFunction) e->code[1]; if (func == am_load_nif && mod == am_erlang && arity == 2) { stp->may_load_nif = 1; } @@ -1514,7 +1514,7 @@ is_bif(Eterm mod, Eterm func, unsigned arity) if (e == NULL) { return 0; } - if (e->code[3] != (BeamInstr) em_apply_bif) { + if (e->code[0] != (BeamInstr) em_apply_bif) { return 0; } if (mod == am_erlang && func == am_apply && arity == 3) { @@ -1893,7 +1893,7 @@ load_code(LoaderState* stp) * by both the nif functionality and line instructions. */ enum { - FUNC_INFO_SZ = 5 + FUNC_INFO_SZ = sizeof(ErtsCodeInfo) / sizeof(Eterm) }; code = stp->codev; @@ -2565,8 +2565,11 @@ load_code(LoaderState* stp) stp->function = code[ci-2]; stp->arity = code[ci-1]; + /* When this assert is triggered, it is normally a sign that + the size of the ops.tab i_func_info instruction is not + the same as FUNC_INFO_SZ */ ASSERT(stp->labels[last_label].value == ci - FUNC_INFO_SZ); - stp->hdr->functions[function_number] = (BeamInstr*) stp->labels[last_label].patches; + stp->hdr->functions[function_number] = (ErtsCodeInfo*) stp->labels[last_label].patches; offset = function_number; stp->labels[last_label].patches = offset; function_number++; @@ -4513,7 +4516,7 @@ freeze_code(LoaderState* stp) * function table in the beginning of the file. */ - code_hdr->functions[stp->num_functions] = (codev + stp->ci - 1); + code_hdr->functions[stp->num_functions] = (ErtsCodeInfo*)(codev + stp->ci - 1); CHKBLK(ERTS_ALC_T_CODE,code_hdr); /* @@ -4781,7 +4784,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) * callable yet. Keep any function in the current * code callable. */ - ep->code[4] = (BeamInstr) address; + ep->code[1] = (BeamInstr) address; } } @@ -4959,7 +4962,7 @@ transform_engine(LoaderState* st) if (i >= st->num_imports || st->import[i].bf == NULL) goto restart; if (bif_number != -1 && - bif_export[bif_number]->code[4] != (BeamInstr) st->import[i].bf) { + bif_export[bif_number]->code[1] != (BeamInstr) st->import[i].bf) { goto restart; } } @@ -5584,18 +5587,16 @@ functions_in_module(Process* p, /* Process whose heap to use. */ hp = HAlloc(p, need); hp_end = hp + need; for (i = num_functions-1; i >= 0 ; i--) { - BeamInstr* func_info = code_hdr->functions[i]; - Eterm name = (Eterm) func_info[3]; - int arity = (int) func_info[4]; + ErtsCodeInfo* ci = code_hdr->functions[i]; Eterm tuple; /* * If the function name is [], this entry is a stub for * a BIF that should be ignored. */ - ASSERT(is_atom(name) || is_nil(name)); - if (is_atom(name)) { - tuple = TUPLE2(hp, name, make_small(arity)); + ASSERT(is_atom(ci->mfa.function) || is_nil(ci->mfa.function)); + if (is_atom(ci->mfa.function)) { + tuple = TUPLE2(hp, ci->mfa.function, make_small(ci->mfa.arity)); hp += 3; result = CONS(hp, tuple, result); hp += 2; @@ -5683,17 +5684,17 @@ native_addresses(Process* p, BeamCodeHeader* code_hdr) hp = HAlloc(p, need); hp_end = hp + need; for (i = num_functions-1; i >= 0 ; i--) { - BeamInstr* func_info = code_hdr->functions[i]; - Eterm name = (Eterm) func_info[3]; - int arity = (int) func_info[4]; + ErtsCodeInfo *ci = code_hdr->functions[i]; Eterm tuple; - ASSERT(is_atom(name) || is_nil(name)); /* [] if BIF stub */ - if (func_info[1] != 0) { - Eterm addr; - ASSERT(is_atom(name)); - addr = erts_bld_uint(&hp, NULL, func_info[1]); - tuple = erts_bld_tuple(&hp, NULL, 3, name, make_small(arity), addr); + ASSERT(is_atom(ci->mfa.function) + || is_nil(ci->mfa.function)); /* [] if BIF stub */ + if (ci->native != 0) { + Eterm addr; + ASSERT(is_atom(ci->mfa.function)); + addr = erts_bld_uint(&hp, NULL, ci->native); + tuple = erts_bld_tuple(&hp, NULL, 3, ci->mfa.function, + make_small(ci->mfa.arity), addr); result = erts_bld_cons(&hp, NULL, tuple, result); } } @@ -5719,11 +5720,11 @@ exported_from_module(Process* p, /* Process whose heap to use. */ for (i = 0; i < export_list_size(code_ix); i++) { Export* ep = export_list(i,code_ix); - if (ep->code[0] == mod) { + if (ep->info.mfa.module == mod) { Eterm tuple; - if (ep->addressv[code_ix] == ep->code+3 && - ep->code[3] == (BeamInstr) em_call_error_handler) { + if (ep->addressv[code_ix] == ep->code && + ep->code[0] == (BeamInstr) em_call_error_handler) { /* There is a call to the function, but it does not exist. */ continue; } @@ -5733,7 +5734,8 @@ exported_from_module(Process* p, /* Process whose heap to use. */ hp = HAlloc(p, need); hend = hp + need; } - tuple = TUPLE2(hp, ep->code[1], make_small(ep->code[2])); + tuple = TUPLE2(hp, ep->info.mfa.function, + make_small(ep->info.mfa.arity)); hp += 3; result = CONS(hp, tuple, result); hp += 2; @@ -5807,7 +5809,6 @@ md5_of_module(Process* p, /* Process whose heap to use. */ Eterm* erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) { - BeamInstr* current = fi->current; Eterm loc = NIL; if (fi->loc != LINE_INVALID_LOCATION) { @@ -5817,7 +5818,7 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) Eterm file_term = NIL; if (file == 0) { - Atom* ap = atom_tab(atom_val(fi->current[0])); + Atom* ap = atom_tab(atom_val(fi->ci->mfa.module)); file_term = buf_to_intlist(&hp, ".erl", 4, NIL); file_term = buf_to_intlist(&hp, (char*)ap->name, ap->len, file_term); } else { @@ -5836,10 +5837,12 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) } if (is_list(args) || is_nil(args)) { - *mfa_p = TUPLE4(hp, current[0], current[1], args, loc); + *mfa_p = TUPLE4(hp, fi->ci->mfa.module, fi->ci->mfa.function, + args, loc); } else { - Eterm arity = make_small(current[2]); - *mfa_p = TUPLE4(hp, current[0], current[1], arity, loc); + Eterm arity = make_small(fi->ci->mfa.arity); + *mfa_p = TUPLE4(hp, fi->ci->mfa.module, fi->ci->mfa.function, + arity, loc); } return hp + 5; } @@ -5850,9 +5853,9 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) * the function. */ void -erts_set_current_function(FunctionInfo* fi, BeamInstr* current) +erts_set_current_function(FunctionInfo* fi, ErtsCodeMFA* mfa) { - fi->current = current; + fi->ci = erts_code_to_codeinfo(erts_codemfa_to_code(mfa)); fi->needed = 5; fi->loc = LINE_INVALID_LOCATION; } @@ -5861,13 +5864,13 @@ erts_set_current_function(FunctionInfo* fi, BeamInstr* current) /* * Returns a pointer to {module, function, arity}, or NULL if not found. */ -BeamInstr* +ErtsCodeInfo* find_function_from_pc(BeamInstr* pc) { FunctionInfo fi; erts_lookup_function_info(&fi, pc, 0); - return fi.current; + return fi.ci; } /* @@ -5984,24 +5987,25 @@ code_module_md5_1(BIF_ALIST_1) return res; } -#define WORDS_PER_FUNCTION 6 +#define WORDS_PER_FUNCTION (sizeof(ErtsCodeInfo) / sizeof(UWord) + 1) static BeamInstr* -make_stub(BeamInstr* fp, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode) +make_stub(ErtsCodeInfo* info, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode) { - fp[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI); - fp[1] = native; - fp[2] = mod; - fp[3] = func; - fp[4] = arity; + ASSERT(WORDS_PER_FUNCTION == 6); + info->op = (BeamInstr) BeamOp(op_i_func_info_IaaI); + info->native = native; + info->mfa.module = mod; + info->mfa.function = func; + info->mfa.arity = arity; #ifdef HIPE if (native) { - fp[5] = BeamOpCode(op_move_return_n); - hipe_mfa_save_orig_beam_op(mod, func, arity, fp+5); + erts_codeinfo_to_code(info)[0] = BeamOpCode(op_move_return_n); + hipe_mfa_save_orig_beam_op(mod, func, arity, erts_codeinfo_to_code(info)); } #endif - fp[5] = OpCode; - return fp + WORDS_PER_FUNCTION; + erts_codeinfo_to_code(info)[0] = OpCode; + return erts_codeinfo_to_code(info)+1; } static byte* @@ -6060,22 +6064,19 @@ stub_read_export_table(LoaderState* stp) } static void -stub_final_touch(LoaderState* stp, BeamInstr* fp) +stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci) { unsigned int i; unsigned int n = stp->num_exps; - Eterm mod = fp[2]; - Eterm function = fp[3]; - int arity = fp[4]; #ifdef HIPE Lambda* lp; #endif - if (is_bif(mod, function, arity)) { - fp[1] = 0; - fp[2] = 0; - fp[3] = 0; - fp[4] = 0; + if (is_bif(ci->mfa.module, ci->mfa.function, ci->mfa.arity)) { + ci->native = 0; + ci->mfa.module = 0; + ci->mfa.function = 0; + ci->mfa.arity = 0; return; } @@ -6084,9 +6085,12 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) */ for (i = 0; i < n; i++) { - if (stp->export[i].function == function && stp->export[i].arity == arity) { - Export* ep = erts_export_put(mod, function, arity); - ep->addressv[erts_staging_code_ix()] = fp+5; + if (stp->export[i].function == ci->mfa.function && + stp->export[i].arity == ci->mfa.arity) { + Export* ep = erts_export_put(ci->mfa.module, + ci->mfa.function, + ci->mfa.arity); + ep->addressv[erts_staging_code_ix()] = erts_codeinfo_to_code(ci); return; } } @@ -6100,9 +6104,9 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) n = stp->num_lambdas; for (i = 0, lp = stp->lambdas; i < n; i++, lp++) { ErlFunEntry* fe = stp->lambdas[i].fe; - if (lp->function == function && lp->arity == arity) { - fp[5] = (Eterm) BeamOpCode(op_hipe_trap_call_closure); - fe->address = &(fp[5]); + if (lp->function == ci->mfa.function && lp->arity == ci->mfa.arity) { + *erts_codeinfo_to_code(ci) = (Eterm) BeamOpCode(op_hipe_trap_call_closure); + fe->address = erts_codeinfo_to_code(ci); } } #endif @@ -6426,20 +6430,21 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Set the pointer and make the stub. Put a return instruction * as the body until we know what kind of trap we should put there. */ - code_hdr->functions[i] = fp; + code_hdr->functions[i] = (ErtsCodeInfo*)fp; #ifdef HIPE op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */ #else op = (Eterm) BeamOpCode(op_move_return_n); #endif - fp = make_stub(fp, Mod, func, arity, (Uint)native_address, op); + fp = make_stub((ErtsCodeInfo*)fp, Mod, func, arity, + (Uint)native_address, op); } /* * Insert the last pointer and the int_code_end instruction. */ - code_hdr->functions[i] = fp; + code_hdr->functions[i] = (ErtsCodeInfo*)fp; *fp++ = (BeamInstr) BeamOp(op_int_code_end); /* @@ -6487,7 +6492,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) fp = code_base; for (i = 0; i < n; i++) { - stub_final_touch(stp, fp); + stub_final_touch(stp, (ErtsCodeInfo*)fp); fp += WORDS_PER_FUNCTION; } diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 6be4031822..f67dfd7a81 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -37,14 +37,6 @@ typedef struct gen_op_entry { extern const GenOpEntry gen_opc[]; -#ifdef NO_JUMP_TABLE -#define BeamOp(Op) (Op) -#else -extern void** beam_ops; -#define BeamOp(Op) beam_ops[(Op)] -#endif - - extern BeamInstr beam_debug_apply[]; extern BeamInstr* em_call_error_handler; extern BeamInstr* em_apply_bif; @@ -115,7 +107,7 @@ typedef struct beam_code_header { * The actual loaded code (for the first function) start just beyond * this table. */ - BeamInstr* functions[1]; + ErtsCodeInfo* functions[1]; }BeamCodeHeader; diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c index 55342a38c6..887f80e7e3 100644 --- a/erts/emulator/beam/beam_ranges.c +++ b/erts/emulator/beam/beam_ranges.c @@ -221,13 +221,13 @@ erts_ranges_sz(void) void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) { - BeamInstr** low; - BeamInstr** high; - BeamInstr** mid; + ErtsCodeInfo** low; + ErtsCodeInfo** high; + ErtsCodeInfo** mid; Range* rp; BeamCodeHeader* hdr; - fi->current = NULL; + fi->ci = NULL; fi->needed = 5; fi->loc = LINE_INVALID_LOCATION; rp = find_range(pc); @@ -240,12 +240,12 @@ erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) high = low + hdr->num_functions; while (low < high) { mid = low + (high-low) / 2; - if (pc < mid[0]) { + if (pc < (BeamInstr*)(mid[0])) { high = mid; - } else if (pc < mid[1]) { - fi->current = mid[0]+2; + } else if (pc < (BeamInstr*)(mid[1])) { + fi->ci = mid[0]; if (full_info) { - BeamInstr** fp = hdr->functions; + ErtsCodeInfo** fp = hdr->functions; int idx = mid - fp; lookup_loc(fi, pc, hdr, idx); } @@ -316,7 +316,7 @@ lookup_loc(FunctionInfo* fi, const BeamInstr* pc, file = LOC_FILE(fi->loc); if (file == 0) { /* Special case: Module name with ".erl" appended */ - Atom* mod_atom = atom_tab(atom_val(fi->current[0])); + Atom* mod_atom = atom_tab(atom_val(fi->ci->mfa.module)); fi->needed += 2*(mod_atom->len+4); } else { Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1])); diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 84eab5f651..324f7d4268 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -4963,13 +4963,13 @@ void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, int i; sys_memset((void *) ep, 0, sizeof(Export)); for (i=0; iaddressv[i] = &ep->code[3]; + ep->addressv[i] = ep->code; } - ep->code[0] = m; - ep->code[1] = f; - ep->code[2] = a; - ep->code[3] = (BeamInstr) em_apply_bif; - ep->code[4] = (BeamInstr) bif; + ep->info.mfa.module = m; + ep->info.mfa.function = f; + ep->info.mfa.arity = a; + ep->code[0] = (BeamInstr) em_apply_bif; + ep->code[1] = (BeamInstr) bif; } void erts_init_bif(void) diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index 2203182a0d..0c85e19ef0 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -159,7 +159,7 @@ do { \ #define ERTS_BIF_ERROR_TRAPPED0(Proc, Reason, Bif) \ do { \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ return THE_NON_VALUE; \ } while (0) @@ -167,7 +167,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ return THE_NON_VALUE; \ } while (0) @@ -176,7 +176,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ return THE_NON_VALUE; \ @@ -186,7 +186,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ reg[2] = (Eterm) (A2); \ @@ -202,7 +202,7 @@ do { \ #define ERTS_BIF_PREP_ERROR_TRAPPED0(Ret, Proc, Reason, Bif) \ do { \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -210,7 +210,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -219,7 +219,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ (Ret) = THE_NON_VALUE; \ @@ -229,7 +229,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ reg[2] = (Eterm) (A2); \ diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 3c19e82b66..4ee00b53be 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -231,9 +231,9 @@ print_process_info(int to, void *to_arg, Process *p) * Display the initial function name */ erts_print(to, to_arg, "Spawned as: %T:%T/%bpu\n", - p->u.initial[INITIAL_MOD], - p->u.initial[INITIAL_FUN], - p->u.initial[INITIAL_ARI]); + p->u.initial.module, + p->u.initial.function, + p->u.initial.arity); if (p->current != NULL) { if (running) { @@ -242,9 +242,9 @@ print_process_info(int to, void *to_arg, Process *p) erts_print(to, to_arg, "Current call: "); } erts_print(to, to_arg, "%T:%T/%bpu\n", - p->current[0], - p->current[1], - p->current[2]); + p->current->module, + p->current->function, + p->current->arity); } erts_print(to, to_arg, "Spawned by: %T\n", p->parent); @@ -291,9 +291,9 @@ print_process_info(int to, void *to_arg, Process *p) erts_print(to, to_arg, "timeout"); else erts_print(to, to_arg, "%T:%T/%bpu\n", - scb->ct[j]->code[0], - scb->ct[j]->code[1], - scb->ct[j]->code[2]); + scb->ct[j]->info.mfa.module, + scb->ct[j]->info.mfa.function, + scb->ct[j]->info.mfa.arity); } erts_print(to, to_arg, "\n"); } diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h index 584a605771..1b451bf921 100644 --- a/erts/emulator/beam/code_ix.h +++ b/erts/emulator/beam/code_ix.h @@ -56,12 +56,49 @@ # endif # include "sys.h" #endif + +#include "beam_opcodes.h" + struct process; #define ERTS_NUM_CODE_IX 3 typedef unsigned ErtsCodeIndex; +typedef struct ErtsCodeMFA_ { + Eterm module; + Eterm function; + Uint arity; +} ErtsCodeMFA; + +/* + * The ErtsCodeInfo structure is used both in the Export entry + * and in the code as the function header. + */ + +/* If you change the size of this, you also have to update the code + in ops.tab to reflect the new func_info size */ +typedef struct ErtsCodeInfo_ { + BeamInstr op; /* OpCode(i_func_info) */ + BeamInstr native; /* Used by hipe and trace to store extra data */ + ErtsCodeMFA mfa; +} ErtsCodeInfo; + +/* Get the code associated with a ErtsCodeInfo ptr. */ +ERTS_GLB_INLINE +BeamInstr *erts_codeinfo_to_code(ErtsCodeInfo *ci); + +/* Get the ErtsCodeInfo for from a code ptr. */ +ERTS_GLB_INLINE +ErtsCodeInfo *erts_code_to_codeinfo(BeamInstr *I); + +/* Get the code associated with a ErtsCodeMFA ptr. */ +ERTS_GLB_INLINE +BeamInstr *erts_codemfa_to_code(ErtsCodeMFA *mfa); + +/* Get the ErtsCodeMFA from a code ptr. */ +ERTS_GLB_INLINE +ErtsCodeMFA *erts_code_to_codemfa(BeamInstr *I); /* Called once at emulator initialization. */ @@ -121,10 +158,47 @@ void erts_abort_staging_code_ix(void); int erts_has_code_write_permission(void); #endif - +/* module/function/arity can be NIL/NIL/-1 when the MFA is pointing to some + invalid code, for instance unloaded_fun. */ +#define ASSERT_MFA(MFA) \ + ASSERT((is_atom((MFA)->module) || is_nil((MFA)->module)) && \ + (is_atom((MFA)->function) || is_nil((MFA)->function)) && \ + (((MFA)->arity >= 0 && (MFA)->arity < 1024) || (MFA)->arity == -1)) #if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE +BeamInstr *erts_codeinfo_to_code(ErtsCodeInfo *ci) +{ + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI) || !ci->op); + ASSERT_MFA(&ci->mfa); + return (BeamInstr*)(ci + 1); +} + +ERTS_GLB_INLINE +ErtsCodeInfo *erts_code_to_codeinfo(BeamInstr *I) +{ + ErtsCodeInfo *ci = ((ErtsCodeInfo *)(((char *)(I)) - sizeof(ErtsCodeInfo))); + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI) || !ci->op); + ASSERT_MFA(&ci->mfa); + return ci; +} + +ERTS_GLB_INLINE +BeamInstr *erts_codemfa_to_code(ErtsCodeMFA *mfa) +{ + ASSERT_MFA(mfa); + return (BeamInstr*)(mfa + 1); +} + +ERTS_GLB_INLINE +ErtsCodeMFA *erts_code_to_codemfa(BeamInstr *I) +{ + ErtsCodeMFA *mfa = ((ErtsCodeMFA *)(((char *)(I)) - sizeof(ErtsCodeMFA))); + ASSERT_MFA(mfa); + return mfa; +} + extern erts_smp_atomic32_t the_active_code_index; extern erts_smp_atomic32_t the_staging_code_index; diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 75389107bb..a985ce4918 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1114,9 +1114,9 @@ process_info_aux(Process *BIF_P, case am_initial_call: hp = HAlloc(BIF_P, 3+4); res = TUPLE3(hp, - rp->u.initial[INITIAL_MOD], - rp->u.initial[INITIAL_FUN], - make_small(rp->u.initial[INITIAL_ARI])); + rp->u.initial.module, + rp->u.initial.function, + make_small(rp->u.initial.arity)); hp += 4; break; @@ -1563,9 +1563,9 @@ process_info_aux(Process *BIF_P, term = am_timeout; else { term = TUPLE3(hp, - scb->ct[j]->code[0], - scb->ct[j]->code[1], - make_small(scb->ct[j]->code[2])); + scb->ct[j]->info.mfa.module, + scb->ct[j]->info.mfa.function, + make_small(scb->ct[j]->info.mfa.arity)); hp += 4; } list = CONS(hp, term, list); @@ -1614,10 +1614,12 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info) if (rp->current == NULL) { erts_lookup_function_info(&fi, rp->i, full_info); - rp->current = fi.current; + rp->current = &fi.ci->mfa; + ASSERT_MFA(rp->current); } else if (full_info) { + ASSERT_MFA(rp->current); erts_lookup_function_info(&fi, rp->i, full_info); - if (fi.current == NULL) { + if (fi.ci == NULL) { /* Use the current function without location info */ erts_set_current_function(&fi, rp->current); } @@ -1633,9 +1635,9 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info) * instead if it can be looked up. */ erts_lookup_function_info(&fi2, rp->cp, full_info); - if (fi2.current) { + if (fi2.ci) { fi = fi2; - rp->current = fi2.current; + rp->current = &fi2.ci->mfa; } } @@ -1650,8 +1652,9 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info) hp = erts_build_mfa_item(&fi, hp, am_true, &res); } else { hp = HAlloc(BIF_P, 3+4); - res = TUPLE3(hp, rp->current[0], - rp->current[1], make_small(rp->current[2])); + res = TUPLE3(hp, rp->current->module, + rp->current->function, + make_small(rp->current->arity)); hp += 4; } *hpp = hp; @@ -1692,7 +1695,7 @@ current_stacktrace(Process* p, Process* rp, Eterm** hpp) heap_size = 3; for (i = 0; i < depth; i++) { erts_lookup_function_info(stkp, s->trace[i], 1); - if (stkp->current) { + if (stkp->ci) { heap_size += stkp->needed + 2; stkp++; } @@ -3227,7 +3230,7 @@ fun_info_2(BIF_ALIST_2) break; case am_module: hp = HAlloc(p, 3); - val = exp->code[0]; + val = exp->info.mfa.module; break; case am_new_index: hp = HAlloc(p, 3); @@ -3255,11 +3258,11 @@ fun_info_2(BIF_ALIST_2) break; case am_arity: hp = HAlloc(p, 3); - val = make_small(exp->code[2]); + val = make_small(exp->info.mfa.arity); break; case am_name: hp = HAlloc(p, 3); - val = exp->code[1]; + val = exp->info.mfa.function; break; default: goto error; @@ -3285,7 +3288,9 @@ fun_info_mfa_1(BIF_ALIST_1) } else if (is_export(fun)) { Export* exp = (Export *) ((UWord) (export_val(fun))[1]); hp = HAlloc(p, 4); - BIF_RET(TUPLE3(hp,exp->code[0],exp->code[1],make_small(exp->code[2]))); + BIF_RET(TUPLE3(hp,exp->info.mfa.module, + exp->info.mfa.function, + make_small(exp->info.mfa.arity))); } BIF_ERROR(p, BADARG); } diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c index aecb8bf0c1..a594ec1493 100644 --- a/erts/emulator/beam/erl_bif_op.c +++ b/erts/emulator/beam/erl_bif_op.c @@ -260,7 +260,7 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2) } else if (is_export(arg1)) { Export* exp = (Export *) (export_val(arg1)[1]); - if (exp->code[2] == (Uint) arity) { + if (exp->info.mfa.arity == (Uint) arity) { BIF_RET(am_true); } } diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 66e5146da0..e3355208e5 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -125,7 +125,6 @@ erts_internal_trace_pattern_3(BIF_ALIST_3) static Eterm trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) { - DeclareTmpHeap(mfa,3,p); /* Not really heap here, but might be when setting pattern */ int i; int matches = -1; int specified = 0; @@ -308,30 +307,30 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) } matches = 0; } else if (is_tuple(MFA)) { + ErtsCodeMFA mfa; Eterm *tp = tuple_val(MFA); if (tp[0] != make_arityval(3)) { goto error; } - mfa[0] = tp[1]; - mfa[1] = tp[2]; - mfa[2] = tp[3]; - if (!is_atom(mfa[0]) || !is_atom(mfa[1]) || - (!is_small(mfa[2]) && mfa[2] != am_Underscore)) { + if (!is_atom(tp[1]) || !is_atom(tp[2]) || + (!is_small(tp[3]) && tp[3] != am_Underscore)) { goto error; } - for (i = 0; i < 3 && mfa[i] != am_Underscore; i++, specified++) { + for (i = 0; i < 3 && tp[i+1] != am_Underscore; i++, specified++) { /* Empty loop body */ } for (i = specified; i < 3; i++) { - if (mfa[i] != am_Underscore) { + if (tp[i+1] != am_Underscore) { goto error; } } - if (is_small(mfa[2])) { - mfa[2] = signed_val(mfa[2]); + mfa.module = tp[1]; + mfa.function = tp[2]; + if (specified == 3) { + mfa.arity = signed_val(tp[3]); } - matches = erts_set_trace_pattern(p, mfa, specified, + matches = erts_set_trace_pattern(p, &mfa, specified, match_prog_set, match_prog_set, on, flags, meta_tracer, 0); } else if (is_atom(MFA)) { @@ -343,7 +342,6 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) error: MatchSetUnref(match_prog_set); - UnUseTmpHeap(3,p); ERTS_TRACER_CLEAR(&meta_tracer); @@ -975,13 +973,14 @@ static int function_is_traced(Process *p, Export e; Export* ep; BeamInstr* pc; + ErtsCodeInfo *ci; /* First look for an export entry */ - e.code[0] = mfa[0]; - e.code[1] = mfa[1]; - e.code[2] = mfa[2]; + e.info.mfa.module = mfa[0]; + e.info.mfa.function = mfa[1]; + e.info.mfa.arity = mfa[2]; if ((ep = export_get(&e)) != NULL) { - pc = ep->code+3; + pc = ep->code; if (ep->addressv[erts_active_code_ix()] == pc && *pc != (BeamInstr) em_call_error_handler) { @@ -990,17 +989,17 @@ static int function_is_traced(Process *p, ASSERT(*pc == (BeamInstr) em_apply_bif || *pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)); - if (erts_is_trace_break(pc, ms, 0)) { + if (erts_is_trace_break(&ep->info, ms, 0)) { return FUNC_TRACE_GLOBAL_TRACE; } - if (erts_is_trace_break(pc, ms, 1)) { + if (erts_is_trace_break(&ep->info, ms, 1)) { r |= FUNC_TRACE_LOCAL_TRACE; } - if (erts_is_mtrace_break(pc, ms_meta, tracer_pid_meta)) { + if (erts_is_mtrace_break(&ep->info, ms_meta, tracer_pid_meta)) { r |= FUNC_TRACE_META_TRACE; } - if (erts_is_time_break(p, pc, call_time)) { + if (erts_is_time_break(p, &ep->info, call_time)) { r |= FUNC_TRACE_TIME_TRACE; } return r ? r : FUNC_TRACE_UNTRACED; @@ -1008,15 +1007,15 @@ static int function_is_traced(Process *p, } /* OK, now look for breakpoint tracing */ - if ((pc = erts_find_local_func(mfa)) != NULL) { + if ((ci = erts_find_local_func(&e.info.mfa)) != NULL) { int r = - (erts_is_trace_break(pc, ms, 1) + (erts_is_trace_break(ci, ms, 1) ? FUNC_TRACE_LOCAL_TRACE : 0) - | (erts_is_mtrace_break(pc, ms_meta, tracer_pid_meta) + | (erts_is_mtrace_break(ci, ms_meta, tracer_pid_meta) ? FUNC_TRACE_META_TRACE : 0) - | (erts_is_count_break(pc, count) + | (erts_is_count_break(ci, count) ? FUNC_TRACE_COUNT_TRACE : 0) - | (erts_is_time_break(p, pc, call_time) + | (erts_is_time_break(p, ci, call_time) ? FUNC_TRACE_TIME_TRACE : 0); return r ? r : FUNC_TRACE_UNTRACED; @@ -1350,7 +1349,7 @@ trace_info_event(Process* p, Eterm event, Eterm key) #undef FUNC_TRACE_LOCAL_TRACE int -erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, +erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified, Binary* match_prog_set, Binary *meta_match_prog_set, int on, struct trace_pattern_flags flags, ErtsTracer meta_tracer, int is_blocking) @@ -1370,22 +1369,23 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, n = finish_bp.e.matched; for (i = 0; i < n; i++) { - BeamInstr* pc = fp[i].pc; - Export* ep = ErtsContainerStruct(pc, Export, code[3]); + ErtsCodeInfo *ci = fp[i].ci; + BeamInstr* pc = erts_codeinfo_to_code(ci); + Export* ep = ErtsContainerStruct(ci, Export, info); if (on && !flags.breakpoint) { /* Turn on global call tracing */ if (ep->addressv[code_ix] != pc) { fp[i].mod->curr.num_traced_exports++; #ifdef DEBUG - pc[-5] = (BeamInstr) BeamOp(op_i_func_info_IaaI); + ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI); #endif - pc[0] = (BeamInstr) BeamOp(op_jump_f); - pc[1] = (BeamInstr) ep->addressv[code_ix]; + ep->code[0] = (BeamInstr) BeamOp(op_jump_f); + ep->code[1] = (BeamInstr) ep->addressv[code_ix]; } - erts_set_call_trace_bif(pc, match_prog_set, 0); + erts_set_call_trace_bif(ci, match_prog_set, 0); if (ep->addressv[code_ix] != pc) { - pc[0] = (BeamInstr) BeamOp(op_i_generic_breakpoint); + ep->code[0] = (BeamInstr) BeamOp(op_i_generic_breakpoint); } } else if (!on && flags.breakpoint) { /* Turn off breakpoint tracing -- nothing to do here. */ @@ -1394,9 +1394,9 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, * Turn off global tracing, either explicitly or implicitly * before turning on breakpoint tracing. */ - erts_clear_call_trace_bif(pc, 0); - if (pc[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { - pc[0] = (BeamInstr) BeamOp(op_jump_f); + erts_clear_call_trace_bif(ci, 0); + if (ep->code[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { + ep->code[0] = (BeamInstr) BeamOp(op_jump_f); } } } @@ -1406,68 +1406,76 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, */ for (i = 0; i < BIF_SIZE; ++i) { Export *ep = bif_export[i]; - int j; - + if (!ExportIsBuiltIn(ep)) { continue; } - + if (bif_table[i].f == bif_table[i].traced) { /* Trace wrapper same as regular function - untraceable */ continue; } - - for (j = 0; j < specified && mfa[j] == ep->code[j]; j++) { - /* Empty loop body */ - } - if (j == specified) { - BeamInstr* pc = (BeamInstr *)bif_export[i]->code + 3; - if (! flags.breakpoint) { /* Export entry call trace */ - if (on) { - erts_clear_call_trace_bif(pc, 1); - erts_clear_mtrace_bif(pc); - erts_set_call_trace_bif(pc, match_prog_set, 0); - } else { /* off */ - erts_clear_call_trace_bif(pc, 0); - } - matches++; - } else { /* Breakpoint call trace */ - int m = 0; - - if (on) { - if (flags.local) { - erts_clear_call_trace_bif(pc, 0); - erts_set_call_trace_bif(pc, match_prog_set, 1); - m = 1; - } - if (flags.meta) { - erts_set_mtrace_bif(pc, meta_match_prog_set, - meta_tracer); - m = 1; - } - if (flags.call_time) { - erts_set_time_trace_bif(pc, on); - /* I don't want to remove any other tracers */ - m = 1; - } - } else { /* off */ - if (flags.local) { - erts_clear_call_trace_bif(pc, 1); - m = 1; - } - if (flags.meta) { - erts_clear_mtrace_bif(pc); - m = 1; - } - if (flags.call_time) { - erts_clear_time_trace_bif(pc); - m = 1; - } - } - matches += m; - } - } + switch (specified) { + case 3: + if (mfa->arity != ep->info.mfa.arity) + continue; + case 2: + if (mfa->function != ep->info.mfa.function) + continue; + case 1: + if (mfa->module != ep->info.mfa.module) + continue; + case 0: + break; + default: + ASSERT(0); + } + + if (! flags.breakpoint) { /* Export entry call trace */ + if (on) { + erts_clear_call_trace_bif(&ep->info, 1); + erts_clear_mtrace_bif(&ep->info); + erts_set_call_trace_bif(&ep->info, match_prog_set, 0); + } else { /* off */ + erts_clear_call_trace_bif(&ep->info, 0); + } + matches++; + } else { /* Breakpoint call trace */ + int m = 0; + + if (on) { + if (flags.local) { + erts_clear_call_trace_bif(&ep->info, 0); + erts_set_call_trace_bif(&ep->info, match_prog_set, 1); + m = 1; + } + if (flags.meta) { + erts_set_mtrace_bif(&ep->info, meta_match_prog_set, + meta_tracer); + m = 1; + } + if (flags.call_time) { + erts_set_time_trace_bif(&ep->info, on); + /* I don't want to remove any other tracers */ + m = 1; + } + } else { /* off */ + if (flags.local) { + erts_clear_call_trace_bif(&ep->info, 1); + m = 1; + } + if (flags.meta) { + erts_clear_mtrace_bif(&ep->info); + m = 1; + } + if (flags.call_time) { + erts_clear_time_trace_bif(&ep->info); + m = 1; + } + } + matches += m; + } } /* @@ -1669,10 +1677,9 @@ install_exp_breakpoints(BpFunctions* f) Uint i; for (i = 0; i < ne; i++) { - BeamInstr* pc = fp[i].pc; - Export* ep = ErtsContainerStruct(pc, Export, code[3]); + Export* ep = ErtsContainerStruct(fp[i].ci, Export, info); - ep->addressv[code_ix] = pc; + ep->addressv[code_ix] = ep->code; } } @@ -1685,14 +1692,13 @@ uninstall_exp_breakpoints(BpFunctions* f) Uint i; for (i = 0; i < ne; i++) { - BeamInstr* pc = fp[i].pc; - Export* ep = ErtsContainerStruct(pc, Export, code[3]); + Export* ep = ErtsContainerStruct(fp[i].ci, Export, info); - if (ep->addressv[code_ix] != pc) { + if (ep->addressv[code_ix] != ep->code) { continue; } - ASSERT(*pc == (BeamInstr) BeamOp(op_jump_f)); - ep->addressv[code_ix] = (BeamInstr *) ep->code[4]; + ASSERT(ep->code[0] == (BeamInstr) BeamOp(op_jump_f)); + ep->addressv[code_ix] = (BeamInstr *) ep->code[1]; } } @@ -1705,15 +1711,14 @@ clean_export_entries(BpFunctions* f) Uint i; for (i = 0; i < ne; i++) { - BeamInstr* pc = fp[i].pc; - Export* ep = ErtsContainerStruct(pc, Export, code[3]); + Export* ep = ErtsContainerStruct(fp[i].ci, Export, info); - if (ep->addressv[code_ix] == pc) { + if (ep->addressv[code_ix] == ep->code) { continue; } - if (*pc == (BeamInstr) BeamOp(op_jump_f)) { - ep->code[3] = (BeamInstr) 0; - ep->code[4] = (BeamInstr) 0; + if (ep->code[0] == (BeamInstr) BeamOp(op_jump_f)) { + ep->code[0] = (BeamInstr) 0; + ep->code[1] = (BeamInstr) 0; } } } @@ -1725,11 +1730,11 @@ setup_bif_trace(void) for (i = 0; i < BIF_SIZE; ++i) { Export *ep = bif_export[i]; - GenericBp* g = (GenericBp *) ep->fake_op_func_info_for_hipe[1]; + GenericBp* g = (GenericBp *) ep->info.native; if (g) { if (ExportIsBuiltIn(ep)) { - ASSERT(ep->code[4]); - ep->code[4] = (BeamInstr) bif_table[i].traced; + ASSERT(ep->code[1]); + ep->code[1] = (BeamInstr) bif_table[i].traced; } } } @@ -1743,12 +1748,11 @@ reset_bif_trace(void) for (i = 0; i < BIF_SIZE; ++i) { Export *ep = bif_export[i]; - BeamInstr* pc = ep->code+3; - GenericBp* g = (GenericBp *) pc[-4]; + GenericBp* g = (GenericBp *) ep->info.native; if (g && g->data[active].flags == 0) { if (ExportIsBuiltIn(ep)) { - ASSERT(ep->code[4]); - ep->code[4] = (BeamInstr) bif_table[i].f; + ASSERT(ep->code[1]); + ep->code[1] = (BeamInstr) bif_table[i].f; } } } diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 6732b708a8..988467ce44 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1769,7 +1769,7 @@ Eterm db_prog_match(Process *c_p, Eterm t; Eterm *esp; MatchVariable* variables; - BeamInstr *cp; + ErtsCodeInfo *cp; const UWord *pc = prog->text; Eterm *ehp; Eterm ret; @@ -2408,9 +2408,9 @@ restart: ehp = HAllocX(build_proc, 4, HEAP_XTRA); *esp++ = make_tuple(ehp); ehp[0] = make_arityval(3); - ehp[1] = cp[0]; - ehp[2] = cp[1]; - ehp[3] = make_small((Uint) cp[2]); + ehp[1] = cp->mfa.module; + ehp[2] = cp->mfa.function; + ehp[3] = make_small((Uint) cp->mfa.arity); } break; case matchSilent: diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index c639ba623f..9deec946d5 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -49,8 +49,8 @@ static void fun_free(ErlFunEntry* obj); * to unloaded_fun[]. The -1 in unloaded_fun[0] will be interpreted * as an illegal arity when attempting to call a fun. */ -static BeamInstr unloaded_fun_code[3] = {NIL, -1, 0}; -static BeamInstr* unloaded_fun = unloaded_fun_code + 2; +static BeamInstr unloaded_fun_code[4] = {NIL, NIL, -1, 0}; +static BeamInstr* unloaded_fun = unloaded_fun_code + 3; void erts_init_fun_table(void) diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 7442e99cb3..f0535451fc 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2285,7 +2285,7 @@ typedef struct { Export exp; struct erl_module_nif* m; NativeFunPtr fp; - BeamInstr *saved_current; + ErtsCodeMFA *saved_current; int exception_thrown; int saved_argc; int rootset_extra; @@ -2328,9 +2328,9 @@ allocate_nif_sched_data(Process* proc, int argc) ep->rootset_extra = argc; ep->rootset[0] = NIL; for (i=0; iexp.addressv[i] = &ep->exp.code[3]; + ep->exp.addressv[i] = &ep->exp.code[0]; } - ep->exp.code[3] = (BeamInstr) em_call_nif; + ep->exp.code[0] = (BeamInstr) em_call_nif; (void) ERTS_PROC_SET_NIF_TRAP_EXPORT(proc, ep); return ep; } @@ -2401,10 +2401,10 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec ep->saved_argc = argc; } proc->i = (BeamInstr*) ep->exp.addressv[0]; - ep->exp.code[0] = (BeamInstr) proc->current[0]; - ep->exp.code[1] = (BeamInstr) proc->current[1]; - ep->exp.code[2] = argc; - ep->exp.code[4] = (BeamInstr) direct_fp; + ep->exp.info.mfa.module = proc->current->module; + ep->exp.info.mfa.function = proc->current->function; + ep->exp.info.mfa.arity = argc; + ep->exp.code[1] = (BeamInstr) direct_fp; ep->m = env->mod_nif; ep->fp = indirect_fp; proc->freason = TRAP; @@ -2426,7 +2426,7 @@ restore_nif_mfa(Process* proc, NifExport* ep, int exception) ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(proc) & ERTS_PROC_LOCK_MAIN); - ASSERT(ep->saved_current != &ep->exp.code[0]); + ASSERT(ep->saved_current != &ep->exp.info.mfa); proc->current = ep->saved_current; ep->saved_current = NULL; if (exception) { @@ -2500,7 +2500,8 @@ execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) execution_state(env, &proc, NULL); - fp = (NativeFunPtr) proc->current[6]; + ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); + fp = ep->fp; ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc))); @@ -2569,7 +2570,8 @@ schedule_dirty_nif(ErlNifEnv* env, int flags, int argc, const ERL_NIF_TERM argv[ erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); } - fp = (NativeFunPtr) proc->current[6]; + ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); + fp = ep->fp; ASSERT(fp); @@ -2629,7 +2631,8 @@ execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ERL_NIF_TERM result; execution_state(env, &proc, NULL); - fp = (NativeFunPtr) proc->current[6]; + ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); + fp = ep->fp; ASSERT(!env->exception_thrown); ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); @@ -2697,7 +2700,7 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); ASSERT(ep); - ep->exp.code[1] = (BeamInstr) fun_name_atom; + ep->exp.info.mfa.function = (BeamInstr) fun_name_atom; done: if (scheduler < 0) @@ -3011,17 +3014,16 @@ int enif_map_iterator_get_pair(ErlNifEnv *env, ***************************************************************************/ -static BeamInstr** get_func_pp(BeamCodeHeader* mod_code, Eterm f_atom, unsigned arity) +static ErtsCodeInfo** get_func_pp(BeamCodeHeader* mod_code, Eterm f_atom, unsigned arity) { int n = (int) mod_code->num_functions; int j; for (j = 0; j < n; ++j) { - BeamInstr* code_ptr = (BeamInstr*) mod_code->functions[j]; - ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - if (f_atom == ((Eterm) code_ptr[3]) - && arity == ((unsigned) code_ptr[4])) { - - return (BeamInstr**) &mod_code->functions[j]; + ErtsCodeInfo* ci = mod_code->functions[j]; + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + if (f_atom == ci->mfa.function + && arity == ci->mfa.arity) { + return mod_code->functions+j; } } return NULL; @@ -3156,7 +3158,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) Eterm mod_atom; const Atom* mod_atomp; Eterm f_atom; - BeamInstr* caller; + ErtsCodeInfo* caller; ErtsSysDdllError errdesc = ERTS_SYS_DDLL_ERROR_INIT; Eterm ret = am_ok; int veto; @@ -3189,12 +3191,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) /* Find calling module */ ASSERT(BIF_P->current != NULL); - ASSERT(BIF_P->current[0] == am_erlang - && BIF_P->current[1] == am_load_nif - && BIF_P->current[2] == 2); + ASSERT(BIF_P->current->module == am_erlang + && BIF_P->current->function == am_load_nif + && BIF_P->current->arity == 2); caller = find_function_from_pc(BIF_P->cp); ASSERT(caller != NULL); - mod_atom = caller[0]; + mod_atom = caller->mfa.module; ASSERT(is_atom(mod_atom)); module_p = erts_get_module(mod_atom, erts_active_code_ix()); ASSERT(module_p != NULL); @@ -3271,9 +3273,9 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) int incr = 0; ErlNifFunc* f = entry->funcs; for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) { - BeamInstr** code_pp; + ErtsCodeInfo** ci_pp; if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1) - || (code_pp = get_func_pp(this_mi->code_hdr, f_atom, f->arity))==NULL) { + || (ci_pp = get_func_pp(this_mi->code_hdr, f_atom, f->arity))==NULL) { ret = load_nif_error(BIF_P,bad_lib,"Function not found %T:%s/%u", mod_atom, f->name, f->arity); } @@ -3295,9 +3297,9 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) #endif } #ifdef ERTS_DIRTY_SCHEDULERS - else if (code_pp[1] - code_pp[0] < (5+4)) + else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) < (4)) #else - else if (code_pp[1] - code_pp[0] < (5+3)) + else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) < (3)) #endif { ret = load_nif_error(BIF_P,bad_lib,"No explicit call to load_nif" @@ -3365,31 +3367,33 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) this_mi->nif = lib; for (i=0; i < entry->num_of_funcs; i++) { - BeamInstr* code_ptr; + ErtsCodeInfo* ci; + BeamInstr *code_ptr; erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1); - code_ptr = *get_func_pp(this_mi->code_hdr, f_atom, f->arity); + ci = *get_func_pp(this_mi->code_hdr, f_atom, f->arity); + code_ptr = erts_codeinfo_to_code(ci); - if (code_ptr[1] == 0) { - code_ptr[5+0] = (BeamInstr) BeamOp(op_call_nif); + if (ci->native == 0) { + code_ptr[0] = (BeamInstr) BeamOp(op_call_nif); } else { /* Function traced, patch the original instruction word */ - GenericBp* g = (GenericBp *) code_ptr[1]; - ASSERT(code_ptr[5+0] == + GenericBp* g = (GenericBp *) ci->native; + ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)); g->orig_instr = (BeamInstr) BeamOp(op_call_nif); } #ifdef ERTS_DIRTY_SCHEDULERS if ((entry->major > 2 || (entry->major == 2 && entry->minor >= 7)) && (entry->options & ERL_NIF_DIRTY_NIF_OPTION) && f->flags) { - code_ptr[5+3] = (BeamInstr) f->fptr; - code_ptr[5+1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ? + code_ptr[3] = (BeamInstr) f->fptr; + code_ptr[1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ? (BeamInstr) schedule_dirty_io_nif : (BeamInstr) schedule_dirty_cpu_nif; } else #endif - code_ptr[5+1] = (BeamInstr) f->fptr; - code_ptr[5+2] = (BeamInstr) lib; + code_ptr[1] = (BeamInstr) f->fptr; + code_ptr[2] = (BeamInstr) lib; f = next_func(entry, &incr, f); } } diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index 1a579704a8..b43a1b0190 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -526,8 +526,8 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) { case EXPORT_DEF: { Export* ep = *((Export **) (export_val(wobj) + 1)); - Atom* module = atom_tab(atom_val(ep->code[0])); - Atom* name = atom_tab(atom_val(ep->code[1])); + Atom* module = atom_tab(atom_val(ep->info.mfa.module)); + Atom* name = atom_tab(atom_val(ep->info.mfa.function)); PRINT_STRING(res, fn, arg, "#Fun<"); PRINT_BUF(res, fn, arg, module->name, module->len); @@ -535,7 +535,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) { PRINT_BUF(res, fn, arg, name->name, name->len); PRINT_CHAR(res, fn, arg, '.'); PRINT_SWORD(res, fn, arg, 'd', 0, 1, - (ErlPfSWord) ep->code[2]); + (ErlPfSWord) ep->info.mfa.arity); PRINT_CHAR(res, fn, arg, '>'); } break; diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 269518d8d6..f7f391e322 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -11464,9 +11464,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->schedule_count = 0; ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0)); - p->u.initial[INITIAL_MOD] = mod; - p->u.initial[INITIAL_FUN] = func; - p->u.initial[INITIAL_ARI] = (Uint) arity; + p->u.initial.module = mod; + p->u.initial.function = func; + p->u.initial.arity = (Uint) arity; /* * Must initialize binary lists here before copying binaries to process. @@ -11508,7 +11508,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). /* No need to initialize p->fcalls. */ - p->current = p->u.initial+INITIAL_MOD; + p->current = &p->u.initial; p->i = (BeamInstr *) beam_apply; p->cp = (BeamInstr *) beam_apply+1; @@ -11769,9 +11769,9 @@ void erts_init_empty_process(Process *p) p->seq_trace_clock = 0; p->seq_trace_lastcnt = 0; p->seq_trace_token = NIL; - p->u.initial[0] = 0; - p->u.initial[1] = 0; - p->u.initial[2] = 0; + p->u.initial.module = 0; + p->u.initial.function = 0; + p->u.initial.arity = 0; p->catches = 0; p->cp = NULL; p->i = NULL; @@ -13220,8 +13220,8 @@ erts_program_counter_info(int to, void *to_arg, Process *p) static void print_function_from_pc(int to, void *to_arg, BeamInstr* x) { - BeamInstr* addr = find_function_from_pc(x); - if (addr == NULL) { + ErtsCodeInfo *ci = find_function_from_pc(x); + if (ci == NULL) { if (x == beam_exit) { erts_print(to, to_arg, ""); } else if (x == beam_continue_exit) { @@ -13235,7 +13235,8 @@ print_function_from_pc(int to, void *to_arg, BeamInstr* x) } } else { erts_print(to, to_arg, "%T:%T/%d + %d", - addr[0], addr[1], addr[2], ((x-addr)-2) * sizeof(Eterm)); + ci->mfa.module, ci->mfa.function, ci->mfa.arity, + (x-(BeamInstr*)ci) * sizeof(Eterm)); } } diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 3347a7a60e..b266d32e76 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -93,10 +93,6 @@ struct ErtsNodesMonitor_; #define ERTS_HEAP_FREE(Type, Ptr, Size) \ erts_free((Type), (Ptr)) -#define INITIAL_MOD 0 -#define INITIAL_FUN 1 -#define INITIAL_ARI 2 - #include "export.h" struct saved_calls { @@ -1022,15 +1018,16 @@ struct process { #endif union { void *terminate; - BeamInstr initial[3]; /* Initial module(0), function(1), arity(2), often used instead - of pointer to funcinfo instruction, hence the BeamInstr datatype */ + ErtsCodeMFA initial; /* Initial module(0), function(1), arity(2), + often used instead of pointer to funcinfo + instruction. */ } u; - BeamInstr* current; /* Current Erlang function, part of the funcinfo: + ErtsCodeMFA* current; /* Current Erlang function, part of the funcinfo: * module(0), function(1), arity(2) * (module and functions are tagged atoms; - * arity an untagged integer). BeamInstr * because it references code + * arity an untagged integer). */ - + /* * Information mainly for post-mortem use (erl crash dump). */ diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index a70dfb8e73..645bedd8f1 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -334,8 +334,8 @@ stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg) static void print_function_from_pc(int to, void *to_arg, BeamInstr* x) { - BeamInstr* addr = find_function_from_pc(x); - if (addr == NULL) { + ErtsCodeInfo* ci = find_function_from_pc(x); + if (ci == NULL) { if (x == beam_exit) { erts_print(to, to_arg, ""); } else if (x == beam_continue_exit) { @@ -347,7 +347,8 @@ print_function_from_pc(int to, void *to_arg, BeamInstr* x) } } else { erts_print(to, to_arg, "%T:%T/%bpu + %bpu", - addr[0], addr[1], addr[2], ((x-addr)-2) * sizeof(Eterm)); + ci->mfa.module, ci->mfa.function, ci->mfa.arity, + (x-(BeamInstr*)ci) * sizeof(Eterm)); } } diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 8c84303997..566529d2d4 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -773,7 +773,7 @@ trace_sched_aux(Process *p, ErtsProcLocks locks, Eterm what) curr_func = 0; else { if (!p->current) - p->current = find_function_from_pc(p->i); + p->current = &find_function_from_pc(p->i)->mfa; curr_func = p->current != NULL; } @@ -781,7 +781,8 @@ trace_sched_aux(Process *p, ErtsProcLocks locks, Eterm what) tmp = make_small(0); } else { hp = HAlloc(p, 4); - tmp = TUPLE3(hp,p->current[0],p->current[1],make_small(p->current[2])); + tmp = TUPLE3(hp,p->current->module,p->current->function, + make_small(p->current->arity)); hp += 4; } @@ -1027,14 +1028,14 @@ erts_trace_return_to(Process *p, BeamInstr *pc) { Eterm mfa; - BeamInstr *code_ptr = find_function_from_pc(pc); + ErtsCodeInfo *ci = find_function_from_pc(pc); - - if (!code_ptr) { + if (!ci) { mfa = am_undefined; } else { Eterm *hp = HAlloc(p, 4); - mfa = TUPLE3(hp, code_ptr[0], code_ptr[1], make_small(code_ptr[2])); + mfa = TUPLE3(hp, ci->mfa.module, ci->mfa.function, + make_small(ci->mfa.arity)); } send_to_tracer_nif(p, &p->common, p->common.id, NULL, TRACE_FUN_T_CALL, @@ -1046,11 +1047,11 @@ erts_trace_return_to(Process *p, BeamInstr *pc) * or {trace, Pid, return_from, {Mod, Name, Arity}, Retval} */ void -erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, ErtsTracer *tracer) +erts_trace_return(Process* p, ErtsCodeMFA *mfa, + Eterm retval, ErtsTracer *tracer) { Eterm* hp; - Eterm mfa, mod, name; - int arity; + Eterm mfa_tuple; Uint meta_flags, *tracee_flags; ASSERT(tracer); @@ -1084,15 +1085,13 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, ErtsTracer *tracer) tracee_flags = &meta_flags; } - mod = fi[0]; - name = fi[1]; - arity = fi[2]; - hp = HAlloc(p, 4); - mfa = TUPLE3(hp, mod, name, make_small(arity)); + mfa_tuple = TUPLE3(hp, mfa->module, mfa->function, + make_small(mfa->arity)); hp += 4; send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id, - NULL, TRACE_FUN_T_CALL, am_return_from, mfa, retval, am_true); + NULL, TRACE_FUN_T_CALL, am_return_from, mfa_tuple, + retval, am_true); } /* Send {trace_ts, Pid, exception_from, {Mod, Name, Arity}, {Class,Value}, @@ -1103,7 +1102,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, ErtsTracer *tracer) * Where Class is atomic but Value is any term. */ void -erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, +erts_trace_exception(Process* p, ErtsCodeMFA *mfa, Eterm class, Eterm value, ErtsTracer *tracer) { Eterm* hp; @@ -1142,7 +1141,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, } hp = HAlloc(p, 7);; - mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], make_small((Eterm)mfa[2])); + mfa_tuple = TUPLE3(hp, mfa->module, mfa->function, make_small(mfa->arity)); hp += 4; cv = TUPLE2(hp, class, value); hp += 3; @@ -1165,7 +1164,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, * if it is a pid or port we do a meta trace. */ Uint32 -erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, +erts_call_trace(Process* p, ErtsCodeInfo *info, Binary *match_spec, Eterm* args, int local, ErtsTracer *tracer) { Eterm* hp; @@ -1244,7 +1243,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, * such as size_object() and copy_struct(), we must make sure that we * temporarily convert any match contexts to sub binaries. */ - arity = (Eterm) mfa[2]; + arity = info->mfa.arity; for (i = 0; i < arity; i++) { Eterm arg = args[i]; if (is_boxed(arg) && header_is_bin_matchstate(*boxed_val(arg))) { @@ -1339,7 +1338,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, hp += 2; } } - mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], mfa_tuple); + mfa_tuple = TUPLE3(hp, info->mfa.module, info->mfa.function, mfa_tuple); hp += 4; /* @@ -1459,7 +1458,8 @@ trace_gc(Process *p, Eterm what, Uint size, Eterm msg) } void -monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint time) +monitor_long_schedule_proc(Process *p, ErtsCodeInfo *in_fp, + ErtsCodeInfo *out_fp, Uint time) { ErlHeapFragment *bp; ErlOffHeap *off_heap; @@ -1490,11 +1490,13 @@ monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, monitor_p); tmo = erts_bld_uint(&hp, NULL, time); if (in_fp != NULL) { - in_mfa = TUPLE3(hp,(Eterm) in_fp[0], (Eterm) in_fp[1], make_small(in_fp[2])); + in_mfa = TUPLE3(hp, in_fp->mfa.module, in_fp->mfa.function, + make_small(in_fp->mfa.arity)); hp +=4; } if (out_fp != NULL) { - out_mfa = TUPLE3(hp,(Eterm) out_fp[0], (Eterm) out_fp[1], make_small(out_fp[2])); + out_mfa = TUPLE3(hp, out_fp->mfa.module, out_fp->mfa.function, + make_small(out_fp->mfa.arity)); hp +=4; } tmo_tpl = TUPLE2(hp,am_timeout, tmo); @@ -2129,7 +2131,7 @@ profile_runnable_proc(Process *p, Eterm status){ Eterm *hp, msg; Eterm where = am_undefined; ErlHeapFragment *bp = NULL; - BeamInstr *current = NULL; + ErtsCodeInfo *ci = NULL; #ifndef ERTS_SMP #define LOCAL_HEAP_SIZE (4 + 6 + ERTS_TRACE_PATCH_TS_MAX_SIZE) @@ -2151,14 +2153,14 @@ profile_runnable_proc(Process *p, Eterm status){ if (!ERTS_PROC_IS_EXITING(p)) { if (p->current) { - current = p->current; + ci = erts_code_to_codeinfo(erts_codemfa_to_code(p->current)); } else { - current = find_function_from_pc(p->i); + ci = find_function_from_pc(p->i); } } #ifdef ERTS_SMP - if (!current) { + if (!ci) { hsz -= 4; } @@ -2166,8 +2168,10 @@ profile_runnable_proc(Process *p, Eterm status){ hp = bp->mem; #endif - if (current) { - where = TUPLE3(hp, current[0], current[1], make_small(current[2])); hp += 4; + if (ci) { + where = TUPLE3(hp, ci->mfa.module, ci->mfa.function, + make_small(ci->mfa.arity)); + hp += 4; } else { where = make_small(0); } diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h index 0095d4386b..74df3e50e1 100644 --- a/erts/emulator/beam/erl_trace.h +++ b/erts/emulator/beam/erl_trace.h @@ -101,11 +101,11 @@ void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *); void trace_send(Process*, Eterm, Eterm); void trace_receive(Process*, Eterm, Eterm, ErtsTracingEvent*); -Uint32 erts_call_trace(Process *p, BeamInstr mfa[], struct binary *match_spec, +Uint32 erts_call_trace(Process *p, ErtsCodeInfo *info, struct binary *match_spec, Eterm* args, int local, ErtsTracer *tracer); -void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, +void erts_trace_return(Process* p, ErtsCodeMFA *mfa, Eterm retval, ErtsTracer *tracer); -void erts_trace_exception(Process* p, BeamInstr mfa[], Eterm class, Eterm value, +void erts_trace_exception(Process* p, ErtsCodeMFA *mfa, Eterm class, Eterm value, ErtsTracer *tracer); void erts_trace_return_to(Process *p, BeamInstr *pc); void trace_sched(Process*, ErtsProcLocks, Eterm); @@ -134,7 +134,8 @@ void erts_system_profile_setup_active_schedulers(void); /* system_monitor */ void monitor_long_gc(Process *p, Uint time); -void monitor_long_schedule_proc(Process *p, BeamInstr *in_i, BeamInstr *out_i, Uint time); +void monitor_long_schedule_proc(Process *p, ErtsCodeInfo *in_i, + ErtsCodeInfo *out_i, Uint time); void monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time); void monitor_large_heap(Process *p); void monitor_generic(Process *p, Eterm type, Eterm spec); @@ -176,7 +177,7 @@ struct trace_pattern_flags { }; extern const struct trace_pattern_flags erts_trace_pattern_flags_off; extern int erts_call_time_breakpoint_tracing; -int erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, +int erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified, struct binary* match_prog_set, struct binary *meta_match_prog_set, int on, struct trace_pattern_flags, diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index 60c2349f36..93cfe08105 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -186,4 +186,11 @@ extern int erts_pd_initial_size;/* Initial Process dictionary table size */ #include "erl_term.h" +#ifdef NO_JUMP_TABLE +#define BeamOp(Op) (Op) +#else +extern void** beam_ops; +#define BeamOp(Op) beam_ops[(Op)] +#endif + #endif /* __ERL_VM_H__ */ diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h index 6c33b12dd0..e431c3051b 100644 --- a/erts/emulator/beam/error.h +++ b/erts/emulator/beam/error.h @@ -21,6 +21,8 @@ #ifndef __ERROR_H__ #define __ERROR_H__ +#include "code_ix.h" + /* * There are three primary exception classes: * @@ -197,7 +199,7 @@ struct StackTrace { Eterm header; /* bignum header - must be first in struct */ Eterm freason; /* original exception reason is saved in the struct */ BeamInstr* pc; - BeamInstr* current; + ErtsCodeMFA* current; int depth; /* number of saved pointers in trace[] */ BeamInstr *trace[1]; /* varying size - must be last in struct */ }; diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 2a19211987..0a36175398 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -103,7 +103,8 @@ static HashValue export_hash(struct export_entry* ee) { Export* x = ee->ep; - return EXPORT_HASH(x->code[0], x->code[1], x->code[2]); + return EXPORT_HASH(x->info.mfa.module, x->info.mfa.function, + x->info.mfa.arity); } static int @@ -111,9 +112,9 @@ export_cmp(struct export_entry* tmpl_e, struct export_entry* obj_e) { Export* tmpl = tmpl_e->ep; Export* obj = obj_e->ep; - return !(tmpl->code[0] == obj->code[0] && - tmpl->code[1] == obj->code[1] && - tmpl->code[2] == obj->code[2]); + return !(tmpl->info.mfa.module == obj->info.mfa.module && + tmpl->info.mfa.function == obj->info.mfa.function && + tmpl->info.mfa.arity == obj->info.mfa.arity); } @@ -130,16 +131,16 @@ export_alloc(struct export_entry* tmpl_e) blob = (struct export_blob*) erts_alloc(ERTS_ALC_T_EXPORT, sizeof(*blob)); erts_smp_atomic_add_nob(&total_entries_bytes, sizeof(*blob)); obj = &blob->exp; - obj->fake_op_func_info_for_hipe[0] = 0; - obj->fake_op_func_info_for_hipe[1] = 0; - obj->code[0] = tmpl->code[0]; - obj->code[1] = tmpl->code[1]; - obj->code[2] = tmpl->code[2]; - obj->code[3] = (BeamInstr) em_call_error_handler; - obj->code[4] = 0; + obj->info.op = 0; + obj->info.native = 0; + obj->info.mfa.module = tmpl->info.mfa.module; + obj->info.mfa.function = tmpl->info.mfa.function; + obj->info.mfa.arity = tmpl->info.mfa.arity; + obj->code[0] = (BeamInstr) em_call_error_handler; + obj->code[1] = 0; for (ix=0; ixaddressv[ix] = obj->code+3; + obj->addressv[ix] = obj->code; blob->entryv[ix].slot.index = -1; blob->entryv[ix].ep = &blob->exp; @@ -224,7 +225,9 @@ erts_find_export_entry(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix) while (b != (HashBucket*) 0) { Export* ep = ((struct export_entry*) b)->ep; - if (ep->code[0] == m && ep->code[1] == f && ep->code[2] == a) { + if (ep->info.mfa.module == m && + ep->info.mfa.function == f && + ep->info.mfa.arity == a) { return ep; } b = b->next; @@ -237,9 +240,9 @@ static struct export_entry* init_template(struct export_templ* templ, { templ->entry.ep = &templ->exp; templ->entry.slot.index = -1; - templ->exp.code[0] = m; - templ->exp.code[1] = f; - templ->exp.code[2] = a; + templ->exp.info.mfa.module = m; + templ->exp.info.mfa.function = f; + templ->exp.info.mfa.arity = a; return &templ->entry; } @@ -263,8 +266,8 @@ erts_find_function(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix) ee = hash_get(&export_tables[code_ix].htable, init_template(&templ, m, f, a)); if (ee == NULL || - (ee->ep->addressv[code_ix] == ee->ep->code+3 && - ee->ep->code[3] != (BeamInstr) BeamOp(op_i_generic_breakpoint))) { + (ee->ep->addressv[code_ix] == ee->ep->code && + ee->ep->code[0] != (BeamInstr) BeamOp(op_i_generic_breakpoint))) { return NULL; } return ee->ep; diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h index 1e7bb8514b..a807efef94 100644 --- a/erts/emulator/beam/export.h +++ b/erts/emulator/beam/export.h @@ -33,22 +33,20 @@ typedef struct export { void* addressv[ERTS_NUM_CODE_IX]; /* Pointer to code for function. */ - BeamInstr fake_op_func_info_for_hipe[2]; /* MUST be just before code[] */ + ErtsCodeInfo info; /* MUST be just before code[] */ + /* - * code[0]: Tagged atom for module. - * code[1]: Tagged atom for function. - * code[2]: Arity (untagged integer). - * code[3]: This entry is 0 unless the 'address' field points to it. + * code[0]: This entry is 0 unless the 'address' field points to it. * Threaded code instruction to load function * (em_call_error_handler), execute BIF (em_apply_bif), * or a breakpoint instruction (op_i_generic_breakpoint). - * code[4]: Function pointer to BIF function (for BIFs only), + * code[1]: Function pointer to BIF function (for BIFs only), * or pointer to threaded code if the module has an * on_load function that has not been run yet, or pointer - * to code for function code[3] is a breakpont instruction. + * to code if function code[0] is a breakpoint instruction. * Otherwise: 0. */ - BeamInstr code[5]; + BeamInstr code[2]; } Export; @@ -74,8 +72,8 @@ extern erts_smp_mtx_t export_staging_lock; #include "beam_load.h" /* For em_* extern declarations */ #define ExportIsBuiltIn(EntryPtr) \ -(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->code + 3) && \ - ((EntryPtr)->code[3] == (BeamInstr) em_apply_bif)) +(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->code) && \ + ((EntryPtr)->code[0] == (BeamInstr) em_apply_bif)) #if ERTS_GLB_INLINE_INCL_FUNC_DEF diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index beed847578..c297bb26b3 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2826,9 +2826,10 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Export* exp = *((Export **) (export_val(obj) + 1)); if ((dflags & DFLAG_EXPORT_PTR_TAG) != 0) { *ep++ = EXPORT_EXT; - ep = enc_atom(acmp, exp->code[0], ep, dflags); - ep = enc_atom(acmp, exp->code[1], ep, dflags); - ep = enc_term(acmp, make_small(exp->code[2]), ep, dflags, off_heap); + ep = enc_atom(acmp, exp->info.mfa.module, ep, dflags); + ep = enc_atom(acmp, exp->info.mfa.function, ep, dflags); + ep = enc_term(acmp, make_small(exp->info.mfa.arity), + ep, dflags, off_heap); } else { /* Tag, arity */ *ep++ = SMALL_TUPLE_EXT; @@ -2836,10 +2837,10 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, ep += 1; /* Module name */ - ep = enc_atom(acmp, exp->code[0], ep, dflags); + ep = enc_atom(acmp, exp->info.mfa.module, ep, dflags); /* Function name */ - ep = enc_atom(acmp, exp->code[1], ep, dflags); + ep = enc_atom(acmp, exp->info.mfa.function, ep, dflags); } break; } @@ -4262,9 +4263,9 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, { Export* ep = *((Export **) (export_val(obj) + 1)); result += 1; - result += encode_size_struct2(acmp, ep->code[0], dflags); - result += encode_size_struct2(acmp, ep->code[1], dflags); - result += encode_size_struct2(acmp, make_small(ep->code[2]), dflags); + result += encode_size_struct2(acmp, ep->info.mfa.module, dflags); + result += encode_size_struct2(acmp, ep->info.mfa.function, dflags); + result += encode_size_struct2(acmp, make_small(ep->info.mfa.arity), dflags); } break; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 93996e8b41..c9e41bcac7 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1024,7 +1024,7 @@ extern Process *erts_code_purger; /* beam_load.c */ typedef struct { - BeamInstr* current; /* Pointer to: Mod, Name, Arity */ + ErtsCodeInfo* ci; /* Pointer to: Mod, Name, Arity */ Uint needed; /* Heap space needed for entire tuple */ Uint32 loc; /* Location in source code */ Eterm* fname_ptr; /* Pointer to fname table */ @@ -1041,10 +1041,10 @@ Eterm erts_finish_loading(Binary* loader_state, Process* c_p, Eterm erts_preload_module(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm* mod, byte* code, Uint size); void init_load(void); -BeamInstr* find_function_from_pc(BeamInstr* pc); +ErtsCodeInfo* find_function_from_pc(BeamInstr* pc); Eterm* erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p); -void erts_set_current_function(FunctionInfo* fi, BeamInstr* current); +void erts_set_current_function(FunctionInfo* fi, ErtsCodeMFA* mfa); Eterm erts_module_info_0(Process* p, Eterm module); Eterm erts_module_info_1(Process* p, Eterm module, Eterm what); Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info); @@ -1577,8 +1577,7 @@ int erts_beam_jump_table(void); ERTS_GLB_INLINE void dtrace_pid_str(Eterm pid, char *process_buf); ERTS_GLB_INLINE void dtrace_proc_str(Process *process, char *process_buf); ERTS_GLB_INLINE void dtrace_port_str(Port *port, char *port_buf); -ERTS_GLB_INLINE void dtrace_fun_decode(Process *process, - Eterm module, Eterm function, int arity, +ERTS_GLB_INLINE void dtrace_fun_decode(Process *process, ErtsCodeMFA *mfa, char *process_buf, char *mfa_buf); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -1612,8 +1611,7 @@ dtrace_port_str(Port *port, char *port_buf) } ERTS_GLB_INLINE void -dtrace_fun_decode(Process *process, - Eterm module, Eterm function, int arity, +dtrace_fun_decode(Process *process, ErtsCodeMFA *mfa, char *process_buf, char *mfa_buf) { if (process_buf) { @@ -1621,7 +1619,7 @@ dtrace_fun_decode(Process *process, } erts_snprintf(mfa_buf, DTRACE_TERM_BUF_SIZE, "%T:%T/%d", - module, function, arity); + mfa->module, mfa->function, mfa->arity); } #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 6786657faf..cd7131f5df 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -893,11 +893,11 @@ tail_recur: { Export* ep = *((Export **) (export_val(term) + 1)); - hash = hash * FUNNY_NUMBER11 + ep->code[2]; + hash = hash * FUNNY_NUMBER11 + ep->info.mfa.arity; hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue); + (atom_tab(atom_val(ep->info.mfa.module))->slot.bucket.hvalue); hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(ep->code[1]))->slot.bucket.hvalue); + (atom_tab(atom_val(ep->info.mfa.function))->slot.bucket.hvalue); break; } @@ -1330,11 +1330,11 @@ make_hash2(Eterm term) { Export* ep = *((Export **) (export_val(term) + 1)); UINT32_HASH_2 - (ep->code[2], - atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue, + (ep->info.mfa.arity, + atom_tab(atom_val(ep->info.mfa.module))->slot.bucket.hvalue, HCONST); UINT32_HASH - (atom_tab(atom_val(ep->code[1]))->slot.bucket.hvalue, + (atom_tab(atom_val(ep->info.mfa.function))->slot.bucket.hvalue, HCONST_14); goto hash2_common; } @@ -2025,11 +2025,11 @@ tail_recur: { Export* ep = *((Export **) (export_val(term) + 1)); - hash = hash * FUNNY_NUMBER11 + ep->code[2]; + hash = hash * FUNNY_NUMBER11 + ep->info.mfa.arity; hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue); + (atom_tab(atom_val(ep->info.mfa.module))->slot.bucket.hvalue); hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(ep->code[1]))->slot.bucket.hvalue); + (atom_tab(atom_val(ep->info.mfa.function))->slot.bucket.hvalue); break; } @@ -3306,13 +3306,15 @@ tailrecur_ne: Export* a_exp = *((Export **) (export_val(a) + 1)); Export* b_exp = *((Export **) (export_val(b) + 1)); - if ((j = erts_cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) { + if ((j = erts_cmp_atoms(a_exp->info.mfa.module, + b_exp->info.mfa.module)) != 0) { RETURN_NEQ(j); } - if ((j = erts_cmp_atoms(a_exp->code[1], b_exp->code[1])) != 0) { + if ((j = erts_cmp_atoms(a_exp->info.mfa.function, + b_exp->info.mfa.function)) != 0) { RETURN_NEQ(j); } - ON_CMP_GOTO((Sint) a_exp->code[2] - (Sint) b_exp->code[2]); + ON_CMP_GOTO((Sint) a_exp->info.mfa.arity - (Sint) b_exp->info.mfa.arity); } break; case (_TAG_HEADER_FUN >> _TAG_PRIMARY_SIZE): -- cgit v1.2.3 From 8e2490086b45b9ce4d51883e594c38e2e17b5b47 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Wed, 5 Oct 2016 12:16:35 +0200 Subject: erts: Refactor find_function_from_pc to return MFA --- erts/emulator/beam/beam_debug.c | 52 +++++++++++++++++------------------ erts/emulator/beam/beam_emu.c | 8 +++--- erts/emulator/beam/beam_load.c | 14 +++++----- erts/emulator/beam/beam_ranges.c | 6 ++-- erts/emulator/beam/erl_bif_info.c | 12 ++++---- erts/emulator/beam/erl_db_util.c | 8 +++--- erts/emulator/beam/erl_nif.c | 4 +-- erts/emulator/beam/erl_process.c | 8 +++--- erts/emulator/beam/erl_process_dump.c | 8 +++--- erts/emulator/beam/erl_trace.c | 36 ++++++++++++------------ erts/emulator/beam/erl_trace.h | 4 +-- erts/emulator/beam/global.h | 4 +-- 12 files changed, 81 insertions(+), 83 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 8d4be4667b..c855c734d6 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -243,7 +243,7 @@ erts_debug_disassemble_1(BIF_ALIST_1) Eterm* tp; Eterm bin; Eterm mfa; - ErtsCodeInfo *ci = NULL; + ErtsCodeMFA *cmfa = NULL; BeamCodeHeader* code_hdr; BeamInstr *code_ptr; BeamInstr instr; @@ -253,7 +253,7 @@ erts_debug_disassemble_1(BIF_ALIST_1) if (term_to_UWord(addr, &uaddr)) { BeamInstr *pc = (BeamInstr *) uaddr; - if ((ci = find_function_from_pc(pc)) == NULL) { + if ((cmfa = find_function_from_pc(pc)) == NULL) { BIF_RET(am_false); } } else if (is_tuple(addr)) { @@ -292,14 +292,14 @@ erts_debug_disassemble_1(BIF_ALIST_1) * But this code_ptr will point to the start of the Export, * not the function's func_info instruction. BOOM !? */ - ci = erts_code_to_codeinfo(ep->addressv[code_ix]); + cmfa = erts_code_to_codemfa(ep->addressv[code_ix]); } else if (modp == NULL || (code_hdr = modp->curr.code_hdr) == NULL) { BIF_RET(am_undef); } else { n = code_hdr->num_functions; for (i = 0; i < n; i++) { - ci = code_hdr->functions[i]; - if (ci->mfa.function == name && ci->mfa.arity == arity) { + cmfa = &code_hdr->functions[i]->mfa; + if (cmfa->function == name && cmfa->arity == arity) { break; } } @@ -312,7 +312,7 @@ erts_debug_disassemble_1(BIF_ALIST_1) } - code_ptr = erts_codeinfo_to_code(ci); + code_ptr = erts_codemfa_to_code(cmfa); dsbufp = erts_create_tmp_dsbuf(0); erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr); instr = (BeamInstr) code_ptr[0]; @@ -334,10 +334,10 @@ erts_debug_disassemble_1(BIF_ALIST_1) (void) erts_bld_uword(NULL, &hsz, (BeamInstr) code_ptr); hp = HAlloc(p, hsz); addr = erts_bld_uword(&hp, NULL, (BeamInstr) code_ptr); - ASSERT(is_atom(ci->mfa.module) || is_nil(ci->mfa.module)); - ASSERT(is_atom(ci->mfa.function) || is_nil(ci->mfa.function == NIL)); - mfa = TUPLE3(hp, ci->mfa.module, ci->mfa.function, - make_small(ci->mfa.arity)); + ASSERT(is_atom(cmfa->module) || is_nil(cmfa->module)); + ASSERT(is_atom(cmfa->function) || is_nil(cmfa->function == NIL)); + mfa = TUPLE3(hp, cmfa->module, cmfa->function, + make_small(cmfa->arity)); hp += 4; return TUPLE3(hp, addr, bin, mfa); } @@ -349,12 +349,12 @@ dbg_bt(Process* p, Eterm* sp) while (sp < stack) { if (is_CP(*sp)) { - ErtsCodeInfo* ci = find_function_from_pc(cp_val(*sp)); - if (ci) + ErtsCodeMFA* cmfa = find_function_from_pc(cp_val(*sp)); + if (cmfa) erts_fprintf(stderr, HEXF ": %T:%T/%bpu\n", - &ci->mfa.module, ci->mfa.module, - ci->mfa.function, ci->mfa.arity); + &cmfa->module, cmfa->module, + cmfa->function, cmfa->arity); } sp++; } @@ -363,17 +363,17 @@ dbg_bt(Process* p, Eterm* sp) void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg) { - ErtsCodeInfo* ci = find_function_from_pc(addr); + ErtsCodeMFA* cmfa = find_function_from_pc(addr); - if (ci == NULL) { + if (cmfa == NULL) { erts_fprintf(stderr, "???\n"); } else { int arity; int i; - arity = ci->mfa.arity; + arity = cmfa->arity; erts_fprintf(stderr, HEXF ": %T:%T(", addr, - ci->mfa.module, ci->mfa.function); + cmfa->module, cmfa->function); for (i = 0; i < arity; i++) erts_fprintf(stderr, i ? ", %T" : "%T", i ? reg[i] : x0); erts_fprintf(stderr, ")\n"); @@ -549,24 +549,24 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) break; case 'f': /* Destination label */ { - ErtsCodeInfo* ci = find_function_from_pc((BeamInstr *)*ap); - if (erts_codeinfo_to_code(ci) != (BeamInstr *) *ap) { + ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap); + if (erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) { erts_print(to, to_arg, "f(" HEXF ")", *ap); } else { - erts_print(to, to_arg, "%T:%T/%bpu", ci->mfa.module, - ci->mfa.function, ci->mfa.arity); + erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module, + cmfa->function, cmfa->arity); } ap++; } break; case 'p': /* Pointer (to label) */ { - ErtsCodeInfo* ci = find_function_from_pc((BeamInstr *)*ap); - if (erts_codeinfo_to_code(ci) != (BeamInstr *) *ap) { + ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap); + if (erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) { erts_print(to, to_arg, "p(" HEXF ")", *ap); } else { - erts_print(to, to_arg, "%T:%T/%bpu", ci->mfa.module, - ci->mfa.function, ci->mfa.arity); + erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module, + cmfa->function, cmfa->arity); } ap++; } diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 4ff177f215..2c47386925 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1317,8 +1317,8 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) if (start_time != 0) { Sint64 diff = erts_timestamp_millis() - start_time; if (diff > 0 && (Uint) diff > erts_system_monitor_long_schedule) { - ErtsCodeInfo *inptr = find_function_from_pc(start_time_i); - ErtsCodeInfo *outptr = find_function_from_pc(c_p->i); + ErtsCodeMFA *inptr = find_function_from_pc(start_time_i); + ErtsCodeMFA *outptr = find_function_from_pc(c_p->i); monitor_long_schedule_proc(c_p,inptr,outptr,(Uint) diff); } } @@ -6074,7 +6074,7 @@ build_stacktrace(Process* c_p, Eterm exc) { * If fi.current is still NULL, default to the initial function * (e.g. spawn_link(erlang, abs, [1])). */ - if (fi.ci == NULL) { + if (fi.mfa == NULL) { erts_set_current_function(&fi, &c_p->u.initial); args = am_true; /* Just in case */ } else { @@ -6091,7 +6091,7 @@ build_stacktrace(Process* c_p, Eterm exc) { heap_size = fi.needed + 2; for (i = 0; i < depth; i++) { erts_lookup_function_info(stkp, s->trace[i], 1); - if (stkp->ci) { + if (stkp->mfa) { heap_size += stkp->needed + 2; stkp++; } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index c822cd8606..76a92d41f7 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -5818,7 +5818,7 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) Eterm file_term = NIL; if (file == 0) { - Atom* ap = atom_tab(atom_val(fi->ci->mfa.module)); + Atom* ap = atom_tab(atom_val(fi->mfa->module)); file_term = buf_to_intlist(&hp, ".erl", 4, NIL); file_term = buf_to_intlist(&hp, (char*)ap->name, ap->len, file_term); } else { @@ -5837,11 +5837,11 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) } if (is_list(args) || is_nil(args)) { - *mfa_p = TUPLE4(hp, fi->ci->mfa.module, fi->ci->mfa.function, + *mfa_p = TUPLE4(hp, fi->mfa->module, fi->mfa->function, args, loc); } else { - Eterm arity = make_small(fi->ci->mfa.arity); - *mfa_p = TUPLE4(hp, fi->ci->mfa.module, fi->ci->mfa.function, + Eterm arity = make_small(fi->mfa->arity); + *mfa_p = TUPLE4(hp, fi->mfa->module, fi->mfa->function, arity, loc); } return hp + 5; @@ -5855,7 +5855,7 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) void erts_set_current_function(FunctionInfo* fi, ErtsCodeMFA* mfa) { - fi->ci = erts_code_to_codeinfo(erts_codemfa_to_code(mfa)); + fi->mfa = mfa; fi->needed = 5; fi->loc = LINE_INVALID_LOCATION; } @@ -5864,13 +5864,13 @@ erts_set_current_function(FunctionInfo* fi, ErtsCodeMFA* mfa) /* * Returns a pointer to {module, function, arity}, or NULL if not found. */ -ErtsCodeInfo* +ErtsCodeMFA* find_function_from_pc(BeamInstr* pc) { FunctionInfo fi; erts_lookup_function_info(&fi, pc, 0); - return fi.ci; + return fi.mfa; } /* diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c index 887f80e7e3..9b0335e83d 100644 --- a/erts/emulator/beam/beam_ranges.c +++ b/erts/emulator/beam/beam_ranges.c @@ -227,7 +227,7 @@ erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) Range* rp; BeamCodeHeader* hdr; - fi->ci = NULL; + fi->mfa = NULL; fi->needed = 5; fi->loc = LINE_INVALID_LOCATION; rp = find_range(pc); @@ -243,7 +243,7 @@ erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) if (pc < (BeamInstr*)(mid[0])) { high = mid; } else if (pc < (BeamInstr*)(mid[1])) { - fi->ci = mid[0]; + fi->mfa = &mid[0]->mfa; if (full_info) { ErtsCodeInfo** fp = hdr->functions; int idx = mid - fp; @@ -316,7 +316,7 @@ lookup_loc(FunctionInfo* fi, const BeamInstr* pc, file = LOC_FILE(fi->loc); if (file == 0) { /* Special case: Module name with ".erl" appended */ - Atom* mod_atom = atom_tab(atom_val(fi->ci->mfa.module)); + Atom* mod_atom = atom_tab(atom_val(fi->mfa->module)); fi->needed += 2*(mod_atom->len+4); } else { Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1])); diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index a985ce4918..dfc4beb719 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1614,12 +1614,10 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info) if (rp->current == NULL) { erts_lookup_function_info(&fi, rp->i, full_info); - rp->current = &fi.ci->mfa; - ASSERT_MFA(rp->current); + rp->current = fi.mfa; } else if (full_info) { - ASSERT_MFA(rp->current); erts_lookup_function_info(&fi, rp->i, full_info); - if (fi.ci == NULL) { + if (fi.mfa == NULL) { /* Use the current function without location info */ erts_set_current_function(&fi, rp->current); } @@ -1635,9 +1633,9 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info) * instead if it can be looked up. */ erts_lookup_function_info(&fi2, rp->cp, full_info); - if (fi2.ci) { + if (fi2.mfa) { fi = fi2; - rp->current = &fi2.ci->mfa; + rp->current = fi2.mfa; } } @@ -1695,7 +1693,7 @@ current_stacktrace(Process* p, Process* rp, Eterm** hpp) heap_size = 3; for (i = 0; i < depth; i++) { erts_lookup_function_info(stkp, s->trace[i], 1); - if (stkp->ci) { + if (stkp->mfa) { heap_size += stkp->needed + 2; stkp++; } diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 988467ce44..4e987d5bee 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1769,7 +1769,7 @@ Eterm db_prog_match(Process *c_p, Eterm t; Eterm *esp; MatchVariable* variables; - ErtsCodeInfo *cp; + ErtsCodeMFA *cp; const UWord *pc = prog->text; Eterm *ehp; Eterm ret; @@ -2408,9 +2408,9 @@ restart: ehp = HAllocX(build_proc, 4, HEAP_XTRA); *esp++ = make_tuple(ehp); ehp[0] = make_arityval(3); - ehp[1] = cp->mfa.module; - ehp[2] = cp->mfa.function; - ehp[3] = make_small((Uint) cp->mfa.arity); + ehp[1] = cp->module; + ehp[2] = cp->function; + ehp[3] = make_small((Uint) cp->arity); } break; case matchSilent: diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index f0535451fc..167cded28f 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -3158,7 +3158,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) Eterm mod_atom; const Atom* mod_atomp; Eterm f_atom; - ErtsCodeInfo* caller; + ErtsCodeMFA* caller; ErtsSysDdllError errdesc = ERTS_SYS_DDLL_ERROR_INIT; Eterm ret = am_ok; int veto; @@ -3196,7 +3196,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) && BIF_P->current->arity == 2); caller = find_function_from_pc(BIF_P->cp); ASSERT(caller != NULL); - mod_atom = caller->mfa.module; + mod_atom = caller->module; ASSERT(is_atom(mod_atom)); module_p = erts_get_module(mod_atom, erts_active_code_ix()); ASSERT(module_p != NULL); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index f7f391e322..d0a0310544 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -13220,8 +13220,8 @@ erts_program_counter_info(int to, void *to_arg, Process *p) static void print_function_from_pc(int to, void *to_arg, BeamInstr* x) { - ErtsCodeInfo *ci = find_function_from_pc(x); - if (ci == NULL) { + ErtsCodeMFA *cmfa = find_function_from_pc(x); + if (cmfa == NULL) { if (x == beam_exit) { erts_print(to, to_arg, ""); } else if (x == beam_continue_exit) { @@ -13235,8 +13235,8 @@ print_function_from_pc(int to, void *to_arg, BeamInstr* x) } } else { erts_print(to, to_arg, "%T:%T/%d + %d", - ci->mfa.module, ci->mfa.function, ci->mfa.arity, - (x-(BeamInstr*)ci) * sizeof(Eterm)); + cmfa->module, cmfa->function, cmfa->arity, + (x-(BeamInstr*)cmfa) * sizeof(Eterm)); } } diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 645bedd8f1..e7a311b430 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -334,8 +334,8 @@ stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg) static void print_function_from_pc(int to, void *to_arg, BeamInstr* x) { - ErtsCodeInfo* ci = find_function_from_pc(x); - if (ci == NULL) { + ErtsCodeMFA* cmfa = find_function_from_pc(x); + if (cmfa == NULL) { if (x == beam_exit) { erts_print(to, to_arg, ""); } else if (x == beam_continue_exit) { @@ -347,8 +347,8 @@ print_function_from_pc(int to, void *to_arg, BeamInstr* x) } } else { erts_print(to, to_arg, "%T:%T/%bpu + %bpu", - ci->mfa.module, ci->mfa.function, ci->mfa.arity, - (x-(BeamInstr*)ci) * sizeof(Eterm)); + cmfa->module, cmfa->function, cmfa->arity, + (x-(BeamInstr*)cmfa) * sizeof(Eterm)); } } diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 566529d2d4..9be4741ec8 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -773,7 +773,7 @@ trace_sched_aux(Process *p, ErtsProcLocks locks, Eterm what) curr_func = 0; else { if (!p->current) - p->current = &find_function_from_pc(p->i)->mfa; + p->current = find_function_from_pc(p->i); curr_func = p->current != NULL; } @@ -1028,14 +1028,14 @@ erts_trace_return_to(Process *p, BeamInstr *pc) { Eterm mfa; - ErtsCodeInfo *ci = find_function_from_pc(pc); + ErtsCodeMFA *cmfa = find_function_from_pc(pc); - if (!ci) { + if (!cmfa) { mfa = am_undefined; } else { Eterm *hp = HAlloc(p, 4); - mfa = TUPLE3(hp, ci->mfa.module, ci->mfa.function, - make_small(ci->mfa.arity)); + mfa = TUPLE3(hp, cmfa->module, cmfa->function, + make_small(cmfa->arity)); } send_to_tracer_nif(p, &p->common, p->common.id, NULL, TRACE_FUN_T_CALL, @@ -1458,8 +1458,8 @@ trace_gc(Process *p, Eterm what, Uint size, Eterm msg) } void -monitor_long_schedule_proc(Process *p, ErtsCodeInfo *in_fp, - ErtsCodeInfo *out_fp, Uint time) +monitor_long_schedule_proc(Process *p, ErtsCodeMFA *in_fp, + ErtsCodeMFA *out_fp, Uint time) { ErlHeapFragment *bp; ErlOffHeap *off_heap; @@ -1490,13 +1490,13 @@ monitor_long_schedule_proc(Process *p, ErtsCodeInfo *in_fp, hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, monitor_p); tmo = erts_bld_uint(&hp, NULL, time); if (in_fp != NULL) { - in_mfa = TUPLE3(hp, in_fp->mfa.module, in_fp->mfa.function, - make_small(in_fp->mfa.arity)); + in_mfa = TUPLE3(hp, in_fp->module, in_fp->function, + make_small(in_fp->arity)); hp +=4; } if (out_fp != NULL) { - out_mfa = TUPLE3(hp, out_fp->mfa.module, out_fp->mfa.function, - make_small(out_fp->mfa.arity)); + out_mfa = TUPLE3(hp, out_fp->module, out_fp->function, + make_small(out_fp->arity)); hp +=4; } tmo_tpl = TUPLE2(hp,am_timeout, tmo); @@ -2131,7 +2131,7 @@ profile_runnable_proc(Process *p, Eterm status){ Eterm *hp, msg; Eterm where = am_undefined; ErlHeapFragment *bp = NULL; - ErtsCodeInfo *ci = NULL; + ErtsCodeMFA *cmfa = NULL; #ifndef ERTS_SMP #define LOCAL_HEAP_SIZE (4 + 6 + ERTS_TRACE_PATCH_TS_MAX_SIZE) @@ -2153,14 +2153,14 @@ profile_runnable_proc(Process *p, Eterm status){ if (!ERTS_PROC_IS_EXITING(p)) { if (p->current) { - ci = erts_code_to_codeinfo(erts_codemfa_to_code(p->current)); + cmfa = p->current; } else { - ci = find_function_from_pc(p->i); + cmfa = find_function_from_pc(p->i); } } #ifdef ERTS_SMP - if (!ci) { + if (!cmfa) { hsz -= 4; } @@ -2168,9 +2168,9 @@ profile_runnable_proc(Process *p, Eterm status){ hp = bp->mem; #endif - if (ci) { - where = TUPLE3(hp, ci->mfa.module, ci->mfa.function, - make_small(ci->mfa.arity)); + if (cmfa) { + where = TUPLE3(hp, cmfa->module, cmfa->function, + make_small(cmfa->arity)); hp += 4; } else { where = make_small(0); diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h index 74df3e50e1..378b6de49c 100644 --- a/erts/emulator/beam/erl_trace.h +++ b/erts/emulator/beam/erl_trace.h @@ -134,8 +134,8 @@ void erts_system_profile_setup_active_schedulers(void); /* system_monitor */ void monitor_long_gc(Process *p, Uint time); -void monitor_long_schedule_proc(Process *p, ErtsCodeInfo *in_i, - ErtsCodeInfo *out_i, Uint time); +void monitor_long_schedule_proc(Process *p, ErtsCodeMFA *in_i, + ErtsCodeMFA *out_i, Uint time); void monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time); void monitor_large_heap(Process *p); void monitor_generic(Process *p, Eterm type, Eterm spec); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index c9e41bcac7..7ee046ac12 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1024,7 +1024,7 @@ extern Process *erts_code_purger; /* beam_load.c */ typedef struct { - ErtsCodeInfo* ci; /* Pointer to: Mod, Name, Arity */ + ErtsCodeMFA* mfa; /* Pointer to: Mod, Name, Arity */ Uint needed; /* Heap space needed for entire tuple */ Uint32 loc; /* Location in source code */ Eterm* fname_ptr; /* Pointer to fname table */ @@ -1041,7 +1041,7 @@ Eterm erts_finish_loading(Binary* loader_state, Process* c_p, Eterm erts_preload_module(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm* mod, byte* code, Uint size); void init_load(void); -ErtsCodeInfo* find_function_from_pc(BeamInstr* pc); +ErtsCodeMFA* find_function_from_pc(BeamInstr* pc); Eterm* erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p); void erts_set_current_function(FunctionInfo* fi, ErtsCodeMFA* mfa); -- cgit v1.2.3 From 81e6a32de18bd3d8ba4b8801631e5c46892a5a18 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Thu, 6 Oct 2016 10:50:50 +0200 Subject: erts: Fix erts_debug:df with new func_info --- erts/emulator/beam/beam_debug.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index c855c734d6..68559e3992 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -252,8 +252,8 @@ erts_debug_disassemble_1(BIF_ALIST_1) int i; if (term_to_UWord(addr, &uaddr)) { - BeamInstr *pc = (BeamInstr *) uaddr; - if ((cmfa = find_function_from_pc(pc)) == NULL) { + code_ptr = (BeamInstr *) uaddr; + if ((cmfa = find_function_from_pc(code_ptr)) == NULL) { BIF_RET(am_false); } } else if (is_tuple(addr)) { @@ -307,12 +307,11 @@ erts_debug_disassemble_1(BIF_ALIST_1) BIF_RET(am_undef); } } + code_ptr = erts_codemfa_to_code(cmfa); } else { goto error; } - - code_ptr = erts_codemfa_to_code(cmfa); dsbufp = erts_create_tmp_dsbuf(0); erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr); instr = (BeamInstr) code_ptr[0]; @@ -335,7 +334,7 @@ erts_debug_disassemble_1(BIF_ALIST_1) hp = HAlloc(p, hsz); addr = erts_bld_uword(&hp, NULL, (BeamInstr) code_ptr); ASSERT(is_atom(cmfa->module) || is_nil(cmfa->module)); - ASSERT(is_atom(cmfa->function) || is_nil(cmfa->function == NIL)); + ASSERT(is_atom(cmfa->function) || is_nil(cmfa->function)); mfa = TUPLE3(hp, cmfa->module, cmfa->function, make_small(cmfa->arity)); hp += 4; @@ -550,7 +549,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) case 'f': /* Destination label */ { ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap); - if (erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) { + if (!cmfa || erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) { erts_print(to, to_arg, "f(" HEXF ")", *ap); } else { erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module, @@ -562,7 +561,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) case 'p': /* Pointer (to label) */ { ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap); - if (erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) { + if (!cmfa || erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) { erts_print(to, to_arg, "p(" HEXF ")", *ap); } else { erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module, -- cgit v1.2.3 From ab3ea86a2ad0015eb8a5a46e01a1ad4ad51da4e1 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 10 Oct 2016 17:36:42 +0200 Subject: erts: Refactor rename Export.code[] to Export.beam[] to avoid scary merge errors. --- erts/emulator/beam/beam_bif_load.c | 32 ++++++++++++++++---------------- erts/emulator/beam/beam_bp.c | 4 ++-- erts/emulator/beam/beam_emu.c | 10 +++++----- erts/emulator/beam/beam_load.c | 26 +++++++++++++------------- erts/emulator/beam/bif.c | 6 +++--- erts/emulator/beam/erl_bif_trace.c | 36 ++++++++++++++++++------------------ erts/emulator/beam/erl_nif.c | 6 +++--- erts/emulator/beam/export.c | 10 +++++----- erts/emulator/beam/export.h | 14 +++++++------- 9 files changed, 72 insertions(+), 72 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index f584b486ed..bce57fb7c1 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -795,16 +795,16 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { continue; } - if (ep->code[1] != 0) { - ep->addressv[code_ix] = (void *) ep->code[1]; - ep->code[1] = 0; + if (ep->beam[1] != 0) { + ep->addressv[code_ix] = (void *) ep->beam[1]; + ep->beam[1] = 0; } else { - if (ep->addressv[code_ix] == ep->code && - ep->code[0] == (BeamInstr) em_apply_bif) { + if (ep->addressv[code_ix] == ep->beam && + ep->beam[0] == (BeamInstr) em_apply_bif) { continue; } - ep->addressv[code_ix] = ep->code; - ep->code[0] = (BeamInstr) em_call_error_handler; + ep->addressv[code_ix] = ep->beam; + ep->beam[0] = (BeamInstr) em_call_error_handler; } } modp->curr.code_hdr->on_load_function_ptr = NULL; @@ -822,10 +822,10 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { continue; } - if (ep->code[0] == (BeamInstr) em_apply_bif) { + if (ep->beam[0] == (BeamInstr) em_apply_bif) { continue; } - ep->code[1] = 0; + ep->beam[1] = 0; } } erts_smp_thr_progress_unblock(); @@ -1711,22 +1711,22 @@ delete_code(Module* modp) for (i = 0; i < export_list_size(code_ix); i++) { Export *ep = export_list(i, code_ix); if (ep != NULL && (ep->info.mfa.module == module)) { - if (ep->addressv[code_ix] == ep->code) { - if (ep->code[0] == (BeamInstr) em_apply_bif) { + if (ep->addressv[code_ix] == ep->beam) { + if (ep->beam[0] == (BeamInstr) em_apply_bif) { continue; } - else if (ep->code[0] == + else if (ep->beam[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(modp->curr.num_traced_exports > 0); erts_clear_export_break(modp, &ep->info); } - else ASSERT(ep->code[0] == (BeamInstr) em_call_error_handler + else ASSERT(ep->beam[0] == (BeamInstr) em_call_error_handler || !erts_initialized); } - ep->addressv[code_ix] = ep->code; - ep->code[0] = (BeamInstr) em_call_error_handler; - ep->code[1] = 0; + ep->addressv[code_ix] = ep->beam; + ep->beam[0] = (BeamInstr) em_call_error_handler; + ep->beam[1] = 0; } } diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 2cdf66f441..62464f3864 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -238,7 +238,7 @@ erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified) ASSERT(0); } - pc = ep->code; + pc = ep->beam; if (ep->addressv[code_ix] == pc) { if ((*pc == (BeamInstr) em_apply_bif || *pc == (BeamInstr) em_call_error_handler)) { @@ -712,7 +712,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) Export* ep = bif_export[bif_index]; Uint32 flags = 0, flags_meta = 0; ErtsTracer meta_tracer = erts_tracer_nil; - int applying = (I == ep->code); /* Yup, the apply code for a bif + int applying = (I == ep->beam); /* Yup, the apply code for a bif * is actually in the * export entry */ BeamInstr *cp = p->cp; diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 2c47386925..8e88d6ccd5 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -119,7 +119,7 @@ do { \ #define GET_BIF_MODULE(p) (p->info.mfa.module) #define GET_BIF_FUNCTION(p) (p->info.mfa.function) #define GET_BIF_ARITY(p) (p->info.mfa.arity) -#define GET_BIF_ADDRESS(p) ((BifFunction) (p->code[1])) +#define GET_BIF_ADDRESS(p) ((BifFunction) (p->beam[1])) #define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm)))) @@ -5147,8 +5147,8 @@ do { \ bif_table[i].name, bif_table[i].arity); bif_export[i] = ep; - ep->code[0] = (BeamInstr) OpCode(apply_bif); - ep->code[1] = (BeamInstr) bif_table[i].f; + ep->beam[0] = (BeamInstr) OpCode(apply_bif); + ep->beam[1] = (BeamInstr) bif_table[i].f; /* XXX: set func info for bifs */ ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI); } @@ -7219,8 +7219,8 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity) if ((ep = export_get(&e)) == NULL) { return 0; } - return ep->addressv[erts_active_code_ix()] == ep->code - && (ep->code[0] == (BeamInstr) em_apply_bif); + return ep->addressv[erts_active_code_ix()] == ep->beam + && (ep->beam[0] == (BeamInstr) em_apply_bif); } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 76a92d41f7..13552ae88c 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -809,18 +809,18 @@ erts_finish_loading(Binary* magic, Process* c_p, if (ep == NULL || ep->info.mfa.module != module) { continue; } - if (ep->addressv[code_ix] == ep->code) { - if (ep->code[0] == (BeamInstr) em_apply_bif) { + if (ep->addressv[code_ix] == ep->beam) { + if (ep->beam[0] == (BeamInstr) em_apply_bif) { continue; - } else if (ep->code[0] == + } else if (ep->beam[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(mod_tab_p->curr.num_traced_exports > 0); erts_clear_export_break(mod_tab_p, &ep->info); - ep->addressv[code_ix] = (BeamInstr *) ep->code[1]; - ep->code[1] = 0; + ep->addressv[code_ix] = (BeamInstr *) ep->beam[1]; + ep->beam[1] = 0; } - ASSERT(ep->code[1] == 0); + ASSERT(ep->beam[1] == 0); } } ASSERT(mod_tab_p->curr.num_breakpoints == 0); @@ -1420,8 +1420,8 @@ load_import_table(LoaderState* stp) * the BIF function. */ if ((e = erts_active_export_entry(mod, func, arity)) != NULL) { - if (e->code[0] == (BeamInstr) em_apply_bif) { - stp->import[i].bf = (BifFunction) e->code[1]; + if (e->beam[0] == (BeamInstr) em_apply_bif) { + stp->import[i].bf = (BifFunction) e->beam[1]; if (func == am_load_nif && mod == am_erlang && arity == 2) { stp->may_load_nif = 1; } @@ -1514,7 +1514,7 @@ is_bif(Eterm mod, Eterm func, unsigned arity) if (e == NULL) { return 0; } - if (e->code[0] != (BeamInstr) em_apply_bif) { + if (e->beam[0] != (BeamInstr) em_apply_bif) { return 0; } if (mod == am_erlang && func == am_apply && arity == 3) { @@ -4784,7 +4784,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) * callable yet. Keep any function in the current * code callable. */ - ep->code[1] = (BeamInstr) address; + ep->beam[1] = (BeamInstr) address; } } @@ -4962,7 +4962,7 @@ transform_engine(LoaderState* st) if (i >= st->num_imports || st->import[i].bf == NULL) goto restart; if (bif_number != -1 && - bif_export[bif_number]->code[1] != (BeamInstr) st->import[i].bf) { + bif_export[bif_number]->beam[1] != (BeamInstr) st->import[i].bf) { goto restart; } } @@ -5723,8 +5723,8 @@ exported_from_module(Process* p, /* Process whose heap to use. */ if (ep->info.mfa.module == mod) { Eterm tuple; - if (ep->addressv[code_ix] == ep->code && - ep->code[0] == (BeamInstr) em_call_error_handler) { + if (ep->addressv[code_ix] == ep->beam && + ep->beam[0] == (BeamInstr) em_call_error_handler) { /* There is a call to the function, but it does not exist. */ continue; } diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 324f7d4268..d886c2985e 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -4963,13 +4963,13 @@ void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, int i; sys_memset((void *) ep, 0, sizeof(Export)); for (i=0; iaddressv[i] = ep->code; + ep->addressv[i] = ep->beam; } ep->info.mfa.module = m; ep->info.mfa.function = f; ep->info.mfa.arity = a; - ep->code[0] = (BeamInstr) em_apply_bif; - ep->code[1] = (BeamInstr) bif; + ep->beam[0] = (BeamInstr) em_apply_bif; + ep->beam[1] = (BeamInstr) bif; } void erts_init_bif(void) diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index e3355208e5..1a20cc911e 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -980,7 +980,7 @@ static int function_is_traced(Process *p, e.info.mfa.function = mfa[1]; e.info.mfa.arity = mfa[2]; if ((ep = export_get(&e)) != NULL) { - pc = ep->code; + pc = ep->beam; if (ep->addressv[erts_active_code_ix()] == pc && *pc != (BeamInstr) em_call_error_handler) { @@ -1380,12 +1380,12 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified, #ifdef DEBUG ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI); #endif - ep->code[0] = (BeamInstr) BeamOp(op_jump_f); - ep->code[1] = (BeamInstr) ep->addressv[code_ix]; + ep->beam[0] = (BeamInstr) BeamOp(op_jump_f); + ep->beam[1] = (BeamInstr) ep->addressv[code_ix]; } erts_set_call_trace_bif(ci, match_prog_set, 0); if (ep->addressv[code_ix] != pc) { - ep->code[0] = (BeamInstr) BeamOp(op_i_generic_breakpoint); + ep->beam[0] = (BeamInstr) BeamOp(op_i_generic_breakpoint); } } else if (!on && flags.breakpoint) { /* Turn off breakpoint tracing -- nothing to do here. */ @@ -1395,8 +1395,8 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified, * before turning on breakpoint tracing. */ erts_clear_call_trace_bif(ci, 0); - if (ep->code[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { - ep->code[0] = (BeamInstr) BeamOp(op_jump_f); + if (ep->beam[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { + ep->beam[0] = (BeamInstr) BeamOp(op_jump_f); } } } @@ -1679,7 +1679,7 @@ install_exp_breakpoints(BpFunctions* f) for (i = 0; i < ne; i++) { Export* ep = ErtsContainerStruct(fp[i].ci, Export, info); - ep->addressv[code_ix] = ep->code; + ep->addressv[code_ix] = ep->beam; } } @@ -1694,11 +1694,11 @@ uninstall_exp_breakpoints(BpFunctions* f) for (i = 0; i < ne; i++) { Export* ep = ErtsContainerStruct(fp[i].ci, Export, info); - if (ep->addressv[code_ix] != ep->code) { + if (ep->addressv[code_ix] != ep->beam) { continue; } - ASSERT(ep->code[0] == (BeamInstr) BeamOp(op_jump_f)); - ep->addressv[code_ix] = (BeamInstr *) ep->code[1]; + ASSERT(ep->beam[0] == (BeamInstr) BeamOp(op_jump_f)); + ep->addressv[code_ix] = (BeamInstr *) ep->beam[1]; } } @@ -1713,12 +1713,12 @@ clean_export_entries(BpFunctions* f) for (i = 0; i < ne; i++) { Export* ep = ErtsContainerStruct(fp[i].ci, Export, info); - if (ep->addressv[code_ix] == ep->code) { + if (ep->addressv[code_ix] == ep->beam) { continue; } - if (ep->code[0] == (BeamInstr) BeamOp(op_jump_f)) { - ep->code[0] = (BeamInstr) 0; - ep->code[1] = (BeamInstr) 0; + if (ep->beam[0] == (BeamInstr) BeamOp(op_jump_f)) { + ep->beam[0] = (BeamInstr) 0; + ep->beam[1] = (BeamInstr) 0; } } } @@ -1733,8 +1733,8 @@ setup_bif_trace(void) GenericBp* g = (GenericBp *) ep->info.native; if (g) { if (ExportIsBuiltIn(ep)) { - ASSERT(ep->code[1]); - ep->code[1] = (BeamInstr) bif_table[i].traced; + ASSERT(ep->beam[1]); + ep->beam[1] = (BeamInstr) bif_table[i].traced; } } } @@ -1751,8 +1751,8 @@ reset_bif_trace(void) GenericBp* g = (GenericBp *) ep->info.native; if (g && g->data[active].flags == 0) { if (ExportIsBuiltIn(ep)) { - ASSERT(ep->code[1]); - ep->code[1] = (BeamInstr) bif_table[i].f; + ASSERT(ep->beam[1]); + ep->beam[1] = (BeamInstr) bif_table[i].f; } } } diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 167cded28f..d2f18667bf 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2328,9 +2328,9 @@ allocate_nif_sched_data(Process* proc, int argc) ep->rootset_extra = argc; ep->rootset[0] = NIL; for (i=0; iexp.addressv[i] = &ep->exp.code[0]; + ep->exp.addressv[i] = &ep->exp.beam[0]; } - ep->exp.code[0] = (BeamInstr) em_call_nif; + ep->exp.beam[0] = (BeamInstr) em_call_nif; (void) ERTS_PROC_SET_NIF_TRAP_EXPORT(proc, ep); return ep; } @@ -2404,7 +2404,7 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec ep->exp.info.mfa.module = proc->current->module; ep->exp.info.mfa.function = proc->current->function; ep->exp.info.mfa.arity = argc; - ep->exp.code[1] = (BeamInstr) direct_fp; + ep->exp.beam[1] = (BeamInstr) direct_fp; ep->m = env->mod_nif; ep->fp = indirect_fp; proc->freason = TRAP; diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 0a36175398..060fde074c 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -136,11 +136,11 @@ export_alloc(struct export_entry* tmpl_e) obj->info.mfa.module = tmpl->info.mfa.module; obj->info.mfa.function = tmpl->info.mfa.function; obj->info.mfa.arity = tmpl->info.mfa.arity; - obj->code[0] = (BeamInstr) em_call_error_handler; - obj->code[1] = 0; + obj->beam[0] = (BeamInstr) em_call_error_handler; + obj->beam[1] = 0; for (ix=0; ixaddressv[ix] = obj->code; + obj->addressv[ix] = obj->beam; blob->entryv[ix].slot.index = -1; blob->entryv[ix].ep = &blob->exp; @@ -266,8 +266,8 @@ erts_find_function(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix) ee = hash_get(&export_tables[code_ix].htable, init_template(&templ, m, f, a)); if (ee == NULL || - (ee->ep->addressv[code_ix] == ee->ep->code && - ee->ep->code[0] != (BeamInstr) BeamOp(op_i_generic_breakpoint))) { + (ee->ep->addressv[code_ix] == ee->ep->beam && + ee->ep->beam[0] != (BeamInstr) BeamOp(op_i_generic_breakpoint))) { return NULL; } return ee->ep; diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h index a807efef94..198b90c839 100644 --- a/erts/emulator/beam/export.h +++ b/erts/emulator/beam/export.h @@ -33,20 +33,20 @@ typedef struct export { void* addressv[ERTS_NUM_CODE_IX]; /* Pointer to code for function. */ - ErtsCodeInfo info; /* MUST be just before code[] */ + ErtsCodeInfo info; /* MUST be just before beam[] */ /* - * code[0]: This entry is 0 unless the 'address' field points to it. + * beam[0]: This entry is 0 unless the 'addressv' field points to it. * Threaded code instruction to load function * (em_call_error_handler), execute BIF (em_apply_bif), * or a breakpoint instruction (op_i_generic_breakpoint). - * code[1]: Function pointer to BIF function (for BIFs only), + * beam[1]: Function pointer to BIF function (for BIFs only), * or pointer to threaded code if the module has an * on_load function that has not been run yet, or pointer - * to code if function code[0] is a breakpoint instruction. + * to code if function beam[0] is a breakpoint instruction. * Otherwise: 0. */ - BeamInstr code[2]; + BeamInstr beam[2]; } Export; @@ -72,8 +72,8 @@ extern erts_smp_mtx_t export_staging_lock; #include "beam_load.h" /* For em_* extern declarations */ #define ExportIsBuiltIn(EntryPtr) \ -(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->code) && \ - ((EntryPtr)->code[0] == (BeamInstr) em_apply_bif)) +(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->beam) && \ + ((EntryPtr)->beam[0] == (BeamInstr) em_apply_bif)) #if ERTS_GLB_INLINE_INCL_FUNC_DEF -- cgit v1.2.3 From c283f299551d53b62027ae1a8b189285c57573e8 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Tue, 11 Oct 2016 08:14:04 +0200 Subject: erts: Fix some dtrace related compile issues --- erts/emulator/beam/beam_emu.c | 31 ++++++++++++++----------------- erts/emulator/beam/erl_process.c | 7 ++++--- 2 files changed, 18 insertions(+), 20 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 8e88d6ccd5..3b02df8bac 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1160,7 +1160,7 @@ init_emulator(void) #define DTRACE_NIF_RETURN(p, mfa) \ if (DTRACE_ENABLED(nif_return)) { \ DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa_mfa, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ DTRACE2(nif_return, process_name, mfa_buf); \ } @@ -1175,9 +1175,9 @@ init_emulator(void) #define DTRACE_RETURN_FROM_PC(p) \ do { \ - ErtsCodeInfo* ci; \ - if (DTRACE_ENABLED(function_return) && (ci = find_function_from_pc((p)->cp))) { \ - DTRACE_RETURN((p), &ci->mfa); \ + ErtsCodeMFA* cmfa; \ + if (DTRACE_ENABLED(function_return) && (cmfa = find_function_from_pc((p)->cp))) { \ + DTRACE_RETURN((p), cmfa); \ } \ } while(0) @@ -1392,10 +1392,9 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) if (ERTS_PROC_IS_EXITING(c_p)) { strcpy(fun_buf, ""); } else { - BeamInstr *fptr = find_function_from_pc(c_p->i); - if (fptr) { - dtrace_fun_decode(c_p, (Eterm)fptr[0], - (Eterm)fptr[1], (Uint)fptr[2], + ErtsCodeMFA *cmfa = find_function_from_pc(c_p->i); + if (cmfa) { + dtrace_fun_decode(c_p, cmfa, NULL, fun_buf); } else { erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)), @@ -5356,11 +5355,9 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) if (ERTS_PROC_IS_EXITING(c_p)) { strcpy(fun_buf, ""); } else { - BeamInstr *fptr = find_function_from_pc(c_p->i); - if (fptr) { - dtrace_fun_decode(c_p, (Eterm)fptr[0], - (Eterm)fptr[1], (Uint)fptr[2], - NULL, fun_buf); + ErtsCodeMFA *cmfa = find_function_from_pc(c_p->i); + if (cmfa) { + dtrace_fun_decode(c_p, cmfa, NULL, fun_buf); } else { erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)), "", *I); @@ -6401,11 +6398,11 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re #ifdef USE_VM_PROBES if (DTRACE_ENABLED(process_hibernate)) { + ErtsCodeMFA cmfa = { module, function, arity}; DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); - dtrace_fun_decode(c_p, module, function, arity, - process_name, mfa); - DTRACE2(process_hibernate, process_name, mfa); + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); + dtrace_fun_decode(c_p, &cmfa, process_name, mfa_buf); + DTRACE2(process_hibernate, process_name, mfa_buf); } #endif /* diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index d0a0310544..dd5943b5cf 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -11690,11 +11690,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #ifdef USE_VM_PROBES if (DTRACE_ENABLED(process_spawn)) { + ErtsCodeMFA cmfa = {mod, func, arity}; DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); - dtrace_fun_decode(p, mod, func, arity, process_name, mfa); - DTRACE2(process_spawn, process_name, mfa); + dtrace_fun_decode(p, &cmfa, process_name, mfa_buf); + DTRACE2(process_spawn, process_name, mfa_buf); } #endif return res; -- cgit v1.2.3 From e22f99f0303bb5e70b128118bad320ee9368e4ba Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 12 Oct 2016 16:14:51 +0200 Subject: erts: Add strict size assert to fix alloc and remove outdated min size adjustment. The is similar to lost commit 31bc414dc9639ccf94f9011ed32 except we now assume and assert strict size equality. --- erts/emulator/beam/erl_alloc_util.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 2995f2f822..dd8ca5fac1 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -1361,6 +1361,7 @@ fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + ASSERT(size == fix->type_size); res = fix->list; if (res) { @@ -1372,8 +1373,6 @@ fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) fix_cpool_check_shrink(allctr, type, fix, NULL); return res; } - if (size < 2*sizeof(UWord)) - size += sizeof(UWord); if (size >= allctr->sbc_threshold) { Block_t *blk; blk = create_carrier(allctr, size, CFLG_SBC); @@ -1493,6 +1492,7 @@ fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + ASSERT(size == fix->type_size); ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); fix->u.nocpool.used++; @@ -1515,8 +1515,6 @@ fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); return res; } - if (size < 2*sizeof(UWord)) - size += sizeof(UWord); if (fix->u.nocpool.limit < fix->u.nocpool.used) fix->u.nocpool.limit = fix->u.nocpool.used; if (fix->u.nocpool.max_used < fix->u.nocpool.used) -- cgit v1.2.3 From 2dfca60a24c2c6728a4b8431f37aeee05212a4ac Mon Sep 17 00:00:00 2001 From: Gabriele Santomaggio Date: Thu, 13 Oct 2016 21:17:10 +0200 Subject: Add system_info(atom_limit) Add system_info(atom_limit) to provide a way to retrieve the maximum number of atoms allowed. Add tests and documentation for it too. Also split system_info_SUITE:start_node/2 to start_node_ets/2 and start_node_atm/2 to avoid code duplication. --- erts/emulator/beam/atom.c | 6 ++++++ erts/emulator/beam/atom.h | 1 + erts/emulator/beam/erl_bif_info.c | 3 +++ 3 files changed, 10 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index a5e778e4aa..d2fad24805 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -483,3 +483,9 @@ dump_atoms(int to, void *to_arg) } } } + +Uint +erts_get_atom_limit(void) +{ + return erts_atom_table.limit; +} \ No newline at end of file diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h index ae60904785..0d34c47a55 100644 --- a/erts/emulator/beam/atom.h +++ b/erts/emulator/beam/atom.h @@ -138,6 +138,7 @@ int atom_static_put(byte*, int); void init_atom_table(void); void atom_info(int, void *); void dump_atoms(int, void *); +Uint erts_get_atom_limit(void); int erts_atom_get(const char* name, int len, Eterm* ap, ErtsAtomEncoding enc); void erts_atom_get_text_space_sizes(Uint *reserved, Uint *used); #endif diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 29ba12dfdb..cfe01f244e 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -2859,6 +2859,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("ets_limit",BIF_ARG_1)) { BIF_RET(make_small(erts_db_get_max_tabs())); } + else if (ERTS_IS_ATOM_STR("atom_limit",BIF_ARG_1)) { + BIF_RET(make_small(erts_get_atom_limit())); + } else if (ERTS_IS_ATOM_STR("tolerant_timeofday",BIF_ARG_1)) { if (erts_has_time_correction() && erts_time_offset_state() == ERTS_TIME_OFFSET_FINAL) { -- cgit v1.2.3 From 39072836944d00c288beebfd98b14593f9609006 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Magnus=20L=C3=A5ng?= Date: Thu, 19 May 2016 17:38:54 +0200 Subject: Add a loader state for HiPE code loading Just like the BEAM loader state (as returned by erlang:prepare_loading/2), the HiPE loader state is contained in a magic binary. Eventually, we will separate HiPE loading into a prepare and a finalise phase, like the BEAM loader, where the prepare phase will be implemented by hipe_unified_loader and the finalise phase be implemented in C by hipe_load.c and beam_load.c, making prepare side-effect free and finalise atomic. The finalise phase will be exposed through the erlang:finish_loading/1 API, just like the BEAM loader, as this will allow HiPE and BEAM modules to be mixed in the same atomic "commit". The usage of a loader state makes it easier to keep track of all resources allocated during loading, and will not only make it easy to prevent leaks when hipe_unified_loader crashes, but also paves the way for proper, leak-free, unloading of HiPE modules. --- erts/emulator/beam/beam_bif_load.c | 14 +++++--- erts/emulator/beam/beam_load.c | 69 ++++++++++++++++++++++++-------------- erts/emulator/beam/module.c | 5 +-- erts/emulator/beam/module.h | 9 ++--- 4 files changed, 59 insertions(+), 38 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 237513095a..3302de0787 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -150,7 +150,13 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) { Module* modp; - Eterm res; + Eterm res, mod; + + if (!ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_1) || + is_not_atom(mod = erts_module_for_prepared_code + (((ProcBin*)binary_val(BIF_ARG_1))->val))) { + BIF_ERROR(BIF_P, BADARG); + } if (!erts_try_seize_code_write_permission(BIF_P)) { ERTS_BIF_YIELD3(bif_export[BIF_code_make_stub_module_3], @@ -160,7 +166,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); - modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); + modp = erts_get_module(mod, erts_active_code_ix()); if (modp && modp->curr.num_breakpoints > 0) { ASSERT(modp->curr.code_hdr != NULL); @@ -172,12 +178,12 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); - if (res == BIF_ARG_1) { + if (res == mod) { erts_end_staging_code_ix(); erts_commit_staging_code_ix(); #ifdef HIPE if (!modp) - modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); + modp = erts_get_module(mod, erts_active_code_ix()); hipe_redirect_to_module(modp); #endif } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 9b206f9a23..7d29f393e5 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -44,6 +44,7 @@ #include "hipe_bif0.h" #include "hipe_mode_switch.h" #include "hipe_arch.h" +#include "hipe_load.h" #endif ErlDrvBinary* erts_gzinflate_buffer(char*, int); @@ -482,8 +483,8 @@ static void free_literal_fragment(ErlHeapFragment*); static void loader_state_dtor(Binary* magic); static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, - BeamCodeHeader* code, Uint size, - void* hipe_code_start, UWord hipe_code_size); + BeamCodeHeader* code_hdr, Uint size, + HipeModule *hipe_code); static int init_iff_file(LoaderState* stp, byte* code, Uint size); static int scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory); @@ -942,6 +943,13 @@ erts_module_for_prepared_code(Binary* magic) LoaderState* stp; if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != loader_state_dtor) { +#ifdef HIPE + HipeLoaderState *hipe_stp; + if ((hipe_stp = hipe_get_loader_state(magic)) + && hipe_stp->text_segment != 0) { + return hipe_stp->module; + } +#endif return NIL; } stp = ERTS_MAGIC_BIN_DATA(magic); @@ -1094,7 +1102,7 @@ static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, BeamCodeHeader* code_hdr, Uint size, - void* hipe_code_start, UWord hipe_code_size) + HipeModule *hipe_code) { Module* modp; Eterm retval; @@ -1121,12 +1129,9 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code new_hipe_refs = %p", modp->new_hipe_refs); modp->curr.first_hipe_ref = modp->new_hipe_refs; modp->curr.first_hipe_sdesc = modp->new_hipe_sdesc; - modp->curr.hipe_code_start = hipe_code_start; modp->new_hipe_refs = NULL; modp->new_hipe_sdesc = NULL; -# ifdef DEBUG - modp->curr.hipe_code_size = hipe_code_size; -# endif + modp->curr.hipe_code = hipe_code; #endif /* @@ -6271,10 +6276,13 @@ patch_funentries(Eterm Patchlist) */ Eterm -erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) +erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) { Binary* magic; + Binary* hipe_magic; LoaderState* stp; + HipeLoaderState* hipe_stp; + HipeModule *hipe_code; BeamInstr Funcs; BeamInstr Patchlist; Eterm MD5Bin; @@ -6290,7 +6298,6 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) byte* temp_alloc = NULL; byte* bytes; Uint size; - UWord hipe_code_start = NULL, hipe_code_size = 0; /* * Must initialize stp->lambdas here because the error handling code @@ -6298,15 +6305,19 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) */ magic = erts_alloc_loader_state(); stp = ERTS_MAGIC_BIN_DATA(magic); + hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code)); - if (is_not_atom(Mod)) { + if (!ERTS_TERM_IS_MAGIC_BINARY(hipe_magic_bin) || + !(hipe_magic = ((ProcBin*)binary_val(hipe_magic_bin))->val, + hipe_stp = hipe_get_loader_state(hipe_magic)) || + hipe_stp->module == NIL || hipe_stp->text_segment == 0) { goto error; } if (is_not_tuple(Info)) { goto error; } tp = tuple_val(Info); - if (tp[0] != make_arityval(5)) { + if (tp[0] != make_arityval(3)) { goto error; } Funcs = tp[1]; @@ -6323,20 +6334,11 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) } size = binary_size(Beam); -#ifdef HIPE - if (!term_to_Uint(tp[4], &hipe_code_start)) - goto error; -# ifdef DEBUG - if (!term_to_Uint(tp[5], &hipe_code_size)) - goto error; -# endif -#endif - /* * Scan the Beam binary and read the interesting sections. */ - stp->module = Mod; + stp->module = hipe_stp->module; stp->group_leader = p->group_leader; stp->num_functions = n; if (!init_iff_file(stp, bytes, size)) { @@ -6450,7 +6452,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) #else op = (Eterm) BeamOpCode(op_move_return_n); #endif - fp = make_stub(fp, Mod, func, arity, (Uint)native_address, op); + fp = make_stub(fp, hipe_stp->module, func, arity, (Uint)native_address, + op); } /* @@ -6489,13 +6492,19 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) erts_free_aligned_binary_bytes(tmp); } + /* + * Initialise HiPE module + */ + hipe_code->text_segment = hipe_stp->text_segment; + hipe_code->text_segment_size = hipe_stp->text_segment_size; + hipe_code->data_segment = hipe_stp->data_segment; + /* * Insert the module in the module table. */ - rval = stub_insert_new_code(p, 0, p->group_leader, Mod, - code_hdr, code_size, - (void*)hipe_code_start, hipe_code_size); + rval = stub_insert_new_code(p, 0, p->group_leader, hipe_stp->module, + code_hdr, code_size, hipe_code); if (rval != NIL) { goto error; } @@ -6511,12 +6520,20 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) } if (patch_funentries(Patchlist)) { + Eterm mod = hipe_stp->module; + /* Prevent code from being freed */ + hipe_stp->text_segment = 0; + hipe_stp->data_segment = 0; + erts_free_aligned_binary_bytes(temp_alloc); free_loader_state(magic); - return Mod; + hipe_free_loader_state(hipe_magic); + + return mod; } error: + erts_free(ERTS_ALC_T_HIPE, hipe_code); erts_free_aligned_binary_bytes(temp_alloc); free_loader_state(magic); BIF_ERROR(p, BADARG); diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index 93ce8fcf9f..5c56ad20df 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -79,10 +79,7 @@ void erts_module_instance_init(struct erl_module_instance* modi) #ifdef HIPE modi->first_hipe_ref = NULL; modi->first_hipe_sdesc = NULL; - modi->hipe_code_start = NULL; -# ifdef DEBUG - modi->hipe_code_size = 0; -# endif + modi->hipe_code = NULL; #endif } diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 694583597b..c3adaf12fc 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -23,6 +23,10 @@ #include "index.h" +#ifdef HIPE +#include "hipe_module.h" +#endif + struct erl_module_instance { BeamCodeHeader* code_hdr; int code_length; /* Length of loaded code in bytes. */ @@ -33,10 +37,7 @@ struct erl_module_instance { #ifdef HIPE struct hipe_ref* first_hipe_ref; /* all external hipe calls from this module */ struct hipe_sdesc* first_hipe_sdesc; /* all stack descriptors for this module */ - void* hipe_code_start; -# ifdef DEBUG - UWord hipe_code_size; -# endif + HipeModule *hipe_code; #endif }; -- cgit v1.2.3 From 5e879fb1d4a9273f0a6183da7509dcdbc24f9653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Magnus=20L=C3=A5ng?= Date: Fri, 16 Sep 2016 19:33:38 +0200 Subject: erts: Check hipe stack in check_process_code This is part of commit 1bd508921dd93086b05e7d0038b816b36c421d86. I did not include the fun-checking as we have a new purge strategy for funs in OTP 20. That remains to be solved some other way for hipe. --- erts/emulator/beam/beam_bif_load.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 3302de0787..ec1db8d9c2 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1069,6 +1069,8 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) BeamInstr* start; char* mod_start; Uint mod_size; + void *nat_start = NULL; + Uint nat_size = 0; Eterm* sp; *redsp += 1; @@ -1099,6 +1101,20 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) } } +#ifdef HIPE + /* + * Check all continuation pointers stored on the native stack if the module + * has native code. + */ + if (modp->old.hipe_code) { + nat_start = modp->old.hipe_code->text_segment; + nat_size = modp->old.hipe_code->text_segment_size; + if (nat_size && nstack_any_cps_in_segment(rp, nat_start, nat_size)) { + return am_true; + } + } +#endif + /* * Check all continuation pointers stored in stackdump * and clear exception stackdump if there is a pointer @@ -1115,8 +1131,16 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) rp->ftrace = NIL; } else { int i; + char *area_start = mod_start; + Uint area_size = mod_size; +#ifdef HIPE + if (rp->freason & EXF_NATIVE) { + area_start = nat_start; + area_size = nat_size; + } +#endif for (i = 0; i < s->depth; i++) { - if (ErtsInArea(s->trace[i], mod_start, mod_size)) { + if (ErtsInArea(s->trace[i], area_start, area_size)) { rp->freason = EXC_NULL; rp->fvalue = NIL; rp->ftrace = NIL; -- cgit v1.2.3 From a82be01198c1f3714f29f0a0e3ce3ed6d6bce0a6 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 16 Sep 2016 19:39:24 +0200 Subject: erts: Fill freed EXEC memory with illegal instructions on DEBUG to get a nice SIGILL crash. For x86 and x86_64 only. --- erts/emulator/beam/erl_alloc.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 3c2c9def3b..d96b90fd55 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -4129,12 +4129,20 @@ debug_free(ErtsAlcType_t n, void *extra, void *ptr) ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; void *dptr; Uint size; + int free_pattern = n; ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); dptr = check_memory_fence(ptr, &size, n, ERTS_ALC_O_FREE); - sys_memset((void *) dptr, n, size + FENCE_SZ); +#ifdef ERTS_ALC_A_EXEC +# if defined(__i386__) || defined(__x86_64__) + if (ERTS_ALC_T2A(ERTS_ALC_N2T(n)) == ERTS_ALC_A_EXEC) { + free_pattern = 0x0f; /* Illegal instruction */ + } +# endif +#endif + sys_memset((void *) dptr, free_pattern, size + FENCE_SZ); (*real_af->free)(n, real_af->extra, dptr); -- cgit v1.2.3 From 966098ceb9dd9d18e9bcd37cd06b96045903e320 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 20 Sep 2016 16:16:50 +0200 Subject: erts: Move new hipe ref and sdesc lists to loader state --- erts/emulator/beam/beam_bif_load.c | 1 - erts/emulator/beam/beam_load.c | 11 ++++++----- erts/emulator/beam/module.c | 6 ------ erts/emulator/beam/module.h | 4 ---- 4 files changed, 6 insertions(+), 16 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index ec1db8d9c2..f5d1ca7e54 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1798,7 +1798,6 @@ delete_code(Module* modp) ASSERT(modp->curr.num_breakpoints == 0); ASSERT(modp->curr.num_traced_exports == 0); - DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "delete_code old.first_hipe_ref=%p", modp->curr.first_hipe_ref); modp->old = modp->curr; erts_module_instance_init(&modp->curr); } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 7d29f393e5..0eb390bf4c 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -1126,11 +1126,8 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, modp->curr.code_length = size; modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */ #if defined(HIPE) - DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code new_hipe_refs = %p", modp->new_hipe_refs); - modp->curr.first_hipe_ref = modp->new_hipe_refs; - modp->curr.first_hipe_sdesc = modp->new_hipe_sdesc; - modp->new_hipe_refs = NULL; - modp->new_hipe_sdesc = NULL; + DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code " + "first_hipe_ref = %p", hipe_code->first_hipe_ref); modp->curr.hipe_code = hipe_code; #endif @@ -6498,6 +6495,8 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) hipe_code->text_segment = hipe_stp->text_segment; hipe_code->text_segment_size = hipe_stp->text_segment_size; hipe_code->data_segment = hipe_stp->data_segment; + hipe_code->first_hipe_ref = hipe_stp->new_hipe_refs; + hipe_code->first_hipe_sdesc = hipe_stp->new_hipe_sdesc; /* * Insert the module in the module table. @@ -6524,6 +6523,8 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) /* Prevent code from being freed */ hipe_stp->text_segment = 0; hipe_stp->data_segment = 0; + hipe_stp->new_hipe_refs = NULL; + hipe_stp->new_hipe_sdesc = NULL; erts_free_aligned_binary_bytes(temp_alloc); free_loader_state(magic); diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index 5c56ad20df..e9b18ba525 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -77,8 +77,6 @@ void erts_module_instance_init(struct erl_module_instance* modi) modi->num_breakpoints = 0; modi->num_traced_exports = 0; #ifdef HIPE - modi->first_hipe_ref = NULL; - modi->first_hipe_sdesc = NULL; modi->hipe_code = NULL; #endif } @@ -95,8 +93,6 @@ static Module* module_alloc(Module* tmpl) obj->on_load = 0; #ifdef HIPE obj->first_hipe_mfa = NULL; - obj->new_hipe_refs = NULL; - obj->new_hipe_sdesc = NULL; #endif DBG_TRACE_MFA(make_atom(obj->module), 0, 0, "module_alloc"); return obj; @@ -214,8 +210,6 @@ static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod) dst_mod->on_load = src_mod->on_load; #ifdef HIPE dst_mod->first_hipe_mfa = src_mod->first_hipe_mfa; - dst_mod->new_hipe_refs = src_mod->new_hipe_refs; - dst_mod->new_hipe_sdesc = src_mod->new_hipe_sdesc; #endif } diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index c3adaf12fc..9de610ccf7 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -35,8 +35,6 @@ struct erl_module_instance { int num_breakpoints; int num_traced_exports; #ifdef HIPE - struct hipe_ref* first_hipe_ref; /* all external hipe calls from this module */ - struct hipe_sdesc* first_hipe_sdesc; /* all stack descriptors for this module */ HipeModule *hipe_code; #endif }; @@ -51,8 +49,6 @@ typedef struct erl_module { struct erl_module_instance* on_load; #ifdef HIPE struct hipe_mfa_info* first_hipe_mfa; - struct hipe_ref* new_hipe_refs; - struct hipe_sdesc* new_hipe_sdesc; #endif } Module; -- cgit v1.2.3 From e77d7a8417368617c4c228af9556a7f6a8f3e84c Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 28 Sep 2016 20:55:40 +0200 Subject: erts: Fix early hipe patch loading by introducing hipe_bifs:commit_patch_load/1 that creates the HipeModule. --- erts/emulator/beam/beam_load.c | 39 +++++++++++++++++++++++++++++++++++++++ erts/emulator/beam/global.h | 1 + 2 files changed, 40 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 0eb390bf4c..4a833689e3 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -6540,6 +6540,45 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) BIF_ERROR(p, BADARG); } +int erts_commit_hipe_patch_load(Eterm hipe_magic_bin) +{ + Binary* hipe_magic; + HipeLoaderState* hipe_stp; + HipeModule *hipe_code; + Module* modp; + + if (!ERTS_TERM_IS_MAGIC_BINARY(hipe_magic_bin) || + !(hipe_magic = ((ProcBin*)binary_val(hipe_magic_bin))->val, + hipe_stp = hipe_get_loader_state(hipe_magic)) || + hipe_stp->module == NIL || hipe_stp->text_segment == 0) { + return 0; + } + + modp = erts_get_module(hipe_stp->module, erts_active_code_ix()); + if (!modp) + return 0; + + /* + * Initialise HiPE module + */ + hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code)); + hipe_code->text_segment = hipe_stp->text_segment; + hipe_code->text_segment_size = hipe_stp->text_segment_size; + hipe_code->data_segment = hipe_stp->data_segment; + hipe_code->first_hipe_ref = hipe_stp->new_hipe_refs; + hipe_code->first_hipe_sdesc = hipe_stp->new_hipe_sdesc; + + modp->curr.hipe_code = hipe_code; + + /* Prevent code from being freed */ + hipe_stp->text_segment = 0; + hipe_stp->data_segment = 0; + hipe_stp->new_hipe_refs = NULL; + hipe_stp->new_hipe_sdesc = NULL; + + return 1; +} + #undef WORDS_PER_FUNCTION static int safe_mul(UWord a, UWord b, UWord* resp) diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 93996e8b41..29a0bc8d0b 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1048,6 +1048,7 @@ void erts_set_current_function(FunctionInfo* fi, BeamInstr* current); Eterm erts_module_info_0(Process* p, Eterm module); Eterm erts_module_info_1(Process* p, Eterm module, Eterm what); Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info); +int erts_commit_hipe_patch_load(Eterm hipe_magic_bin); /* beam_ranges.c */ void erts_init_ranges(void); -- cgit v1.2.3 From 870d49f0f8d383e62ea231700089933c4cf8fe2a Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 29 Sep 2016 17:38:54 +0200 Subject: hipe,erts: Remove cached fun 'native_address' Did not work with purge and made worse by new purge strategy. Did yield terrible performance when fun thing is created *before* fun code is loaded. Like when receiving not yet loaded fun from other node. The cached 'native_address' in ErlFunThing will not be updated leading to mode switch and error_handler being called for every call to the fun from native code. --- erts/emulator/beam/beam_emu.c | 3 --- erts/emulator/beam/erl_fun.h | 3 --- erts/emulator/beam/external.c | 4 ---- 3 files changed, 10 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index ccb0f786ec..3fa4fb5b3f 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -6667,9 +6667,6 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) funp->fe = fe; funp->num_free = num_free; funp->creator = p->common.id; -#ifdef HIPE - funp->native_address = fe->native_address; -#endif funp->arity = (int)fe->address[-1] - num_free; for (i = 0; i < num_free; i++) { *hp++ = reg[i]; diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index 73c3e19c1c..6ba055d92c 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -57,9 +57,6 @@ typedef struct erl_fun_thing { Eterm thing_word; /* Subtag FUN_SUBTAG. */ ErlFunEntry* fe; /* Pointer to fun entry. */ struct erl_off_heap_header* next; -#ifdef HIPE - UWord* native_address; /* Native code for the fun. */ -#endif Uint arity; /* The arity of the fun. */ Uint num_free; /* Number of free variables (in env). */ /* -- The following may be compound Erlang terms ---------------------- */ diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index beed847578..ca79f83184 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -3749,7 +3749,6 @@ dec_term_atom_common: if (funp->fe->native_address == NULL) { hipe_set_closure_stub(funp->fe, num_free); } - funp->native_address = funp->fe->native_address; #endif hp = factory->hp; @@ -3821,9 +3820,6 @@ dec_term_atom_common: funp->fe = erts_put_fun_entry(module, old_uniq, old_index); funp->arity = funp->fe->address[-1] - num_free; -#ifdef HIPE - funp->native_address = funp->fe->native_address; -#endif hp = factory->hp; /* Environment */ -- cgit v1.2.3 From 1e8588e8c310a628fb0215ef15dc3cc29f98c336 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Sun, 2 Oct 2016 16:17:21 +0200 Subject: erts: Refactor hipe_loader_state_dtor into a true destructor that is only called once. Basically switch hipe_free_loader_state and hipe_loader_state_dtor. --- erts/emulator/beam/beam_load.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 4a833689e3..813788f66c 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -6528,7 +6528,7 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) erts_free_aligned_binary_bytes(temp_alloc); free_loader_state(magic); - hipe_free_loader_state(hipe_magic); + hipe_free_loader_state(hipe_stp); return mod; } -- cgit v1.2.3 From 6eee26d55f32113e3ea4575e5500f2fc3c312fff Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 4 Oct 2016 21:46:53 +0200 Subject: erts: Enable exec_alloc for all hipe architectures For non-amd64 it's a "normal" allocator with a wrapper around mseg_alloc to call mprotect(PROT_EXEC). --- erts/emulator/beam/erl_alloc.c | 16 +++++++++---- erts/emulator/beam/erl_alloc_util.c | 48 ++++++++++++++++++++++++++++++++++++- erts/emulator/beam/erl_alloc_util.h | 6 +++++ 3 files changed, 64 insertions(+), 6 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index d96b90fd55..56aa78a08a 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -373,10 +373,16 @@ set_default_exec_alloc_opts(struct au_init *ip) ip->init.util.rmbcmt = 0; ip->init.util.acul = 0; +# ifdef ERTS_HAVE_EXEC_MMAPPER ip->init.util.mseg_alloc = &erts_alcu_mmapper_mseg_alloc; ip->init.util.mseg_realloc = &erts_alcu_mmapper_mseg_realloc; ip->init.util.mseg_dealloc = &erts_alcu_mmapper_mseg_dealloc; ip->init.util.mseg_mmapper = &erts_exec_mmapper; +# else + ip->init.util.mseg_alloc = &erts_alcu_exec_mseg_alloc; + ip->init.util.mseg_realloc = &erts_alcu_exec_mseg_realloc; + ip->init.util.mseg_dealloc = &erts_alcu_exec_mseg_dealloc; +# endif } #endif /* ERTS_ALC_A_EXEC */ @@ -1571,7 +1577,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) break; case 'X': if (has_prefix("scs", argv[i]+3)) { -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER init->mseg.exec_mmap.scs = #endif get_mb_value(argv[i]+6, argv, &i); @@ -2852,7 +2858,7 @@ erts_allocator_info(int to, void *arg) erts_print(to, arg, "=allocator:erts_mmap.literal_mmap\n"); erts_mmap_info(&erts_literal_mmapper, &to, arg, NULL, NULL, &emis); #endif -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER erts_print(to, arg, "=allocator:erts_mmap.exec_mmap\n"); erts_mmap_info(&erts_exec_mmapper, &to, arg, NULL, NULL, &emis); #endif @@ -3010,7 +3016,7 @@ erts_allocator_options(void *proc) #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) terms[length++] = ERTS_MAKE_AM("literal_mmap"); #endif -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER terms[length++] = ERTS_MAKE_AM("exec_mmap"); #endif features = length ? erts_bld_list(hpp, szp, length, terms) : NIL; @@ -3102,7 +3108,7 @@ reply_alloc_info(void *vair) # if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) struct erts_mmap_info_struct mmap_info_literal; # endif -# ifdef ERTS_ALC_A_EXEC +# ifdef ERTS_HAVE_EXEC_MMAPPER struct erts_mmap_info_struct mmap_info_exec; # endif #endif @@ -3232,7 +3238,7 @@ reply_alloc_info(void *vair) erts_bld_atom(hpp,szp,"literal_mmap"), ainfo); # endif -# ifdef ERTS_ALC_A_EXEC +# ifdef ERTS_HAVE_EXEC_MMAPPER ai_list = erts_bld_cons(hpp, szp, ainfo, ai_list); ainfo = (air->only_sz ? NIL : diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 2995f2f822..77be48035d 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -907,7 +907,9 @@ erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, #elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) -/* Used by literal allocator that has its own mmapper (super carrier) */ +/* For allocators that have their own mmapper (super carrier), + * like literal_alloc and exec_alloc on amd64 + */ void* erts_alcu_mmapper_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) { @@ -948,6 +950,50 @@ erts_alcu_mmapper_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, } #endif /* ARCH_64 && ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */ +#if defined(ERTS_ALC_A_EXEC) && !defined(ERTS_HAVE_EXEC_MMAPPER) + +/* + * For exec_alloc on non-amd64 that just need memory with PROT_EXEC + */ +void* +erts_alcu_exec_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) +{ + void* res = erts_alcu_mseg_alloc(allctr, size_p, flags); + + if (res) { + int r = mprotect(res, *size_p, PROT_EXEC | PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + } + return res; +} + +void* +erts_alcu_exec_mseg_realloc(Allctr_t *allctr, void *seg, + Uint old_size, Uint *new_size_p) +{ + void *res; + + if (seg && old_size) { + int r = mprotect(seg, old_size, PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + } + res = erts_alcu_mseg_realloc(allctr, seg, old_size, new_size_p); + if (res) { + int r = mprotect(res, *new_size_p, PROT_EXEC | PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + } + return res; +} + +void +erts_alcu_exec_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) +{ + int r = mprotect(seg, size, PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + erts_alcu_mseg_dealloc(allctr, seg, size, flags); +} +#endif /* ERTS_ALC_A_EXEC && !ERTS_HAVE_EXEC_MMAPPER */ + #endif /* HAVE_ERTS_MSEG */ void* diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index f50f09907a..2958bdf8d1 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -210,6 +210,12 @@ void* erts_alcu_mmapper_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); void* erts_alcu_mmapper_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); void erts_alcu_mmapper_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); # endif + +# if defined(ERTS_ALC_A_EXEC) && !defined(ERTS_HAVE_EXEC_MMAPPER) +void* erts_alcu_exec_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); +void* erts_alcu_exec_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); +void erts_alcu_exec_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); +# endif #endif /* HAVE_ERTS_MSEG */ void* erts_alcu_sys_alloc(Allctr_t*, Uint *size_p, int superalign); -- cgit v1.2.3 From deed6619dee9db9c0347b25987f3a91d51424079 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 7 Oct 2016 16:13:55 +0200 Subject: erts: Let code:make_stub_module raise 'notsup' if hipe is disabled. Makes the code simpler to just ifdef away a lot of hipe stuff. --- erts/emulator/beam/beam_bif_load.c | 10 +++++++--- erts/emulator/beam/beam_load.c | 29 +++++++++-------------------- 2 files changed, 16 insertions(+), 23 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index f5d1ca7e54..0a4d89d51b 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -149,6 +149,9 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) { +#if !defined(HIPE) + BIF_ERROR(BIF_P, EXC_NOTSUP); +#else Module* modp; Eterm res, mod; @@ -181,11 +184,9 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) if (res == mod) { erts_end_staging_code_ix(); erts_commit_staging_code_ix(); -#ifdef HIPE if (!modp) modp = erts_get_module(mod, erts_active_code_ix()); hipe_redirect_to_module(modp); -#endif } else { erts_abort_staging_code_ix(); @@ -194,6 +195,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_release_code_write_permission(); return res; +#endif } BIF_RETTYPE @@ -1069,9 +1071,11 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) BeamInstr* start; char* mod_start; Uint mod_size; + Eterm* sp; +#ifdef HIPE void *nat_start = NULL; Uint nat_size = 0; - Eterm* sp; +#endif *redsp += 1; diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 813788f66c..4833bcc6e9 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -481,10 +481,12 @@ static void free_loader_state(Binary* magic); static ErlHeapFragment* new_literal_fragment(Uint size); static void free_literal_fragment(ErlHeapFragment*); static void loader_state_dtor(Binary* magic); +#ifdef HIPE static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, BeamCodeHeader* code_hdr, Uint size, HipeModule *hipe_code); +#endif static int init_iff_file(LoaderState* stp, byte* code, Uint size); static int scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory); @@ -539,8 +541,6 @@ static Eterm compilation_info_for_module(Process* p, BeamCodeHeader*); static Eterm md5_of_module(Process* p, BeamCodeHeader*); static Eterm has_native(BeamCodeHeader*); static Eterm native_addresses(Process* p, BeamCodeHeader*); -int patch_funentries(Eterm Patchlist); -int patch(Eterm Addresses, Uint fe); static int safe_mul(UWord a, UWord b, UWord* resp); static int must_swap_floats; @@ -1098,6 +1098,7 @@ loader_state_dtor(Binary* magic) ASSERT(stp->genop_blocks == 0); } +#ifdef HIPE static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, @@ -1125,11 +1126,9 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, modp->curr.code_hdr = code_hdr; modp->curr.code_length = size; modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */ -#if defined(HIPE) DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code " "first_hipe_ref = %p", hipe_code->first_hipe_ref); modp->curr.hipe_code = hipe_code; -#endif /* * Update ranges (used for finding a function from a PC value). @@ -1138,6 +1137,7 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, erts_update_ranges((BeamInstr*)modp->curr.code_hdr, size); return NIL; } +#endif static int init_iff_file(LoaderState* stp, byte* code, Uint size) @@ -5997,6 +5997,7 @@ code_module_md5_1(BIF_ALIST_1) return res; } +#ifdef HIPE #define WORDS_PER_FUNCTION 6 static BeamInstr* @@ -6075,9 +6076,7 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) Eterm mod = fp[2]; Eterm function = fp[3]; int arity = fp[4]; -#ifdef HIPE Lambda* lp; -#endif if (is_bif(mod, function, arity)) { fp[1] = 0; @@ -6106,7 +6105,6 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) * Search the lambda table to find out which. */ -#ifdef HIPE n = stp->num_lambdas; for (i = 0, lp = stp->lambdas; i < n; i++, lp++) { ErlFunEntry* fe = stp->lambdas[i].fe; @@ -6115,7 +6113,6 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) fe->address = &(fp[5]); } } -#endif return; } @@ -6124,10 +6121,9 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) [{Adr, Patchtyppe} | Addresses] and the address of a fun_entry. */ -int +static int patch(Eterm Addresses, Uint fe) { -#ifdef HIPE Eterm* listp; Eterm tuple; Eterm* tp; @@ -6163,15 +6159,13 @@ patch(Eterm Addresses, Uint fe) } -#endif return 1; } -int +static int patch_funentries(Eterm Patchlist) { -#ifdef HIPE while (!is_nil(Patchlist)) { Eterm Info; Eterm MFA; @@ -6260,18 +6254,15 @@ patch_funentries(Eterm Patchlist) return 0; } -#endif return 1; /* Signal that all went well */ } - /* * Do a dummy load of a module. No threaded code will be loaded. * Used for loading native code. * Will also patch all references to fun_entries to point to * the new fun_entries created. */ - Eterm erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) { @@ -6444,11 +6435,7 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) * as the body until we know what kind of trap we should put there. */ code_hdr->functions[i] = fp; -#ifdef HIPE op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */ -#else - op = (Eterm) BeamOpCode(op_move_return_n); -#endif fp = make_stub(fp, hipe_stp->module, func, arity, (Uint)native_address, op); } @@ -6580,6 +6567,8 @@ int erts_commit_hipe_patch_load(Eterm hipe_magic_bin) } #undef WORDS_PER_FUNCTION +#endif /* HIPE */ + static int safe_mul(UWord a, UWord b, UWord* resp) { -- cgit v1.2.3 From 7d5021bc9e1150b5d839438756851b3175f3ba85 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 12 Oct 2016 16:56:41 +0200 Subject: erts: Disable DBG_TRACE_MFA for debug build --- erts/emulator/beam/beam_load.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 5f0f34fcdd..43cf6597df 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -151,10 +151,16 @@ struct BeamCodeLineTab_ { #define LOC_FILE(Loc) ((Loc) >> 24) #define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1)) -#ifdef DEBUG -# define ENABLE_DBG_TRACE_MFA -#endif +/* + * MFA event debug "tracing" usage: + * + * #define ENABLE_DBG_TRACE_MFA + * call dbg_set_traced_mfa("mymod","myfunc",arity) + * for the function(s) to trace, in some init function. + * + * Run and get stderr printouts when interesting things happen to your MFA. + */ #ifdef ENABLE_DBG_TRACE_MFA void dbg_set_traced_mfa(const char* m, const char* f, Uint a); -- cgit v1.2.3 From 3582e6f420d84ddac64b55cb13100f1ae7f31b08 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 13 Oct 2016 20:59:58 +0200 Subject: erts: Replace unsafe Module.first_hipe_ref with hash table mod2mfa_tab --- erts/emulator/beam/beam_bif_load.c | 2 +- erts/emulator/beam/module.c | 7 +------ erts/emulator/beam/module.h | 3 --- 3 files changed, 2 insertions(+), 10 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 0a4d89d51b..c31abbb84f 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1711,7 +1711,7 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) modp->old.catches = BEAM_CATCHES_NIL; erts_remove_from_ranges(code); #ifdef HIPE - hipe_purge_module(modp); + hipe_purge_module(modp, is_blocking); #endif ERTS_BIF_PREP_RET(ret, am_true); } diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index e9b18ba525..3e622c0f3a 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -91,9 +91,6 @@ static Module* module_alloc(Module* tmpl) erts_module_instance_init(&obj->curr); erts_module_instance_init(&obj->old); obj->on_load = 0; -#ifdef HIPE - obj->first_hipe_mfa = NULL; -#endif DBG_TRACE_MFA(make_atom(obj->module), 0, 0, "module_alloc"); return obj; } @@ -128,6 +125,7 @@ void init_module_table(void) erts_smp_atomic_init_nob(&tot_module_bytes, 0); } + Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix) { @@ -208,9 +206,6 @@ static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod) dst_mod->curr = src_mod->curr; dst_mod->old = src_mod->old; dst_mod->on_load = src_mod->on_load; -#ifdef HIPE - dst_mod->first_hipe_mfa = src_mod->first_hipe_mfa; -#endif } void module_start_staging(void) diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 9de610ccf7..1d6d9dd463 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -47,9 +47,6 @@ typedef struct erl_module { struct erl_module_instance curr; struct erl_module_instance old; /* protected by "old_code" rwlock */ struct erl_module_instance* on_load; -#ifdef HIPE - struct hipe_mfa_info* first_hipe_mfa; -#endif } Module; void erts_module_instance_init(struct erl_module_instance* modi); -- cgit v1.2.3 From a28149b2500a4db9b3c56b168dcd18effca87a3a Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 17 Oct 2016 11:50:01 +0200 Subject: erts: Cleanup dead code --- erts/emulator/beam/module.c | 9 --------- erts/emulator/beam/module.h | 1 - 2 files changed, 10 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index 3e622c0f3a..705dd1d43f 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -171,15 +171,6 @@ erts_put_module(Eterm mod) return put_module(mod, &module_tables[erts_staging_code_ix()]); } -Module* -erts_put_active_module(Eterm mod) -{ - ASSERT(is_atom(mod)); - //SVERK Why not? ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - - return put_module(mod, &module_tables[erts_active_code_ix()]); -} - Module *module_code(int i, ErtsCodeIndex code_ix) { return (Module*) erts_index_lookup(&module_tables[code_ix], i); diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 1d6d9dd463..8f44e11f3d 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -52,7 +52,6 @@ typedef struct erl_module { void erts_module_instance_init(struct erl_module_instance* modi); Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix); Module* erts_put_module(Eterm mod); -Module* erts_put_active_module(Eterm mod); /* only while blocked */ void init_module_table(void); void module_start_staging(void); -- cgit v1.2.3 From 6408400fc1ca9609169f6a0450f1c4671aced91c Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 26 Oct 2016 16:42:05 +0200 Subject: erts: Avoid delayed fun undef race Avoid suspending fun caller not just if purge is already done but also if purge of another module has started. Another purge of the same module again cannot happen as making current to old transition includes thread progress. --- erts/emulator/beam/beam_bif_load.c | 4 ++-- erts/emulator/beam/beam_emu.c | 2 +- erts/emulator/beam/beam_load.h | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 5969197168..8af7703f51 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1623,10 +1623,10 @@ erts_purge_state_add_fun(ErlFunEntry *fe) } Export * -erts_suspend_process_on_pending_purge_lambda(Process *c_p) +erts_suspend_process_on_pending_purge_lambda(Process *c_p, ErlFunEntry* fe) { erts_smp_mtx_lock(&purge_state.mtx); - if (is_value(purge_state.module)) { + if (purge_state.module == fe->module) { /* * The process c_p is about to call a fun in the code * that we are trying to purge. Suspend it and call diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index ef4cdf9d5a..3be5c0d24c 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -6555,7 +6555,7 @@ call_fun(Process* p, /* Current process. */ * and let it try again when the purge operation is * done (may succeed or not). */ - ep = erts_suspend_process_on_pending_purge_lambda(p); + ep = erts_suspend_process_on_pending_purge_lambda(p, fe); ASSERT(ep); } else { diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 1200bb9c6f..9be5e14e40 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -124,7 +124,8 @@ int erts_is_module_native(BeamCodeHeader* code); void erts_beam_bif_load_init(void); struct erl_fun_entry; void erts_purge_state_add_fun(struct erl_fun_entry *fe); -Export *erts_suspend_process_on_pending_purge_lambda(Process *c_p); +Export *erts_suspend_process_on_pending_purge_lambda(Process *c_p, + struct erl_fun_entry*); /* * Layout of the line table. -- cgit v1.2.3 From 3223059923314313ffff9c7b09ab0a73bf18238b Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 26 Oct 2016 17:22:32 +0200 Subject: erts: Remove unused argument to hipe_set_closure_stub --- erts/emulator/beam/beam_load.c | 2 +- erts/emulator/beam/external.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 36cf64e08d..3cd395c2c1 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -4848,7 +4848,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) } fe->address = code_ptr; #ifdef HIPE - hipe_set_closure_stub(fe, stp->lambdas[i].num_free); + hipe_set_closure_stub(fe); #endif } } diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 98df8a0726..a49b242d7c 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -3748,7 +3748,7 @@ dec_term_atom_common: funp->arity = arity; #ifdef HIPE if (funp->fe->native_address == NULL) { - hipe_set_closure_stub(funp->fe, num_free); + hipe_set_closure_stub(funp->fe); } #endif hp = factory->hp; -- cgit v1.2.3 From 21e49c8b2f3683b50750fb6a756810c8ba52b749 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 26 Oct 2016 17:44:14 +0200 Subject: erts: Fix new fun purge strategy for hipe --- erts/emulator/beam/erl_fun.c | 23 +++++++++++++++++++++-- erts/emulator/beam/erl_fun.h | 3 +++ 2 files changed, 24 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index acb0d609d0..d9118d2549 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -31,6 +31,9 @@ static Hash erts_fun_table; #include "erl_smp.h" +#ifdef HIPE +# include "hipe_mode_switch.h" +#endif static erts_smp_rwmtx_t erts_fun_table_lock; @@ -219,6 +222,10 @@ erts_fun_purge_prepare(BeamInstr* start, BeamInstr* end) fe->pend_purge_address = addr; ERTS_SMP_WRITE_MEMORY_BARRIER; fe->address = unloaded_fun; +#ifdef HIPE + fe->pend_purge_native_address = fe->native_address; + hipe_set_closure_stub(fe); +#endif erts_purge_state_add_fun(fe); } b = b->next; @@ -234,8 +241,12 @@ erts_fun_purge_abort_prepare(ErlFunEntry **funs, Uint no) for (ix = 0; ix < no; ix++) { ErlFunEntry *fe = funs[ix]; - if (fe->address == unloaded_fun) + if (fe->address == unloaded_fun) { fe->address = fe->pend_purge_address; +#ifdef HIPE + fe->native_address = fe->pend_purge_native_address; +#endif + } } } @@ -244,8 +255,12 @@ erts_fun_purge_abort_finalize(ErlFunEntry **funs, Uint no) { Uint ix; - for (ix = 0; ix < no; ix++) + for (ix = 0; ix < no; ix++) { funs[ix]->pend_purge_address = NULL; +#ifdef HIPE + funs[ix]->pend_purge_native_address = NULL; +#endif + } } void @@ -256,6 +271,9 @@ erts_fun_purge_complete(ErlFunEntry **funs, Uint no) for (ix = 0; ix < no; ix++) { ErlFunEntry *fe = funs[ix]; fe->pend_purge_address = NULL; +#ifdef HIPE + fe->pend_purge_native_address = NULL; +#endif if (erts_refc_dectest(&fe->refc, 0) == 0) erts_erase_fun_entry(fe); } @@ -324,6 +342,7 @@ fun_alloc(ErlFunEntry* template) obj->pend_purge_address = NULL; #ifdef HIPE obj->native_address = NULL; + obj->pend_purge_native_address = NULL; #endif return obj; } diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index 6ba055d92c..717bfee0fb 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -45,6 +45,9 @@ typedef struct erl_fun_entry { erts_refc_t refc; /* Reference count: One for code + one for each fun object in each process. */ BeamInstr *pend_purge_address; /* address stored during a pending purge */ +#ifdef HIPE + UWord* pend_purge_native_address; +#endif } ErlFunEntry; /* -- cgit v1.2.3 From 24485c421f15b00d0b5c97633d701373b53897a5 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 1 Nov 2016 11:40:48 +0100 Subject: erts: Add notsup error for load_nif/2 from hipe code --- erts/emulator/beam/erl_nif.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index c5606f72a1..9872a3ab81 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -3166,6 +3166,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) struct erl_module_instance* this_mi; struct erl_module_instance* prev_mi; + if (BIF_P->flags & F_HIPE_MODE) { + ret = load_nif_error(BIF_P, "notsup", "Calling load_nif from HiPE compiled " + "modules not supported"); + BIF_RET(ret); + } + encoding = erts_get_native_filename_encoding(); if (encoding == ERL_FILENAME_WIN_WCHAR) { /* Do not convert the lib name to utf-16le yet, do that in win32 specific code */ -- cgit v1.2.3 From e21a6c9ec713219b0aa2b2a4224f97e713d7337f Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Tue, 25 Oct 2016 22:05:10 +0100 Subject: Add math:fmod/2 BIF Returns the (floating point) remainder of first argument divided by second argument. --- erts/emulator/beam/bif.tab | 1 + erts/emulator/beam/erl_math.c | 5 +++++ 2 files changed, 6 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 852b1135fe..32600f4338 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -667,6 +667,7 @@ gcbif erlang:floor/1 gcbif erlang:ceil/1 bif math:floor/1 bif math:ceil/1 +bif math:fmod/2 # # Obsolete diff --git a/erts/emulator/beam/erl_math.c b/erts/emulator/beam/erl_math.c index 990fa63bd4..1f270eb55f 100644 --- a/erts/emulator/beam/erl_math.c +++ b/erts/emulator/beam/erl_math.c @@ -256,3 +256,8 @@ BIF_RETTYPE math_floor_1(BIF_ALIST_1) { return math_call_1(BIF_P, floor, BIF_ARG_1); } + +BIF_RETTYPE math_fmod_2(BIF_ALIST_2) +{ + return math_call_2(BIF_P, fmod, BIF_ARG_1, BIF_ARG_2); +} -- cgit v1.2.3 From 4c3141e8765bcaaaf998dd84b4f53059c3fe066f Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 15 Nov 2016 15:17:29 +0100 Subject: erts: Remove unused erl_crash_dump() --- erts/emulator/beam/break.c | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 3c19e82b66..aa9bed5714 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -44,7 +44,6 @@ static void process_killer(void); void do_break(void); void erl_crash_dump_v(char *file, int line, char* fmt, va_list args); -void erl_crash_dump(char* file, int line, char* fmt, ...); #ifdef DEBUG static void bin_check(void); @@ -884,12 +883,3 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) erts_fprintf(stderr,"done\n"); } -void -erl_crash_dump(char* file, int line, char* fmt, ...) -{ - va_list args; - - va_start(args, fmt); - erl_crash_dump_v(file, line, fmt, args); - va_end(args); -} -- cgit v1.2.3 From 9a1110de820988afb4e9d195a59872b214de2d26 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 17 Nov 2016 16:50:36 +0100 Subject: erts: Refactor crash dumping with cbprintf Instead of passing around a file descriptor use a function pointer to facilitate more advanced backend write logic such as size limitation or compression. --- erts/emulator/beam/atom.c | 4 +- erts/emulator/beam/atom.h | 4 +- erts/emulator/beam/beam_debug.c | 4 +- erts/emulator/beam/break.c | 97 +++++++++++++++--------------- erts/emulator/beam/dist.c | 14 ++--- erts/emulator/beam/erl_afit_alloc.c | 4 +- erts/emulator/beam/erl_alloc.c | 12 ++-- erts/emulator/beam/erl_alloc.h | 6 +- erts/emulator/beam/erl_alloc_util.c | 32 +++++----- erts/emulator/beam/erl_alloc_util.h | 10 +-- erts/emulator/beam/erl_ao_firstfit_alloc.c | 4 +- erts/emulator/beam/erl_bestfit_alloc.c | 4 +- erts/emulator/beam/erl_bif_info.c | 2 +- erts/emulator/beam/erl_db.c | 6 +- erts/emulator/beam/erl_db.h | 2 +- erts/emulator/beam/erl_db_hash.c | 4 +- erts/emulator/beam/erl_db_tree.c | 4 +- erts/emulator/beam/erl_db_util.h | 2 +- erts/emulator/beam/erl_debug.c | 10 +-- erts/emulator/beam/erl_fun.c | 4 +- erts/emulator/beam/erl_fun.h | 4 +- erts/emulator/beam/erl_goodfit_alloc.c | 4 +- erts/emulator/beam/erl_hl_timer.c | 4 +- erts/emulator/beam/erl_hl_timer.h | 2 +- erts/emulator/beam/erl_instrument.c | 74 ++++++++--------------- erts/emulator/beam/erl_instrument.h | 4 +- erts/emulator/beam/erl_nif.c | 6 +- erts/emulator/beam/erl_node_tables.c | 8 +-- erts/emulator/beam/erl_node_tables.h | 6 +- erts/emulator/beam/erl_port.h | 2 +- erts/emulator/beam/erl_process.c | 14 ++--- erts/emulator/beam/erl_process.h | 14 ++--- erts/emulator/beam/erl_process_dict.c | 6 +- erts/emulator/beam/erl_process_dict.h | 6 +- erts/emulator/beam/erl_process_dump.c | 48 +++++++-------- erts/emulator/beam/export.c | 2 +- erts/emulator/beam/export.h | 2 +- erts/emulator/beam/global.h | 16 ++--- erts/emulator/beam/hash.c | 2 +- erts/emulator/beam/hash.h | 4 +- erts/emulator/beam/index.c | 2 +- erts/emulator/beam/index.h | 2 +- erts/emulator/beam/io.c | 4 +- erts/emulator/beam/module.c | 2 +- erts/emulator/beam/module.h | 2 +- erts/emulator/beam/register.c | 2 +- erts/emulator/beam/register.h | 2 +- erts/emulator/beam/sys.h | 19 +++--- erts/emulator/beam/utils.c | 31 +++++----- 49 files changed, 252 insertions(+), 271 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index a5e778e4aa..2052afe52b 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -68,7 +68,7 @@ static Uint atom_space; /* Amount of atom text space used */ /* * Print info about atom tables */ -void atom_info(int to, void *to_arg) +void atom_info(fmtfn_t to, void *to_arg) { int lock = !ERTS_IS_CRASH_DUMPING; if (lock) @@ -470,7 +470,7 @@ init_atom_table(void) } void -dump_atoms(int to, void *to_arg) +dump_atoms(fmtfn_t to, void *to_arg) { int i = erts_atom_table.entries; diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h index ae60904785..a82efabb9f 100644 --- a/erts/emulator/beam/atom.h +++ b/erts/emulator/beam/atom.h @@ -136,8 +136,8 @@ Eterm erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc); int atom_erase(byte*, int); int atom_static_put(byte*, int); void init_atom_table(void); -void atom_info(int, void *); -void dump_atoms(int, void *); +void atom_info(fmtfn_t, void *); +void dump_atoms(fmtfn_t, void *); int erts_atom_get(const char* name, int len, Eterm* ap, ErtsAtomEncoding enc); void erts_atom_get_text_space_sizes(Uint *reserved, Uint *used); #endif diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index a4ad3e7886..21d336049f 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -50,7 +50,7 @@ void dbg_bt(Process* p, Eterm* sp); void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg); -static int print_op(int to, void *to_arg, int op, int size, BeamInstr* addr); +static int print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr); BIF_RETTYPE erts_debug_same_2(BIF_ALIST_2) @@ -377,7 +377,7 @@ dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg) } static int -print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) +print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) { int i; BeamInstr tag; diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index aa9bed5714..2e8a384dec 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -49,17 +49,17 @@ void erl_crash_dump_v(char *file, int line, char* fmt, va_list args); static void bin_check(void); #endif -static void print_garb_info(int to, void *to_arg, Process* p); +static void print_garb_info(fmtfn_t to, void *to_arg, Process* p); #ifdef OPPROF static void dump_frequencies(void); #endif -static void dump_attributes(int to, void *to_arg, byte* ptr, int size); +static void dump_attributes(fmtfn_t to, void *to_arg, byte* ptr, int size); extern char* erts_system_version[]; static void -port_info(int to, void *to_arg) +port_info(fmtfn_t to, void *to_arg) { int i, max = erts_ptab_max(&erts_port); for (i = 0; i < max; i++) { @@ -70,7 +70,7 @@ port_info(int to, void *to_arg) } void -process_info(int to, void *to_arg) +process_info(fmtfn_t to, void *to_arg) { int i, max = erts_ptab_max(&erts_proc); for (i = 0; i < max; i++) { @@ -147,14 +147,14 @@ process_killer(void) typedef struct { int is_first; - int to; + fmtfn_t to; void *to_arg; } PrintMonitorContext; static void doit_print_link(ErtsLink *lnk, void *vpcontext) { PrintMonitorContext *pcontext = vpcontext; - int to = pcontext->to; + fmtfn_t to = pcontext->to; void *to_arg = pcontext->to_arg; if (pcontext->is_first) { @@ -169,7 +169,7 @@ static void doit_print_link(ErtsLink *lnk, void *vpcontext) static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext) { PrintMonitorContext *pcontext = vpcontext; - int to = pcontext->to; + fmtfn_t to = pcontext->to; void *to_arg = pcontext->to_arg; char *prefix = ", "; @@ -196,7 +196,7 @@ static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext) /* Display info about an individual Erlang process */ void -print_process_info(int to, void *to_arg, Process *p) +print_process_info(fmtfn_t to, void *to_arg, Process *p) { time_t approx_started; int garbing = 0; @@ -299,7 +299,7 @@ print_process_info(int to, void *to_arg, Process *p) /* display the links only if there are any*/ if (ERTS_P_LINKS(p) || ERTS_P_MONITORS(p)) { - PrintMonitorContext context = {1,to}; + PrintMonitorContext context = {1, to, to_arg}; erts_print(to, to_arg,"Link list: ["); erts_doforall_links(ERTS_P_LINKS(p), &doit_print_link, &context); erts_doforall_monitors(ERTS_P_MONITORS(p), &doit_print_monitor, &context); @@ -347,7 +347,7 @@ print_process_info(int to, void *to_arg, Process *p) } static void -print_garb_info(int to, void *to_arg, Process* p) +print_garb_info(fmtfn_t to, void *to_arg, Process* p) { #ifdef ERTS_SMP /* ERTS_SMP: A scheduler is probably concurrently doing gc... */ @@ -364,7 +364,7 @@ print_garb_info(int to, void *to_arg, Process* p) } void -info(int to, void *to_arg) +info(fmtfn_t to, void *to_arg) { erts_memory(&to, to_arg, NULL, THE_NON_VALUE); atom_info(to, to_arg); @@ -380,7 +380,7 @@ info(int to, void *to_arg) } void -loaded(int to, void *to_arg) +loaded(fmtfn_t to, void *to_arg) { int i; int old = 0; @@ -477,7 +477,7 @@ loaded(int to, void *to_arg) static void -dump_attributes(int to, void *to_arg, byte* ptr, int size) +dump_attributes(fmtfn_t to, void *to_arg, byte* ptr, int size) { while (size-- > 0) { erts_print(to, to_arg, "%02X", *ptr++); @@ -678,6 +678,8 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) int secs; int env_erl_crash_dump_seconds_set = 1; int i; + fmtfn_t to = &erts_write_fd; + void* to_arg; if (ERTS_SOMEONE_IS_CRASH_DUMPING) return; @@ -768,39 +770,40 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) fd = open(dumpname,O_WRONLY | O_CREAT | O_TRUNC,0640); if (fd < 0) return; /* Can't create the crash dump, skip it */ + to_arg = (void*)&fd; time(&now); - erts_fdprintf(fd, "=erl_crash_dump:0.3\n%s", ctime(&now)); + erts_cbprintf(to, to_arg, "=erl_crash_dump:0.3\n%s", ctime(&now)); if (file != NULL) - erts_fdprintf(fd, "The error occurred in file %s, line %d\n", file, line); + erts_cbprintf(to, to_arg, "The error occurred in file %s, line %d\n", file, line); if (fmt != NULL && *fmt != '\0') { - erts_fdprintf(fd, "Slogan: "); - erts_vfdprintf(fd, fmt, args); + erts_cbprintf(to, to_arg, "Slogan: "); + erts_vcbprintf(to, to_arg, fmt, args); } - erts_fdprintf(fd, "System version: "); - erts_print_system_version(fd, NULL, NULL); + erts_cbprintf(to, to_arg, "System version: "); + erts_print_system_version(to, to_arg, NULL); #if ERTS_SAVED_COMPILE_TIME - erts_fdprintf(fd, "%s\n", "Compiled: " ERLANG_COMPILE_DATE); + erts_cbprintf(to, to_arg, "%s\n", "Compiled: " ERLANG_COMPILE_DATE); #endif - erts_fdprintf(fd, "Taints: "); - erts_print_nif_taints(fd, NULL); - erts_fdprintf(fd, "Atoms: %d\n", atom_table_size()); + erts_cbprintf(to, to_arg, "Taints: "); + erts_print_nif_taints(to, to_arg); + erts_cbprintf(to, to_arg, "Atoms: %d\n", atom_table_size()); #ifdef USE_THREADS /* We want to note which thread it was that called erts_exit */ if (erts_get_scheduler_data()) { - erts_fdprintf(fd, "Calling Thread: scheduler:%d\n", + erts_cbprintf(to, to_arg, "Calling Thread: scheduler:%d\n", erts_get_scheduler_data()->no); } else { if (!erts_thr_getname(erts_thr_self(), dumpnamebuf, MAXPATHLEN)) - erts_fdprintf(fd, "Calling Thread: %s\n", dumpnamebuf); + erts_cbprintf(to, to_arg, "Calling Thread: %s\n", dumpnamebuf); else - erts_fdprintf(fd, "Calling Thread: %p\n", erts_thr_self()); + erts_cbprintf(to, to_arg, "Calling Thread: %p\n", erts_thr_self()); } #else - erts_fdprintf(fd, "Calling Thread: scheduler:1\n"); + erts_cbprintf(to, to_arg, "Calling Thread: scheduler:1\n"); #endif #if defined(ERTS_HAVE_TRY_CATCH) @@ -815,8 +818,8 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) */ for (i = 0; i < erts_no_schedulers; i++) { ERTS_SYS_TRY_CATCH( - erts_print_scheduler_info(fd, NULL, ERTS_SCHEDULER_IX(i)), - erts_fdprintf(fd, "** crashed **\n")); + erts_print_scheduler_info(to, to_arg, ERTS_SCHEDULER_IX(i)), + erts_cbprintf(to, to_arg, "** crashed **\n")); } #endif @@ -847,38 +850,38 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) #ifndef ERTS_HAVE_TRY_CATCH /* This is safe to call here, as all schedulers are blocked */ for (i = 0; i < erts_no_schedulers; i++) { - erts_print_scheduler_info(fd, NULL, ERTS_SCHEDULER_IX(i)); + erts_print_scheduler_info(to, to_arg, ERTS_SCHEDULER_IX(i)); } #endif - info(fd, NULL); /* General system info */ + info(to, to_arg); /* General system info */ if (erts_ptab_initialized(&erts_proc)) - process_info(fd, NULL); /* Info about each process and port */ - db_info(fd, NULL, 0); - erts_print_bif_timer_info(fd, NULL); - distribution_info(fd, NULL); - erts_fdprintf(fd, "=loaded_modules\n"); - loaded(fd, NULL); - erts_dump_fun_entries(fd, NULL); - erts_deep_process_dump(fd, NULL); - erts_fdprintf(fd, "=atoms\n"); - dump_atoms(fd, NULL); + process_info(to, to_arg); /* Info about each process and port */ + db_info(to, to_arg, 0); + erts_print_bif_timer_info(to, to_arg); + distribution_info(to, to_arg); + erts_cbprintf(to, to_arg, "=loaded_modules\n"); + loaded(to, to_arg); + erts_dump_fun_entries(to, to_arg); + erts_deep_process_dump(to, to_arg); + erts_cbprintf(to, to_arg, "=atoms\n"); + dump_atoms(to, to_arg); /* Keep the instrumentation data at the end of the dump */ if (erts_instr_memory_map || erts_instr_stat) { - erts_fdprintf(fd, "=instr_data\n"); + erts_cbprintf(to, to_arg, "=instr_data\n"); if (erts_instr_stat) { - erts_fdprintf(fd, "=memory_status\n"); - erts_instr_dump_stat_to_fd(fd, 0); + erts_cbprintf(to, to_arg, "=memory_status\n"); + erts_instr_dump_stat_to(to, to_arg, 0); } if (erts_instr_memory_map) { - erts_fdprintf(fd, "=memory_map\n"); - erts_instr_dump_memory_map_to_fd(fd); + erts_cbprintf(to, to_arg, "=memory_map\n"); + erts_instr_dump_memory_map_to(to, to_arg); } } - erts_fdprintf(fd, "=end\n"); + erts_cbprintf(to, to_arg, "=end\n"); close(fd); erts_fprintf(stderr,"done\n"); } diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 09c83f1117..d79245e0e6 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -2402,13 +2402,13 @@ erts_kill_dist_connection(DistEntry *dep, Uint32 connection_id) } struct print_to_data { - int to; + fmtfn_t to; void *arg; }; static void doit_print_monitor_info(ErtsMonitor *mon, void *vptdp) { - int to = ((struct print_to_data *) vptdp)->to; + fmtfn_t to = ((struct print_to_data *) vptdp)->to; void *arg = ((struct print_to_data *) vptdp)->arg; Process *rp; ErtsMonitor *rmon; @@ -2431,7 +2431,7 @@ static void doit_print_monitor_info(ErtsMonitor *mon, void *vptdp) } } -static void print_monitor_info(int to, void *arg, ErtsMonitor *mon) +static void print_monitor_info(fmtfn_t to, void *arg, ErtsMonitor *mon) { struct print_to_data ptd = {to, arg}; erts_doforall_monitors(mon,&doit_print_monitor_info,&ptd); @@ -2457,7 +2457,7 @@ static void doit_print_link_info(ErtsLink *lnk, void *vptdp) } } -static void print_link_info(int to, void *arg, ErtsLink *lnk) +static void print_link_info(fmtfn_t to, void *arg, ErtsLink *lnk) { struct print_to_data ptd = {to, arg}; erts_doforall_links(lnk, &doit_print_link_info, (void *) &ptd); @@ -2478,7 +2478,7 @@ static void doit_print_nodelink_info(ErtsLink *lnk, void *vpcontext) "Remote monitoring: %T %T\n", lnk->pid, pcontext->sysname); } -static void print_nodelink_info(int to, void *arg, ErtsLink *lnk, Eterm sysname) +static void print_nodelink_info(fmtfn_t to, void *arg, ErtsLink *lnk, Eterm sysname) { PrintNodeLinkContext context = {{to, arg}, sysname}; erts_doforall_links(lnk, &doit_print_nodelink_info, &context); @@ -2486,7 +2486,7 @@ static void print_nodelink_info(int to, void *arg, ErtsLink *lnk, Eterm sysname) static int -info_dist_entry(int to, void *arg, DistEntry *dep, int visible, int connected) +info_dist_entry(fmtfn_t to, void *arg, DistEntry *dep, int visible, int connected) { if (visible && connected) { @@ -2535,7 +2535,7 @@ info_dist_entry(int to, void *arg, DistEntry *dep, int visible, int connected) return 0; } -int distribution_info(int to, void *arg) /* Called by break handler */ +int distribution_info(fmtfn_t to, void *arg) /* Called by break handler */ { DistEntry *dep; diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c index eda3ad870a..4ebe37ee1d 100644 --- a/erts/emulator/beam/erl_afit_alloc.c +++ b/erts/emulator/beam/erl_afit_alloc.c @@ -54,7 +54,7 @@ static void link_free_block (Allctr_t *, Block_t *); static void unlink_free_block (Allctr_t *, Block_t *); -static Eterm info_options (Allctr_t *, char *, int *, +static Eterm info_options (Allctr_t *, char *, fmtfn_t *, void *arg, Uint **, Uint *); static void init_atoms (void); @@ -227,7 +227,7 @@ add_2tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2) static Eterm info_options(Allctr_t *allctr, char *prefix, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 3c2c9def3b..214fb1f2af 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -2113,7 +2113,7 @@ add_fix_values(UWord *ap, UWord *up, ErtsAlcUFixInfo_t *fi, ErtsAlcType_t type) } Eterm -erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) +erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) { /* * NOTE! When updating this function, make sure to also update @@ -2476,7 +2476,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) if (print_to_p) { int i; - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; /* Print result... */ @@ -2530,7 +2530,7 @@ struct aa_values { }; Eterm -erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) +erts_allocated_areas(fmtfn_t *print_to_p, void *print_to_arg, void *proc) { #define MAX_AA_VALUES (24) struct aa_values values[MAX_AA_VALUES]; @@ -2665,7 +2665,7 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) if (print_to_p) { /* Print result... */ - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; erts_print(to, arg, "=allocated_areas\n"); @@ -2779,7 +2779,7 @@ erts_alloc_util_allocators(void *proc) } void -erts_allocator_info(int to, void *arg) +erts_allocator_info(fmtfn_t to, void *arg) { ErtsAlcType_t a; @@ -3110,7 +3110,7 @@ reply_alloc_info(void *vair) Eterm (*info_func)(Allctr_t *, int, int, - int *, + fmtfn_t *, void *, Uint **, Uint *) = (air->only_sz diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index 925a081a02..56a3b73bf9 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -69,11 +69,11 @@ void *erts_sys_aligned_realloc(UWord alignment, void *ptr, UWord size, UWord old void erts_sys_aligned_free(UWord alignment, void *ptr); #endif -Eterm erts_memory(int *, void *, void *, Eterm); -Eterm erts_allocated_areas(int *, void *, void *); +Eterm erts_memory(fmtfn_t *, void *, void *, Eterm); +Eterm erts_allocated_areas(fmtfn_t *, void *, void *); Eterm erts_alloc_util_allocators(void *proc); -void erts_allocator_info(int, void *); +void erts_allocator_info(fmtfn_t, void *); Eterm erts_allocator_options(void *proc); struct process; diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 2995f2f822..f4ca689040 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -4473,7 +4473,7 @@ add_fix_types(Allctr_t *allctr, int internal, Uint **hpp, Uint *szp, static Eterm sz_info_fix(Allctr_t *allctr, int internal, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -4494,7 +4494,7 @@ sz_info_fix(Allctr_t *allctr, UWord used = fix->type_size * fix->u.cpool.used; if (print_to_p) { - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; erts_print(to, arg, @@ -4522,7 +4522,7 @@ sz_info_fix(Allctr_t *allctr, UWord used = fix->type_size*fix->u.nocpool.used; if (print_to_p) { - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; erts_print(to, arg, @@ -4548,7 +4548,7 @@ static Eterm sz_info_carriers(Allctr_t *allctr, CarriersStats_t *cs, char *prefix, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -4557,7 +4557,7 @@ sz_info_carriers(Allctr_t *allctr, UWord curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; if (print_to_p) { - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; erts_print(to, arg, @@ -4598,7 +4598,7 @@ static Eterm info_cpool(Allctr_t *allctr, int sz_only, char *prefix, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -4615,7 +4615,7 @@ info_cpool(Allctr_t *allctr, } if (print_to_p) { - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; if (!sz_only) erts_print(to, arg, "%sblocks: %bpu\n", prefix, nob); @@ -4652,7 +4652,7 @@ static Eterm info_carriers(Allctr_t *allctr, CarriersStats_t *cs, char *prefix, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -4664,7 +4664,7 @@ info_carriers(Allctr_t *allctr, curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; if (print_to_p) { - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; erts_print(to, arg, @@ -4790,7 +4790,7 @@ make_name_atoms(Allctr_t *allctr) static Eterm info_calls(Allctr_t *allctr, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -4807,7 +4807,7 @@ info_calls(Allctr_t *allctr, erts_print(TO, TOA, "%s%s calls: %b64u\n",PRFX,NAME,CC) char *prefix = allctr->name_prefix; - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; PRINT_CC_5(to, arg, prefix, "alloc", allctr->calls.this_alloc); @@ -4883,7 +4883,7 @@ info_calls(Allctr_t *allctr, static Eterm info_options(Allctr_t *allctr, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -5035,7 +5035,7 @@ reset_max_values(CarriersStats_t *cs) \* */ Eterm -erts_alcu_au_info_options(int *print_to_p, void *print_to_arg, +erts_alcu_au_info_options(fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) { Eterm res = THE_NON_VALUE; @@ -5078,7 +5078,7 @@ erts_alcu_au_info_options(int *print_to_p, void *print_to_arg, Eterm erts_alcu_info_options(Allctr_t *allctr, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -5110,7 +5110,7 @@ Eterm erts_alcu_sz_info(Allctr_t *allctr, int internal, int begin_max_period, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -5196,7 +5196,7 @@ Eterm erts_alcu_info(Allctr_t *allctr, int internal, int begin_max_period, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index f50f09907a..81180382af 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -178,10 +178,10 @@ void * erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t, void *, void *, Uint); void erts_alcu_free_thr_pref(ErtsAlcType_t, void *, void *); #endif #endif -Eterm erts_alcu_au_info_options(int *, void *, Uint **, Uint *); -Eterm erts_alcu_info_options(Allctr_t *, int *, void *, Uint **, Uint *); -Eterm erts_alcu_sz_info(Allctr_t *, int, int, int *, void *, Uint **, Uint *); -Eterm erts_alcu_info(Allctr_t *, int, int, int *, void *, Uint **, Uint *); +Eterm erts_alcu_au_info_options(fmtfn_t *, void *, Uint **, Uint *); +Eterm erts_alcu_info_options(Allctr_t *, fmtfn_t *, void *, Uint **, Uint *); +Eterm erts_alcu_sz_info(Allctr_t *, int, int, fmtfn_t *, void *, Uint **, Uint *); +Eterm erts_alcu_info(Allctr_t *, int, int, fmtfn_t *, void *, Uint **, Uint *); void erts_alcu_init(AlcUInit_t *); void erts_alcu_current_size(Allctr_t *, AllctrSize_t *, ErtsAlcUFixInfo_t *, int); @@ -586,7 +586,7 @@ struct Allctr_t_ { Block_t *, Uint); void (*link_free_block) (Allctr_t *, Block_t *); void (*unlink_free_block) (Allctr_t *, Block_t *); - Eterm (*info_options) (Allctr_t *, char *, int *, + Eterm (*info_options) (Allctr_t *, char *, fmtfn_t *, void *, Uint **, Uint *); Uint (*get_next_mbc_size) (Allctr_t *); diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 7e239d1f5d..05ba1f9891 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -224,7 +224,7 @@ static AOFF_RBTree_t* rbt_search(AOFF_RBTree_t* root, Uint size); static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node); #endif -static Eterm info_options(Allctr_t *, char *, int *, void *, Uint **, Uint *); +static Eterm info_options(Allctr_t *, char *, fmtfn_t *, void *, Uint **, Uint *); static void init_atoms(void); @@ -1014,7 +1014,7 @@ add_2tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2) static Eterm info_options(Allctr_t *allctr, char *prefix, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c index 379cee39a1..6173c408e1 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.c +++ b/erts/emulator/beam/erl_bestfit_alloc.c @@ -104,7 +104,7 @@ static void bf_link_free_block (Allctr_t *, Block_t *); static ERTS_INLINE void bf_unlink_free_block (Allctr_t *, Block_t *); -static Eterm info_options (Allctr_t *, char *, int *, +static Eterm info_options (Allctr_t *, char *, fmtfn_t *, void *, Uint **, Uint *); static void init_atoms (void); @@ -921,7 +921,7 @@ add_2tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2) static Eterm info_options(Allctr_t *allctr, char *prefix, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 29ba12dfdb..735aabbee3 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -317,7 +317,7 @@ make_link_list(Process *p, ErtsLink *root, Eterm tail) } int -erts_print_system_version(int to, void *arg, Process *c_p) +erts_print_system_version(fmtfn_t to, void *arg, Process *c_p) { int i, rc = -1; char *rc_str = ""; diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 128a7b3865..dceadc46f4 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -201,7 +201,7 @@ static int free_table_cont(Process *p, DbTable *tb, int first, int clean_meta_tab); -static void print_table(int to, void *to_arg, int show, DbTable* tb); +static void print_table(fmtfn_t to, void *to_arg, int show, DbTable* tb); static BIF_RETTYPE ets_select_delete_1(BIF_ALIST_1); static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1); static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1); @@ -3871,7 +3871,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) return ret; } -static void print_table(int to, void *to_arg, int show, DbTable* tb) +static void print_table(fmtfn_t to, void *to_arg, int show, DbTable* tb) { erts_print(to, to_arg, "Table: %T\n", tb->common.id); erts_print(to, to_arg, "Name: %T\n", tb->common.the_name); @@ -3891,7 +3891,7 @@ static void print_table(int to, void *to_arg, int show, DbTable* tb) erts_print(to, to_arg, "Read Concurrency: %T\n", table_info(NULL, tb, am_read_concurrency)); } -void db_info(int to, void *to_arg, int show) /* Called by break handler */ +void db_info(fmtfn_t to, void *to_arg, int show) /* Called by break handler */ { int i; for (i=0; i < db_max_tabs; i++) diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index 1d26c49652..b0508f2e74 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -74,7 +74,7 @@ typedef enum { void init_db(ErtsDbSpinCount); int erts_db_process_exiting(Process *, ErtsProcLocks); -void db_info(int, void *, int); +void db_info(fmtfn_t, void *, int); void erts_db_foreach_table(void (*)(DbTable *, void *), void *); void erts_db_foreach_offheap(DbTable *, void (*func)(ErlOffHeap *, void *), diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 5e6fe4f460..390369fdb9 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -437,7 +437,7 @@ static int db_select_count_continue_hash(Process *p, DbTable *tbl, static int db_select_delete_continue_hash(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); static int db_take_hash(Process *, DbTable *, Eterm, Eterm *); -static void db_print_hash(int to, +static void db_print_hash(fmtfn_t to, void *to_arg, int show, DbTable *tbl); @@ -2155,7 +2155,7 @@ int db_mark_all_deleted_hash(DbTable *tbl) /* Display hash table contents (for dump) */ -static void db_print_hash(int to, void *to_arg, int show, DbTable *tbl) +static void db_print_hash(fmtfn_t to, void *to_arg, int show, DbTable *tbl) { DbTableHash *tb = &tbl->hash; DbHashStats stats; diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 02d211a4bb..dd9403e132 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -385,7 +385,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, static int db_select_delete_continue_tree(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); static int db_take_tree(Process *, DbTable *, Eterm, Eterm *); -static void db_print_tree(int to, void *to_arg, +static void db_print_tree(fmtfn_t to, void *to_arg, int show, DbTable *tbl); static int db_free_table_tree(DbTable *tbl); @@ -1740,7 +1740,7 @@ static int db_take_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) /* Display tree contents (for dump) */ -static void db_print_tree(int to, void *to_arg, +static void db_print_tree(fmtfn_t to, void *to_arg, int show, DbTable *tbl) { diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 4acedbfed0..49e5f6b4cf 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -175,7 +175,7 @@ typedef struct db_table_method int (*db_free_table)(DbTable* db /* [in out] */ ); int (*db_free_table_continue)(DbTable* db); /* [in out] */ - void (*db_print)(int to, + void (*db_print)(fmtfn_t to, void* to_arg, int show, DbTable* tb /* [in out] */ ); diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c index 3e3bfa03a2..3526bb684d 100644 --- a/erts/emulator/beam/erl_debug.c +++ b/erts/emulator/beam/erl_debug.c @@ -60,10 +60,10 @@ static const char dashes[PTR_SIZE+3] = { void pps(Process*, Eterm*); void ptd(Process*, Eterm); -void paranoid_display(int, void*, Process*, Eterm); +void paranoid_display(fmtfn_t, void*, Process*, Eterm); static int dcount; -static int pdisplay1(int to, void *to_arg, Process* p, Eterm obj); +static int pdisplay1(fmtfn_t to, void *to_arg, Process* p, Eterm obj); void ptd(Process* p, Eterm x) { @@ -77,14 +77,14 @@ void ptd(Process* p, Eterm x) */ void -paranoid_display(int to, void *to_arg, Process* p, Eterm obj) +paranoid_display(fmtfn_t to, void *to_arg, Process* p, Eterm obj) { dcount = 100000; pdisplay1(to, to_arg, p, obj); } static int -pdisplay1(int to, void *to_arg, Process* p, Eterm obj) +pdisplay1(fmtfn_t to, void *to_arg, Process* p, Eterm obj) { int i, k; Eterm* nobj; @@ -201,7 +201,7 @@ pdisplay1(int to, void *to_arg, Process* p, Eterm obj) void pps(Process* p, Eterm* stop) { - int to = ERTS_PRINT_STDOUT; + fmtfn_t to = ERTS_PRINT_STDOUT; void *to_arg = NULL; Eterm* sp = STACK_START(p) - 1; diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index c639ba623f..dad77071b3 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -74,7 +74,7 @@ erts_init_fun_table(void) } void -erts_fun_info(int to, void *to_arg) +erts_fun_info(fmtfn_t to, void *to_arg) { int lock = !ERTS_IS_CRASH_DUMPING; if (lock) @@ -264,7 +264,7 @@ erts_fun_purge_complete(ErlFunEntry **funs, Uint no) } void -erts_dump_fun_entries(int to, void *to_arg) +erts_dump_fun_entries(fmtfn_t to, void *to_arg) { int limit; HashBucket** bucket; diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index 73c3e19c1c..caa55c730c 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -71,7 +71,7 @@ typedef struct erl_fun_thing { #define ERL_FUN_SIZE ((sizeof(ErlFunThing)/sizeof(Eterm))-1) void erts_init_fun_table(void); -void erts_fun_info(int, void *); +void erts_fun_info(fmtfn_t, void *); int erts_fun_table_sz(void); ErlFunEntry* erts_put_fun_entry(Eterm mod, int uniq, int index); @@ -86,6 +86,6 @@ void erts_fun_purge_prepare(BeamInstr* start, BeamInstr* end); void erts_fun_purge_abort_prepare(ErlFunEntry **funs, Uint no); void erts_fun_purge_abort_finalize(ErlFunEntry **funs, Uint no); void erts_fun_purge_complete(ErlFunEntry **funs, Uint no); -void erts_dump_fun_entries(int, void *); +void erts_dump_fun_entries(fmtfn_t, void *); #endif diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index 223ba193da..50aa41b4d2 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -168,7 +168,7 @@ static Block_t * get_free_block (Allctr_t *, Uint, static void link_free_block (Allctr_t *, Block_t *); static void unlink_free_block (Allctr_t *, Block_t *); static void update_last_aux_mbc (Allctr_t *, Carrier_t *); -static Eterm info_options (Allctr_t *, char *, int *, +static Eterm info_options (Allctr_t *, char *, fmtfn_t *, void *, Uint **, Uint *); static void init_atoms (void); @@ -551,7 +551,7 @@ add_2tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2) static Eterm info_options(Allctr_t *allctr, char *prefix, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index f1bef28186..d29d079fc5 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -2851,7 +2851,7 @@ erts_read_port_timer(Port *c_prt) */ typedef struct { - int to; + fmtfn_t to; void *to_arg; ErtsMonotonicTime now; } ErtsBTMPrint; @@ -2881,7 +2881,7 @@ btm_print(ErtsHLTimer *tmr, void *vbtmp) } void -erts_print_bif_timer_info(int to, void *to_arg) +erts_print_bif_timer_info(fmtfn_t to, void *to_arg) { ErtsBTMPrint btmp; int six; diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h index 0931bb8965..705be94532 100644 --- a/erts/emulator/beam/erl_hl_timer.h +++ b/erts/emulator/beam/erl_hl_timer.h @@ -72,7 +72,7 @@ erts_handle_canceled_timers(void *vesdp, #endif Uint erts_bif_timer_memory_size(void); -void erts_print_bif_timer_info(int to, void *to_arg); +void erts_print_bif_timer_info(fmtfn_t to, void *to_arg); void erts_debug_bif_timer_foreach(void (*func)(Eterm, Eterm, diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c index f84c63e7a4..4d4defd8b5 100644 --- a/erts/emulator/beam/erl_instrument.c +++ b/erts/emulator/beam/erl_instrument.c @@ -539,7 +539,7 @@ map_stat_free(ErtsAlcType_t n, void *extra, void *ptr) } -static void dump_memory_map_to_stream(FILE *fp) +static void dump_memory_map_to_stream(fmtfn_t to, void* to_arg) { ErtsAlcType_t n; MapStatBlock_t *bp; @@ -551,7 +551,7 @@ static void dump_memory_map_to_stream(FILE *fp) /* Write header */ - fprintf(fp, + erts_cbprintf(to, to_arg, "{instr_hdr,\n" " %lu,\n" " %lu,\n" @@ -574,7 +574,7 @@ static void dump_memory_map_to_stream(FILE *fp) else astr = ERTS_ALC_A2AD(ERTS_ALC_A_SYSTEM); - fprintf(fp, + erts_cbprintf(to, to_arg, "%s{%s,%s,%s}%s", (n == ERTS_ALC_N_MIN) ? "" : " ", ERTS_ALC_N2TD(n), @@ -583,12 +583,12 @@ static void dump_memory_map_to_stream(FILE *fp) (n == ERTS_ALC_N_MAX) ? "" : ",\n"); } - fprintf(fp, "}}.\n"); + erts_cbprintf(to, to_arg, "}}.\n"); /* Write memory data */ for (bp = mem_anchor; bp; bp = bp->next) { if (is_internal_pid(bp->pid)) - fprintf(fp, + erts_cbprintf(to, to_arg, "{%lu, %lu, %lu, {%lu,%lu,%lu}}.\n", (UWord) bp->type_no, (UWord) bp->mem, @@ -597,7 +597,7 @@ static void dump_memory_map_to_stream(FILE *fp) (UWord) pid_number(bp->pid), (UWord) pid_serial(bp->pid)); else - fprintf(fp, + erts_cbprintf(to, to_arg, "{%lu, %lu, %lu, undefined}.\n", (UWord) bp->type_no, (UWord) bp->mem, @@ -608,40 +608,29 @@ static void dump_memory_map_to_stream(FILE *fp) erts_mtx_unlock(&instr_mutex); } -int erts_instr_dump_memory_map_to_fd(int fd) +int erts_instr_dump_memory_map_to(fmtfn_t to, void* to_arg) { - char buf[BUFSIZ]; - FILE *f; - if (!erts_instr_memory_map) return 0; - f = fdopen(fd, "w"); - if (f == NULL) - return 0; - - /* Avoid allocating memory; we may have run out of it at this point. */ - setbuf(f, buf); - - dump_memory_map_to_stream(f); - fflush(f); + dump_memory_map_to_stream(to, to_arg); return 1; } int erts_instr_dump_memory_map(const char *name) { - FILE *f; + int fd; if (!erts_instr_memory_map) return 0; - f = fopen(name, "w"); - if (f == NULL) + fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, 0640); + if (fd < 0) return 0; - dump_memory_map_to_stream(f); + dump_memory_map_to_stream(erts_write_fd, (void*)&fd); - fclose(f); + close(fd); return 1; } @@ -998,19 +987,19 @@ erts_instr_get_stat(Process *proc, Eterm what, int begin_max_period) } static void -dump_stat_to_stream(FILE *fp, int begin_max_period) +dump_stat_to_stream(fmtfn_t to, void* to_arg, int begin_max_period) { ErtsAlcType_t i, a_max, a_min; erts_mtx_lock(&instr_mutex); - fprintf(fp, + erts_cbprintf(to, to_arg, "{instr_vsn,%lu}.\n", (unsigned long) ERTS_INSTR_VSN); update_max_ever_values(&stats->tot, 0, 0); - fprintf(fp, + erts_cbprintf(to, to_arg, "{total,[{total,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}]}.\n", (UWord) stats->tot.size, (UWord) stats->tot.max_size, @@ -1038,7 +1027,7 @@ dump_stat_to_stream(FILE *fp, int begin_max_period) for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { if (erts_allctrs_info[i].enabled) { - fprintf(fp, + erts_cbprintf(to, to_arg, "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s", i == a_min ? "{allocators,\n [" : " ", ERTS_ALC_A2AD(i), @@ -1055,7 +1044,7 @@ dump_stat_to_stream(FILE *fp, int begin_max_period) update_max_ever_values(stats->c, ERTS_ALC_C_MIN, ERTS_ALC_C_MAX); for (i = ERTS_ALC_C_MIN; i <= ERTS_ALC_C_MAX; i++) { - fprintf(fp, + erts_cbprintf(to, to_arg, "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s", i == ERTS_ALC_C_MIN ? "{classes,\n [" : " ", ERTS_ALC_C2CD(i), @@ -1071,7 +1060,7 @@ dump_stat_to_stream(FILE *fp, int begin_max_period) update_max_ever_values(stats->n, ERTS_ALC_N_MIN, ERTS_ALC_N_MAX); for (i = ERTS_ALC_N_MIN; i <= ERTS_ALC_N_MAX; i++) { - fprintf(fp, + erts_cbprintf(to, to_arg, "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s", i == ERTS_ALC_N_MIN ? "{types,\n [" : " ", ERTS_ALC_N2TD(i), @@ -1095,40 +1084,29 @@ dump_stat_to_stream(FILE *fp, int begin_max_period) } -int erts_instr_dump_stat_to_fd(int fd, int begin_max_period) +int erts_instr_dump_stat_to(fmtfn_t to, void* to_arg, int begin_max_period) { - char buf[BUFSIZ]; - FILE *fp; - if (!erts_instr_stat) return 0; - fp = fdopen(fd, "w"); - if (fp == NULL) - return 0; - - /* Avoid allocating memory; we may have run out of it at this point. */ - setbuf(fp, buf); - - dump_stat_to_stream(fp, begin_max_period); - fflush(fp); + dump_stat_to_stream(to, to_arg, begin_max_period); return 1; } int erts_instr_dump_stat(const char *name, int begin_max_period) { - FILE *file; + int fd; if (!erts_instr_stat) return 0; - file = fopen(name, "w"); - if (file == NULL) + fd = open(name, O_WRONLY | O_CREAT | O_TRUNC,0640); + if (fd < 0) return 0; - dump_stat_to_stream(file, begin_max_period); + dump_stat_to_stream(erts_write_fd, (void*)&fd, begin_max_period); - fclose(file); + close(fd); return 1; } diff --git a/erts/emulator/beam/erl_instrument.h b/erts/emulator/beam/erl_instrument.h index 1f04c91d5e..351172b2fa 100644 --- a/erts/emulator/beam/erl_instrument.h +++ b/erts/emulator/beam/erl_instrument.h @@ -29,10 +29,10 @@ extern int erts_instr_memory_map; extern int erts_instr_stat; Uint erts_instr_init(int stat, int map_stat); -int erts_instr_dump_memory_map_to_fd(int fd); +int erts_instr_dump_memory_map_to(fmtfn_t to, void* to_arg); int erts_instr_dump_memory_map(const char *name); Eterm erts_instr_get_memory_map(Process *process); -int erts_instr_dump_stat_to_fd(int fd, int begin_max_period); +int erts_instr_dump_stat_to(fmtfn_t to, void* to_arg, int begin_max_period); int erts_instr_dump_stat(const char *name, int begin_max_period); Eterm erts_instr_get_stat(Process *proc, Eterm what, int begin_max_period); Eterm erts_instr_get_type_info(Process *proc); diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 3a547982da..20b6c76cb7 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -3068,16 +3068,16 @@ Eterm erts_nif_taints(Process* p) return list; } -void erts_print_nif_taints(int to, void* to_arg) +void erts_print_nif_taints(fmtfn_t to, void* to_arg) { struct tainted_module_t* t; const char* delim = ""; for (t=first_tainted_module ; t!=NULL; t=t->next) { const Atom* atom = atom_tab(atom_val(t->module_atom)); - erts_print(to,to_arg,"%s%.*s", delim, atom->len, atom->name); + erts_cbprintf(to,to_arg,"%s%.*s", delim, atom->len, atom->name); delim = ","; } - erts_print(to,to_arg,"\n"); + erts_cbprintf(to,to_arg,"\n"); } diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 646f786651..70500ed6e1 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -188,7 +188,7 @@ dist_table_free(void *vdep) void -erts_dist_table_info(int to, void *to_arg) +erts_dist_table_info(fmtfn_t to, void *to_arg) { int lock = !ERTS_IS_CRASH_DUMPING; if (lock) @@ -564,7 +564,7 @@ erts_node_table_size(void) } void -erts_node_table_info(int to, void *to_arg) +erts_node_table_info(fmtfn_t to, void *to_arg) { int lock = !ERTS_IS_CRASH_DUMPING; if (lock) @@ -649,7 +649,7 @@ void erts_schedule_delete_node(ErlNode *enp) } struct pn_data { - int to; + fmtfn_t to; void *to_arg; Eterm sysname; int no_sysname; @@ -679,7 +679,7 @@ static void print_node(void *venp, void *vpndp) pndp->no_total++; } -void erts_print_node_info(int to, +void erts_print_node_info(fmtfn_t to, void *to_arg, Eterm sysname, int *no_sysname, diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index 7a4434acbf..47a6724c21 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -179,7 +179,7 @@ DistEntry *erts_find_or_insert_dist_entry(Eterm); DistEntry *erts_find_dist_entry(Eterm); void erts_schedule_delete_dist_entry(DistEntry *); Uint erts_dist_table_size(void); -void erts_dist_table_info(int, void *); +void erts_dist_table_info(fmtfn_t, void *); void erts_set_dist_entry_not_connected(DistEntry *); void erts_set_dist_entry_connected(DistEntry *, Eterm, Uint); ErlNode *erts_find_or_insert_node(Eterm, Uint32); @@ -187,8 +187,8 @@ void erts_schedule_delete_node(ErlNode *); void erts_set_this_node(Eterm, Uint); Uint erts_node_table_size(void); void erts_init_node_tables(int); -void erts_node_table_info(int, void *); -void erts_print_node_info(int, void *, Eterm, int*, int*); +void erts_node_table_info(fmtfn_t, void *); +void erts_print_node_info(fmtfn_t, void *, Eterm, int*, int*); Eterm erts_get_node_and_dist_references(struct process *); #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) int erts_lc_is_de_rwlocked(DistEntry *); diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index f90844ccc8..c59b42cdae 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -369,7 +369,7 @@ Eterm erts_request_io_bytes(Process *c_p); #define ERTS_PORT_REDS_INFO (CONTEXT_REDS/100) #define ERTS_PORT_REDS_TERMINATE (CONTEXT_REDS/50) -void print_port_info(Port *, int, void *); +void print_port_info(Port *, fmtfn_t, void *); void erts_port_free(Port *); #ifndef ERTS_SMP void erts_port_cleanup(Port *); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index bc59147c6c..b989d3be7a 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -557,8 +557,8 @@ do { \ */ static void exec_misc_ops(ErtsRunQueue *); -static void print_function_from_pc(int to, void *to_arg, BeamInstr* x); -static int stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg); +static void print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x); +static int stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg); static void aux_work_timeout(void *unused); static void aux_work_timeout_early_init(int no_schedulers); @@ -13171,7 +13171,7 @@ erts_continue_exit_process(Process *p) */ void -erts_stack_dump(int to, void *to_arg, Process *p) +erts_stack_dump(fmtfn_t to, void *to_arg, Process *p) { Eterm* sp; int yreg = -1; @@ -13186,7 +13186,7 @@ erts_stack_dump(int to, void *to_arg, Process *p) } void -erts_program_counter_info(int to, void *to_arg, Process *p) +erts_program_counter_info(fmtfn_t to, void *to_arg, Process *p) { erts_aint32_t state; int i; @@ -13216,7 +13216,7 @@ erts_program_counter_info(int to, void *to_arg, Process *p) } static void -print_function_from_pc(int to, void *to_arg, BeamInstr* x) +print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x) { BeamInstr* addr = find_function_from_pc(x); if (addr == NULL) { @@ -13238,7 +13238,7 @@ print_function_from_pc(int to, void *to_arg, BeamInstr* x) } static int -stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg) +stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg) { Eterm x = *sp; @@ -13270,7 +13270,7 @@ stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg) * Print scheduler information */ void -erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) { +erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { int i; erts_aint32_t flg; Process *p; diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 3347a7a60e..9e5121abd5 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1837,12 +1837,12 @@ void erts_cleanup_empty_process(Process* p); #ifdef DEBUG void erts_debug_verify_clean_empty_process(Process* p); #endif -void erts_stack_dump(int to, void *to_arg, Process *); -void erts_limited_stack_trace(int to, void *to_arg, Process *); -void erts_program_counter_info(int to, void *to_arg, Process *); -void erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp); -void erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg); -void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg); +void erts_stack_dump(fmtfn_t to, void *to_arg, Process *); +void erts_limited_stack_trace(fmtfn_t to, void *to_arg, Process *); +void erts_program_counter_info(fmtfn_t to, void *to_arg, Process *); +void erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp); +void erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg); +void erts_dump_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg); Eterm erts_get_process_priority(Process *p); Eterm erts_set_process_priority(Process *p, Eterm prio); @@ -1878,7 +1878,7 @@ void erts_handle_pending_exit(Process *, ErtsProcLocks); #define ERTS_PROC_PENDING_EXIT(P) 0 #endif -void erts_deep_process_dump(int, void *); +void erts_deep_process_dump(fmtfn_t, void *); Eterm erts_get_reader_groups_map(Process *c_p); Eterm erts_debug_reader_groups_map(Process *c_p, int groups); diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c index d8c2eaba94..42654604cb 100644 --- a/erts/emulator/beam/erl_process_dict.c +++ b/erts/emulator/beam/erl_process_dict.c @@ -156,7 +156,7 @@ erts_pd_set_initial_size(int size) * Called from break handler */ void -erts_dictionary_dump(int to, void *to_arg, ProcDict *pd) +erts_dictionary_dump(fmtfn_t to, void *to_arg, ProcDict *pd) { unsigned int i; #ifdef DEBUG @@ -196,8 +196,8 @@ erts_dictionary_dump(int to, void *to_arg, ProcDict *pd) } void -erts_deep_dictionary_dump(int to, void *to_arg, - ProcDict* pd, void (*cb)(int, void *, Eterm)) +erts_deep_dictionary_dump(fmtfn_t to, void *to_arg, + ProcDict* pd, void (*cb)(fmtfn_t, void *, Eterm)) { unsigned int i; Eterm t; diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h index 387562058c..b50a2af72c 100644 --- a/erts/emulator/beam/erl_process_dict.h +++ b/erts/emulator/beam/erl_process_dict.h @@ -37,9 +37,9 @@ typedef struct proc_dict { int erts_pd_set_initial_size(int size); Uint erts_dicts_mem_size(struct process *p); void erts_erase_dicts(struct process *p); -void erts_dictionary_dump(int to, void *to_arg, ProcDict *pd); -void erts_deep_dictionary_dump(int to, void *to_arg, - ProcDict* pd, void (*cb)(int, void *, Eterm obj)); +void erts_dictionary_dump(fmtfn_t to, void *to_arg, ProcDict *pd); +void erts_deep_dictionary_dump(fmtfn_t to, void *to_arg, + ProcDict* pd, void (*cb)(fmtfn_t, void *, Eterm obj)); Eterm erts_dictionary_copy(struct process *p, ProcDict *pd); Eterm erts_pd_hash_get(struct process *p, Eterm id); diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index a70dfb8e73..d8bb00e8c6 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -40,17 +40,17 @@ #define OUR_NIL _make_header(0,_TAG_HEADER_FLOAT) -static void dump_process_info(int to, void *to_arg, Process *p); -static void dump_element(int to, void *to_arg, Eterm x); -static void dump_dist_ext(int to, void *to_arg, ErtsDistExternal *edep); -static void dump_element_nl(int to, void *to_arg, Eterm x); -static int stack_element_dump(int to, void *to_arg, Eterm* sp, +static void dump_process_info(fmtfn_t to, void *to_arg, Process *p); +static void dump_element(fmtfn_t to, void *to_arg, Eterm x); +static void dump_dist_ext(fmtfn_t to, void *to_arg, ErtsDistExternal *edep); +static void dump_element_nl(fmtfn_t to, void *to_arg, Eterm x); +static int stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg); -static void stack_trace_dump(int to, void *to_arg, Eterm* sp); -static void print_function_from_pc(int to, void *to_arg, BeamInstr* x); -static void heap_dump(int to, void *to_arg, Eterm x); -static void dump_binaries(int to, void *to_arg, Binary* root); -static void dump_externally(int to, void *to_arg, Eterm term); +static void stack_trace_dump(fmtfn_t to, void *to_arg, Eterm* sp); +static void print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x); +static void heap_dump(fmtfn_t to, void *to_arg, Eterm x); +static void dump_binaries(fmtfn_t to, void *to_arg, Binary* root); +static void dump_externally(fmtfn_t to, void *to_arg, Eterm term); static Binary* all_binaries; @@ -60,7 +60,7 @@ extern BeamInstr beam_continue_exit[]; void -erts_deep_process_dump(int to, void *to_arg) +erts_deep_process_dump(fmtfn_t to, void *to_arg) { int i, max = erts_ptab_max(&erts_proc); @@ -117,7 +117,7 @@ Uint erts_process_memory(Process *p, int incl_msg_inq) { } static void -dump_process_info(int to, void *to_arg, Process *p) +dump_process_info(fmtfn_t to, void *to_arg, Process *p) { Eterm* sp; ErtsMessage* mp; @@ -176,7 +176,7 @@ dump_process_info(int to, void *to_arg, Process *p) } static void -dump_dist_ext(int to, void *to_arg, ErtsDistExternal *edep) +dump_dist_ext(fmtfn_t to, void *to_arg, ErtsDistExternal *edep) { if (!edep) erts_print(to, to_arg, "D0:E0:"); @@ -210,7 +210,7 @@ dump_dist_ext(int to, void *to_arg, ErtsDistExternal *edep) } static void -dump_element(int to, void *to_arg, Eterm x) +dump_element(fmtfn_t to, void *to_arg, Eterm x) { if (is_list(x)) { erts_print(to, to_arg, "H" PTR_FMT, list_val(x)); @@ -240,14 +240,14 @@ dump_element(int to, void *to_arg, Eterm x) } static void -dump_element_nl(int to, void *to_arg, Eterm x) +dump_element_nl(fmtfn_t to, void *to_arg, Eterm x) { dump_element(to, to_arg, x); erts_putc(to, to_arg, '\n'); } static void -stack_trace_dump(int to, void *to_arg, Eterm *sp) { +stack_trace_dump(fmtfn_t to, void *to_arg, Eterm *sp) { Eterm x = *sp; if (is_CP(x)) { erts_print(to, to_arg, "%p:", sp); @@ -258,7 +258,7 @@ stack_trace_dump(int to, void *to_arg, Eterm *sp) { } void -erts_limited_stack_trace(int to, void *to_arg, Process *p) +erts_limited_stack_trace(fmtfn_t to, void *to_arg, Process *p) { Eterm* sp; @@ -304,7 +304,7 @@ erts_limited_stack_trace(int to, void *to_arg, Process *p) } static int -stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg) +stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg) { Eterm x = *sp; @@ -332,7 +332,7 @@ stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg) } static void -print_function_from_pc(int to, void *to_arg, BeamInstr* x) +print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x) { BeamInstr* addr = find_function_from_pc(x); if (addr == NULL) { @@ -352,7 +352,7 @@ print_function_from_pc(int to, void *to_arg, BeamInstr* x) } static void -heap_dump(int to, void *to_arg, Eterm x) +heap_dump(fmtfn_t to, void *to_arg, Eterm x) { DeclareTmpHeapNoproc(last,1); Eterm* next = last; @@ -512,7 +512,7 @@ heap_dump(int to, void *to_arg, Eterm x) } static void -dump_binaries(int to, void *to_arg, Binary* current) +dump_binaries(fmtfn_t to, void *to_arg, Binary* current) { while (current) { long i; @@ -530,7 +530,7 @@ dump_binaries(int to, void *to_arg, Binary* current) } static void -dump_externally(int to, void *to_arg, Eterm term) +dump_externally(fmtfn_t to, void *to_arg, Eterm term) { byte sbuf[1024]; /* encode and hope for the best ... */ byte* s; @@ -573,7 +573,7 @@ dump_externally(int to, void *to_arg, Eterm term) } } -void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg) +void erts_dump_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg) { char *s; switch (erts_process_state2status(psflg)) { @@ -591,7 +591,7 @@ void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg) } void -erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg) { +erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg) { int i; diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 2a19211987..7f043e191b 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -83,7 +83,7 @@ static struct export_blob* entry_to_blob(struct export_entry* ee) } void -export_info(int to, void *to_arg) +export_info(fmtfn_t to, void *to_arg) { #ifdef ERTS_SMP int lock = !ERTS_IS_CRASH_DUMPING; diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h index 1e7bb8514b..17fc4828ca 100644 --- a/erts/emulator/beam/export.h +++ b/erts/emulator/beam/export.h @@ -53,7 +53,7 @@ typedef struct export void init_export_table(void); -void export_info(int, void *); +void export_info(fmtfn_t, void *); ERTS_GLB_INLINE Export* erts_active_export_entry(Eterm m, Eterm f, unsigned a); Export* erts_export_put(Eterm mod, Eterm func, unsigned int arity); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index b2c76aa605..244e5edeb6 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -68,7 +68,7 @@ extern void erts_pre_dirty_nif(ErtsSchedulerData *, extern void erts_post_dirty_nif(struct enif_environment_t* env); #endif extern Eterm erts_nif_taints(Process* p); -extern void erts_print_nif_taints(int to, void* to_arg); +extern void erts_print_nif_taints(fmtfn_t to, void* to_arg); void erts_unload_nif(struct erl_module_nif* nif); extern void erl_nif_init(void); extern int erts_nif_get_funcs(struct erl_module_nif*, @@ -1069,10 +1069,10 @@ void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info); void init_break_handler(void); void erts_set_ignore_break(void); void erts_replace_intr(void); -void process_info(int, void *); -void print_process_info(int, void *, Process*); -void info(int, void *); -void loaded(int, void *); +void process_info(fmtfn_t, void *); +void print_process_info(fmtfn_t, void *, Process*); +void info(fmtfn_t, void *); +void loaded(fmtfn_t, void *); /* erl_arith.c */ double erts_get_positive_zero_float(void); @@ -1161,7 +1161,7 @@ extern void erts_delete_nodes_monitors(Process *, ErtsProcLocks); extern Eterm erts_monitor_nodes(Process *, Eterm, Eterm); extern Eterm erts_processes_monitoring_nodes(Process *); extern int erts_do_net_exits(DistEntry*, Eterm); -extern int distribution_info(int, void *); +extern int distribution_info(fmtfn_t, void *); extern int is_node_name_atom(Eterm a); extern int erts_net_message(Port *, DistEntry *, @@ -1350,7 +1350,7 @@ int erts_utf8_to_latin1(byte* dest, const byte* source, int slen); #define ERTS_UTF8_ANALYZE_MORE 3 #define ERTS_UTF8_OK_MAX_CHARS 4 -void bin_write(int, void*, byte*, size_t); +void bin_write(fmtfn_t, void*, byte*, size_t); Sint intlist_to_buf(Eterm, char*, Sint); /* most callers pass plain char*'s */ struct Sint_buf { @@ -1466,7 +1466,7 @@ Eterm erts_gc_binary_part_2(Process* p, Eterm* reg, Uint live); Uint erts_current_reductions(Process* current, Process *p); -int erts_print_system_version(int to, void *arg, Process *c_p); +int erts_print_system_version(fmtfn_t to, void *arg, Process *c_p); int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg); diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c index cd038d100b..8548e30e8b 100644 --- a/erts/emulator/beam/hash.c +++ b/erts/emulator/beam/hash.c @@ -95,7 +95,7 @@ void hash_get_info(HashInfo *hi, Hash *h) ** */ -void hash_info(int to, void *arg, Hash* h) +void hash_info(fmtfn_t to, void *arg, Hash* h) { HashInfo hi; diff --git a/erts/emulator/beam/hash.h b/erts/emulator/beam/hash.h index 4e769c0119..d319aaca83 100644 --- a/erts/emulator/beam/hash.h +++ b/erts/emulator/beam/hash.h @@ -37,7 +37,7 @@ typedef void (*HFREE_FUN)(void*); /* Meta functions */ typedef void* (*HMALLOC_FUN)(int,size_t); typedef void (*HMFREE_FUN)(int,void*); -typedef int (*HMPRINT_FUN)(int,void*,char*, ...); +typedef int (*HMPRINT_FUN)(fmtfn_t,void*,char*, ...); /* ** This bucket must be placed in top of @@ -89,7 +89,7 @@ Hash* hash_init(int, Hash*, char*, int, HashFunctions); void hash_delete(Hash*); void hash_get_info(HashInfo*, Hash*); -void hash_info(int, void *, Hash*); +void hash_info(fmtfn_t, void *, Hash*); int hash_table_sz(Hash *); void* hash_get(Hash*, void*); diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c index 26d6c04ea0..c86a2122f6 100644 --- a/erts/emulator/beam/index.c +++ b/erts/emulator/beam/index.c @@ -27,7 +27,7 @@ #include "global.h" #include "index.h" -void index_info(int to, void *arg, IndexTable *t) +void index_info(fmtfn_t to, void *arg, IndexTable *t) { hash_info(to, arg, &t->htable); erts_print(to, arg, "=index_table:%s\n", t->htable.name); diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h index 0a109d8699..b2e3c0eab5 100644 --- a/erts/emulator/beam/index.h +++ b/erts/emulator/beam/index.h @@ -51,7 +51,7 @@ typedef struct index_table #define INDEX_PAGE_MASK ((1 << INDEX_PAGE_SHIFT)-1) IndexTable *erts_index_init(ErtsAlcType_t,IndexTable*,char*,int,int,HashFunctions); -void index_info(int, void *, IndexTable*); +void index_info(fmtfn_t, void *, IndexTable*); int index_table_sz(IndexTable *); int index_get(IndexTable*, void*); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 77dbe92241..4f131c74de 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -5480,7 +5480,7 @@ erts_request_io_bytes(Process *c_p) typedef struct { - int to; + fmtfn_t to; void *arg; } prt_one_lnk_data; @@ -5497,7 +5497,7 @@ static void prt_one_lnk(ErtsLink *lnk, void *vprtd) } void -print_port_info(Port *p, int to, void *arg) +print_port_info(Port *p, fmtfn_t to, void *arg) { erts_aint32_t state = erts_atomic32_read_nob(&p->state); diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index 4f36377450..5978478323 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -50,7 +50,7 @@ static erts_smp_atomic_t tot_module_bytes; #include "erl_smp.h" -void module_info(int to, void *to_arg) +void module_info(fmtfn_t to, void *to_arg) { index_info(to, to_arg, &module_tables[erts_active_code_ix()]); } diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 1c1afc8461..f105b3f401 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -48,7 +48,7 @@ Module* erts_put_module(Eterm mod); void init_module_table(void); void module_start_staging(void); void module_end_staging(int commit); -void module_info(int, void *); +void module_info(fmtfn_t, void *); Module *module_code(int, ErtsCodeIndex); int module_code_size(ErtsCodeIndex); diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c index ac7096745e..acda51c9fc 100644 --- a/erts/emulator/beam/register.c +++ b/erts/emulator/beam/register.c @@ -102,7 +102,7 @@ is_proc_alive(Process *p) return !ERTS_PROC_IS_EXITING(p); } -void register_info(int to, void *to_arg) +void register_info(fmtfn_t to, void *to_arg) { int lock = !ERTS_IS_CRASH_DUMPING; if (lock) diff --git a/erts/emulator/beam/register.h b/erts/emulator/beam/register.h index d839f55d6b..27a314ca78 100644 --- a/erts/emulator/beam/register.h +++ b/erts/emulator/beam/register.h @@ -44,7 +44,7 @@ typedef struct reg_proc int process_reg_size(void); int process_reg_sz(void); void init_register_table(void); -void register_info(int, void *); +void register_info(fmtfn_t, void *); int erts_register_name(Process *, Eterm, Eterm); Eterm erts_whereis_name_to_id(Process *, Eterm); void erts_whereis_name(Process *, ErtsProcLocks, diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index dfe82cab44..4557f1534d 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -610,22 +610,21 @@ Uint erts_sys_misc_mem_sz(void); #include "erl_printf.h" /* Io constants to erts_print and erts_putc */ -#define ERTS_PRINT_STDERR (2) -#define ERTS_PRINT_STDOUT (1) -#define ERTS_PRINT_FILE (-1) -#define ERTS_PRINT_SBUF (-2) -#define ERTS_PRINT_SNBUF (-3) -#define ERTS_PRINT_DSBUF (-4) - -#define ERTS_PRINT_MIN ERTS_PRINT_DSBUF +#define ERTS_PRINT_STDERR ((fmtfn_t)0) +#define ERTS_PRINT_STDOUT ((fmtfn_t)1) +#define ERTS_PRINT_FILE ((fmtfn_t)2) +#define ERTS_PRINT_SBUF ((fmtfn_t)3) +#define ERTS_PRINT_SNBUF ((fmtfn_t)4) +#define ERTS_PRINT_DSBUF ((fmtfn_t)5) +#define ERTS_PRINT_FD ((fmtfn_t)6) typedef struct { char *buf; size_t size; } erts_print_sn_buf; -int erts_print(int to, void *arg, char *format, ...); /* in utils.c */ -int erts_putc(int to, void *arg, char); /* in utils.c */ +int erts_print(fmtfn_t to, void *arg, char *format, ...); /* in utils.c */ +int erts_putc(fmtfn_t to, void *arg, char); /* in utils.c */ /* logger stuff is declared here instead of in global.h, so sys files won't have to include global.h */ diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 6786657faf..87ea4f05a1 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -348,40 +348,41 @@ int erts_fit_in_bits_uint(Uint value) } int -erts_print(int to, void *arg, char *format, ...) +erts_print(fmtfn_t to, void *arg, char *format, ...) { int res; va_list arg_list; va_start(arg_list, format); - if (to < ERTS_PRINT_MIN) - res = -EINVAL; - else { - switch (to) { - case ERTS_PRINT_STDOUT: + { + switch ((UWord)to) { + case (UWord)ERTS_PRINT_STDOUT: res = erts_vprintf(format, arg_list); break; - case ERTS_PRINT_STDERR: + case (UWord)ERTS_PRINT_STDERR: res = erts_vfprintf(stderr, format, arg_list); break; - case ERTS_PRINT_FILE: + case (UWord)ERTS_PRINT_FILE: res = erts_vfprintf((FILE *) arg, format, arg_list); break; - case ERTS_PRINT_SBUF: + case (UWord)ERTS_PRINT_SBUF: res = erts_vsprintf((char *) arg, format, arg_list); break; - case ERTS_PRINT_SNBUF: + case (UWord)ERTS_PRINT_SNBUF: res = erts_vsnprintf(((erts_print_sn_buf *) arg)->buf, ((erts_print_sn_buf *) arg)->size, format, arg_list); break; - case ERTS_PRINT_DSBUF: + case (UWord)ERTS_PRINT_DSBUF: res = erts_vdsprintf((erts_dsprintf_buf_t *) arg, format, arg_list); break; - default: - res = erts_vfdprintf((int) to, format, arg_list); + case (UWord)ERTS_PRINT_FD: + res = erts_vfdprintf((int)(SWord) arg, format, arg_list); break; + default: + res = erts_vcbprintf(to, arg, format, arg_list); + break; } } @@ -390,7 +391,7 @@ erts_print(int to, void *arg, char *format, ...) } int -erts_putc(int to, void *arg, char c) +erts_putc(fmtfn_t to, void *arg, char c) { return erts_print(to, arg, "%c", c); } @@ -3876,7 +3877,7 @@ store_external_or_ref_in_proc_(Process *proc, Eterm ns) return store_external_or_ref_(&hp, &MSO(proc), ns); } -void bin_write(int to, void *to_arg, byte* buf, size_t sz) +void bin_write(fmtfn_t to, void *to_arg, byte* buf, size_t sz) { size_t i; -- cgit v1.2.3 From dcc7f6b2901a632ff8398c1382c15eaabe562714 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 16 Nov 2016 16:42:30 +0100 Subject: erts: Add ErtsStrToSint64 for simplify string to 64-bit integer parsing. --- erts/emulator/beam/sys.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 4557f1534d..7740dd4373 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -378,6 +378,7 @@ typedef long Sint64; # ifdef LONG_MIN # define ERTS_SINT64_MIN LONG_MIN # endif +# define ErtsStrToSint64 strtol # elif SIZEOF_LONG_LONG == 8 # define HAVE_INT64 1 typedef unsigned long long Uint64; @@ -391,6 +392,7 @@ typedef long long Sint64; # ifdef LLONG_MIN # define ERTS_SINT64_MIN LLONG_MIN # endif +# define ErtsStrToSint64 strtoll # else # error "No 64-bit integer type found" # endif -- cgit v1.2.3 From 27e865d01ad49664897662ccc123d825505c0ff6 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 9 Nov 2016 18:41:43 +0100 Subject: erts: Fix all -Wundef errors --- erts/emulator/beam/beam_emu.c | 2 +- erts/emulator/beam/erl_alloc_util.c | 2 +- erts/emulator/beam/erl_bif_ddll.c | 2 +- erts/emulator/beam/erl_drv_nif.h | 4 ---- erts/emulator/beam/erl_gc.c | 6 +++--- erts/emulator/beam/erl_msacc.h | 8 ++++---- erts/emulator/beam/erl_port.h | 2 +- erts/emulator/beam/erl_process.h | 4 ++-- 8 files changed, 13 insertions(+), 17 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 7eaa7a0d8b..b24995ca3d 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -5053,7 +5053,7 @@ do { \ } else { TestHeap(ERTS_SINT64_HEAP_SIZE(ts),0); r(0) = make_big(HTOP); -#if defined(ARCH_32) || HALFWORD_HEAP +#if defined(ARCH_32) if (ts >= (((Uint64) 1) << 32)) { *HTOP = make_pos_bignum_header(2); BIG_DIGIT(HTOP, 0) = (Uint) (ts & ((Uint) 0xffffffff)); diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 163244237d..9c6e27fc32 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -6083,7 +6083,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) goto error; allctr->min_block_size = UNIT_CEILING(allctr->min_block_size + sizeof(FreeBlkFtr_t)); -#if ERTS_SMP +#ifdef ERTS_SMP if (init->tpref) { Uint sz = ABLK_HDR_SZ; sz += (init->fix ? diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index ef77201544..587859e413 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1099,7 +1099,7 @@ void erts_ddll_increment_port_count(DE_Handle *dh) void erts_ddll_decrement_port_count(DE_Handle *dh) { assert_drv_list_locked(); -#if DEBUG +#ifdef DEBUG ASSERT(erts_smp_atomic32_dec_read_nob(&dh->port_count) >= 0); #else erts_smp_atomic32_dec_nob(&dh->port_count); diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h index 6ec5fbb895..2489099b5c 100644 --- a/erts/emulator/beam/erl_drv_nif.h +++ b/erts/emulator/beam/erl_drv_nif.h @@ -69,10 +69,6 @@ typedef enum { # define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG # undef SIZEOF_LONG_LONG #endif -#ifdef HALFWORD_HEAP_EMULATOR -# define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR -# undef HALFWORD_HEAP_EMULATOR -#endif #include "erl_int_sizes_config.h" #if defined(SIZEOF_CHAR_SAVED__) && SIZEOF_CHAR_SAVED__ != SIZEOF_CHAR # error SIZEOF_CHAR mismatch diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index db262d35c5..6c752b9741 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -35,7 +35,7 @@ #include "error.h" #include "big.h" #include "erl_gc.h" -#if HIPE +#ifdef HIPE #include "hipe_stack.h" #include "hipe_mode_switch.h" #endif @@ -395,11 +395,11 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end, if (is_non_value(result)) { if (p->freason == TRAP) { - #if HIPE +#ifdef HIPE if (regs == NULL) { regs = erts_proc_sched_data(p)->x_reg_array; } - #endif +#endif cost = garbage_collect(p, live_hf_end, 0, regs, p->arity, p->fcalls); } else { cost = garbage_collect(p, live_hf_end, 0, regs, arity, p->fcalls); diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h index ad7c8c5eee..7e02d8e101 100644 --- a/erts/emulator/beam/erl_msacc.h +++ b/erts/emulator/beam/erl_msacc.h @@ -22,7 +22,7 @@ #define ERL_MSACC_H__ /* Can be enabled/disabled via configure */ -#if ERTS_ENABLE_MSACC == 2 +#if defined(ERTS_ENABLE_MSACC) && ERTS_ENABLE_MSACC == 2 #define ERTS_MSACC_EXTENDED_STATES 1 #endif @@ -66,7 +66,7 @@ #define ERTS_MSACC_STATE_COUNT 7 -#if ERTS_MSACC_STATE_STRINGS && ERTS_ENABLE_MSACC +#if defined(ERTS_MSACC_STATE_STRINGS) && defined(ERTS_ENABLE_MSACC) static char *erts_msacc_states[] = { "aux", "check_io", @@ -104,7 +104,7 @@ static char *erts_msacc_states[] = { #define ERTS_MSACC_STATE_COUNT ERTS_MSACC_STATIC_STATE_COUNT #endif -#if ERTS_MSACC_STATE_STRINGS +#ifdef ERTS_MSACC_STATE_STRINGS static char *erts_msacc_states[] = { "alloc", "aux", @@ -157,7 +157,7 @@ struct erl_msacc_t_ { }; -#if ERTS_ENABLE_MSACC +#ifdef ERTS_ENABLE_MSACC #ifdef USE_THREADS extern erts_tsd_key_t erts_msacc_key; diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index f90844ccc8..0844dc0994 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -733,7 +733,7 @@ erts_thr_drvport2port(ErlDrvPort drvport, int lock_pdl) if (lock_pdl && prt->port_data_lock) driver_pdl_lock(prt->port_data_lock); -#if ERTS_ENABLE_LOCK_CHECK +#ifdef ERTS_ENABLE_LOCK_CHECK if (!ERTS_IS_CRASH_DUMPING) { if (erts_lc_is_emu_thr()) { ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index b266d32e76..72344be504 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -21,6 +21,8 @@ #ifndef __PROCESS_H__ #define __PROCESS_H__ +#include "sys.h" + #undef ERTS_INCLUDE_SCHEDULER_INTERNALS #if (defined(ERL_PROCESS_C__) \ || defined(ERL_PORT_TASK_C__) \ @@ -37,8 +39,6 @@ typedef struct process Process; -#include "sys.h" - #define ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__ #include "erl_process_lock.h" /* Only pull out important types... */ #undef ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__ -- cgit v1.2.3 From 011242f927259aa477ebe488a3ee3c4e22081488 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 10 Nov 2016 20:09:48 +0100 Subject: erts: Tweak PSTACK to avoid warning warning: array subscript is below array bounds and ok, it's technically undef behavior to set pointer before first array element. --- erts/emulator/beam/global.h | 36 +++++++++++++++++++----------------- erts/emulator/beam/utils.c | 6 ++---- 2 files changed, 21 insertions(+), 21 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 537aaf3177..507ef76e59 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -773,8 +773,8 @@ do { \ typedef struct ErtsPStack_ { byte* pstart; - byte* psp; - byte* pend; + int offs; /* "stack pointer" as byte offset from pstart */ + int size; /* allocated size in bytes */ ErtsAlcType_t alloc_type; }ErtsPStack; @@ -785,8 +785,8 @@ void erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes); #define PSTACK_DECLARE(s, DEF_PSTACK_SIZE) \ PSTACK_TYPE PSTK_DEF_STACK(s)[DEF_PSTACK_SIZE]; \ ErtsPStack s = { (byte*)PSTK_DEF_STACK(s), /* pstart */ \ - (byte*)(PSTK_DEF_STACK(s) - 1), /* psp */ \ - (byte*)(PSTK_DEF_STACK(s) + (DEF_PSTACK_SIZE)), /* pend */\ + -(int)sizeof(PSTACK_TYPE), /* offs */ \ + DEF_PSTACK_SIZE*sizeof(PSTACK_TYPE), /* size */ \ ERTS_ALC_T_ESTACK /* alloc_type */ \ } @@ -806,19 +806,21 @@ do { \ } \ } while(0) -#define PSTACK_IS_EMPTY(s) (s.psp < s.pstart) +#define PSTACK_IS_EMPTY(s) (s.offs < 0) -#define PSTACK_COUNT(s) (((PSTACK_TYPE*)s.psp + 1) - (PSTACK_TYPE*)s.pstart) +#define PSTACK_COUNT(s) ((s.offs + sizeof(PSTACK_TYPE)) / sizeof(PSTACK_TYPE)) -#define PSTACK_TOP(s) (ASSERT(!PSTACK_IS_EMPTY(s)), (PSTACK_TYPE*)(s.psp)) +#define PSTACK_TOP(s) (ASSERT(!PSTACK_IS_EMPTY(s)), \ + (PSTACK_TYPE*)(s.pstart + s.offs)) -#define PSTACK_PUSH(s) \ - (s.psp += sizeof(PSTACK_TYPE), \ - ((s.psp == s.pend) ? erl_grow_pstack(&s, PSTK_DEF_STACK(s), \ - sizeof(PSTACK_TYPE)) : (void)0), \ - ((PSTACK_TYPE*) s.psp)) +#define PSTACK_PUSH(s) \ + (s.offs += sizeof(PSTACK_TYPE), \ + ((s.offs == s.size) ? erl_grow_pstack(&s, PSTK_DEF_STACK(s), \ + sizeof(PSTACK_TYPE)) : (void)0), \ + ((PSTACK_TYPE*) (s.pstart + s.offs))) -#define PSTACK_POP(s) ((PSTACK_TYPE*) (s.psp -= sizeof(PSTACK_TYPE))) +#define PSTACK_POP(s) ((s.offs -= sizeof(PSTACK_TYPE)), \ + (PSTACK_TYPE*)(s.pstart + s.offs)) /* * Do not free the stack after this, it may have pointers into what @@ -831,8 +833,8 @@ do {\ (dst)->pstart = erts_alloc(s.alloc_type,\ sizeof(PSTK_DEF_STACK(s)));\ sys_memcpy((dst)->pstart, s.pstart, _pbytes);\ - (dst)->psp = (dst)->pstart + _pbytes - sizeof(PSTACK_TYPE);\ - (dst)->pend = (dst)->pstart + sizeof(PSTK_DEF_STACK(s));\ + (dst)->offs = s.offs;\ + (dst)->size = s.size;\ (dst)->alloc_type = s.alloc_type;\ } else\ *(dst) = s;\ @@ -847,8 +849,8 @@ do { \ ASSERT(s.pstart == (byte*)PSTK_DEF_STACK(s)); \ s = *(src); /* struct copy */ \ (src)->pstart = NULL; \ - ASSERT(s.psp >= (s.pstart - sizeof(PSTACK_TYPE))); \ - ASSERT(s.psp < s.pend); \ + ASSERT(s.offs >= -(int)sizeof(PSTACK_TYPE)); \ + ASSERT(s.offs < s.size); \ } while (0) #define PSTACK_DESTROY_SAVED(pstack)\ diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index cd7131f5df..80c116fdac 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -204,9 +204,8 @@ erl_grow_wstack(ErtsWStack* s, Uint need) void erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes) { - Uint old_size = s->pend - s->pstart; + Uint old_size = s->size; Uint new_size; - Uint sp_offs = s->psp - s->pstart; if (need_bytes < old_size) new_size = 2 * old_size; @@ -220,8 +219,7 @@ erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes) sys_memcpy(new_ptr, s->pstart, old_size); s->pstart = new_ptr; } - s->pend = s->pstart + new_size; - s->psp = s->pstart + sp_offs; + s->size = new_size; } /* -- cgit v1.2.3 From 491cd4c0c0a534ab89c3ebb4c413301c91c7167d Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 22 Nov 2016 14:37:31 +0100 Subject: erts: Add env variable ERL_CRASH_DUMP_BYTES to limit crash dump size --- erts/emulator/beam/break.c | 35 +++++++++++++++++++++++++++++++++++ erts/emulator/beam/erl_init.c | 13 ++++++++++++- erts/emulator/beam/global.h | 1 + 3 files changed, 48 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 2e8a384dec..33433df68a 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -661,6 +661,26 @@ bin_check(void) #endif +static Sint64 crash_dump_limit = ERTS_SINT64_MAX; +static Sint64 crash_dump_written = 0; + +static int crash_dump_limited_writer(void* vfdp, char* buf, size_t len) +{ + const char stop_msg[] = "\n=abort: CRASH DUMP SIZE LIMIT REACHED\n"; + + crash_dump_written += len; + if (crash_dump_written <= crash_dump_limit) { + return erts_write_fd(vfdp, buf, len); + } + + len -= (crash_dump_written - crash_dump_limit); + erts_write_fd(vfdp, buf, len); + erts_write_fd(vfdp, (char*)stop_msg, sizeof(stop_msg)-1); + + /* We assume that crash dump was called from erts_exit_vv() */ + erts_exit_epilogue(); +} + /* XXX THIS SHOULD BE IN SYSTEM !!!! */ void erl_crash_dump_v(char *file, int line, char* fmt, va_list args) @@ -760,6 +780,21 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) return; } + crash_dump_limit = ERTS_SINT64_MAX; + envsz = sizeof(env); + if (erts_sys_getenv__("ERL_CRASH_DUMP_BYTES", env, &envsz) == 0) { + Sint64 limit; + char* endptr; + errno = 0; + limit = ErtsStrToSint64(env, &endptr, 10); + if (errno == 0 && limit >= 0 && endptr != env && *endptr == 0) { + if (limit == 0) + return; + crash_dump_limit = limit; + to = &crash_dump_limited_writer; + } + } + if (erts_sys_getenv__("ERL_CRASH_DUMP",&dumpnamebuf[0],&dumpnamebufsize) != 0) dumpname = "erl_crash.dump"; else diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 781bf024dd..362239f765 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -2374,6 +2374,8 @@ system_cleanup(int flush_async) erts_exit_flush_async(); } +static int erts_exit_code; + static __decl_noreturn void __noreturn erts_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) { @@ -2385,12 +2387,21 @@ erts_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) if (fmt != NULL && *fmt != '\0') erl_error(fmt, args2); /* Print error message. */ - /* Produce an Erlang core dump if error */ + erts_exit_code = n; + + /* Produce an Erlang crash dump if error */ if (((n == ERTS_ERROR_EXIT && erts_no_crash_dump == 0) || n == ERTS_DUMP_EXIT) && erts_initialized) { erl_crash_dump_v((char*) NULL, 0, fmt, args1); } + erts_exit_epilogue(); +} + +__decl_noreturn void __noreturn erts_exit_epilogue(void) +{ + int n = erts_exit_code; + sys_tty_reset(n); if (n == ERTS_INTR_EXIT) diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 244e5edeb6..d6df85034c 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1079,6 +1079,7 @@ double erts_get_positive_zero_float(void); /* config.c */ +__decl_noreturn void __noreturn erts_exit_epilogue(void); __decl_noreturn void __noreturn erts_exit(int n, char*, ...); __decl_noreturn void __noreturn erts_flush_async_exit(int n, char*, ...); void erl_error(char*, va_list); -- cgit v1.2.3 From d82ed483ae6bae52745d2725017d77cb2c7810c7 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 22 Nov 2016 15:53:51 +0100 Subject: Fix saving of original arguments when rescheduling via NifExport --- erts/emulator/beam/erl_nif.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 3a547982da..55bd9f7455 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2360,6 +2360,7 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec Eterm* reg; NifExport* ep; int i, scheduler; + int orig_argc; execution_state(env, &proc, &scheduler); @@ -2370,11 +2371,14 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec reg = erts_proc_sched_data(proc)->x_reg_array; + ASSERT(!need_save || proc->current); + orig_argc = need_save ? (int) proc->current[2] : 0; + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); if (!ep) - ep = allocate_nif_sched_data(proc, argc); - else if (need_save && ep->rootset_extra < argc) { - NifExport* new_ep = allocate_nif_sched_data(proc, argc); + ep = allocate_nif_sched_data(proc, orig_argc); + else if (need_save && ep->rootset_extra < orig_argc) { + NifExport* new_ep = allocate_nif_sched_data(proc, orig_argc); destroy_nif_export(ep); ep = new_ep; } @@ -2387,16 +2391,14 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec } if (scheduler > 0) ERTS_VBUMP_ALL_REDS(proc); - for (i = 0; i < argc; i++) { - if (need_save) - ep->rootset[i+1] = reg[i]; - reg[i] = (Eterm) argv[i]; - } if (need_save) { - ASSERT(proc->current); ep->saved_current = proc->current; - ep->saved_argc = argc; + ep->saved_argc = orig_argc; + for (i = 0; i < orig_argc; i++) + ep->rootset[i+1] = reg[i]; } + for (i = 0; i < argc; i++) + reg[i] = (Eterm) argv[i]; proc->i = (BeamInstr*) ep->exp.addressv[0]; ep->exp.code[0] = (BeamInstr) proc->current[0]; ep->exp.code[1] = (BeamInstr) proc->current[1]; -- cgit v1.2.3 From 7a6973a2fa238468bb47d175908e99d230fbc0b1 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 22 Nov 2016 16:00:25 +0100 Subject: Fix GC when NifExport is in use --- erts/emulator/beam/erl_gc.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 6f641a1ea7..af799d09da 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -2325,6 +2325,11 @@ move_msgq_to_heap(Process *p) static Uint setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) { + /* + * NOTE! + * Remember to update offset_rootset() when changing + * this function. + */ Roots* roots; Uint n; @@ -2969,6 +2974,12 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size, offset_heap_ptr(objv, nobj, offs, area, area_size); } offset_off_heap(p, offs, area, area_size); + if (ERTS_PROC_GET_NIF_TRAP_EXPORT(p)) { + Eterm* argv; + int argc; + if (erts_setup_nif_gc(p, &argv, &argc)) + offset_heap_ptr(argv, argc, offs, area, area_size); + } } static void -- cgit v1.2.3 From ad94a4bdb24c93d3cafc7351c4c98f346a5f53cc Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 22 Nov 2016 16:14:57 +0100 Subject: Fix check_process_code() when NifExport is in use --- erts/emulator/beam/beam_bif_load.c | 11 +++++++++++ erts/emulator/beam/erl_nif.c | 11 +++++++++++ erts/emulator/beam/erl_process.h | 1 + 3 files changed, 23 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 5969197168..74a333ecbe 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1086,6 +1086,11 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls || ErtsInArea(rp->cp, mod_start, mod_size)) { return am_true; } + + *redsp += 1; + + if (erts_check_nif_export_in_area(rp, mod_start, mod_size)) + return am_true; *redsp += (STACK_START(rp) - rp->stop) / 32; @@ -1161,6 +1166,12 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls || ErtsInArea(rp->cp, mod_start, mod_size)) { return am_true; } + + *redsp += 1; + + if (erts_check_nif_export_in_area(rp, mod_start, mod_size)) + return am_true; + /* * Check all continuation pointers stored on the stack. diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 55bd9f7455..e2f208badf 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2309,6 +2309,17 @@ erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj) return gc; } +int +erts_check_nif_export_in_area(Process *p, char *start, Uint size) +{ + NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(p); + if (!nep || !nep->saved_current) + return 0; + if (ErtsInArea(nep->saved_current, start, size)) + return 1; + return 0; +} + /* * Allocate a NifExport and set it in proc specific data */ diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 3347a7a60e..435b53fb5f 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1586,6 +1586,7 @@ Uint64 erts_step_proc_interval(void); int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */ void erts_destroy_nif_export(void *); /* see erl_nif.c */ +int erts_check_nif_export_in_area(Process *p, char *start, Uint size); ErtsProcList *erts_proclist_create(Process *); ErtsProcList *erts_proclist_copy(ErtsProcList *); -- cgit v1.2.3 From 80490fe3e8f636f4de3af1d71cdb41721e88ac7c Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 22 Nov 2016 16:19:55 +0100 Subject: Silence debug warning when no beam jump table is used with dirty schedulers --- erts/emulator/beam/beam_emu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index ef4cdf9d5a..c2000929d5 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -5325,7 +5325,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) I = c_p->i; - ASSERT(BeamOp(op_call_nif) == (BeamInstr *) *I); + ASSERT(em_call_nif == (BeamInstr *) *I); /* * Set fcalls even though we ignore it, so we don't -- cgit v1.2.3 From 696167d5e64365c28d626e117c136cb81e4c4028 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 21 Oct 2016 14:16:24 +0200 Subject: Fix alloc-util hard-debug --- erts/emulator/beam/erl_alloc_util.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 2995f2f822..7abe3846be 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -4153,9 +4153,9 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) ASSERT(IS_LAST_BLK(blk)); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - (*allctr->link_free_block)(allctr, blk, 0); + (*allctr->link_free_block)(allctr, blk); HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, blk, 0); + (*allctr->unlink_free_block)(allctr, blk); #endif } #endif @@ -6440,11 +6440,6 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) ASSERT(SBC2BLK(allctr, sbc) == iblk); ASSERT(CARRIER_SZ(sbc) - SBC_HEADER_SIZE >= SBC_BLK_SZ(iblk)); -#if HAVE_ERTS_MSEG - if (IS_MSEG_CARRIER(sbc)) { - ASSERT(CARRIER_SZ(sbc) % ERTS_SACRR_UNIT_SZ == 0); - } -#endif crr = sbc; cl = &allctr->sbc_list; } -- cgit v1.2.3 From a532df810da29fcb28d142d244b6b3c812fa33ca Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 17 Oct 2016 12:00:32 +0200 Subject: Fix dirty scheduler process priority --- erts/emulator/beam/erl_process.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index bc59147c6c..59ee84d115 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -6271,7 +6271,11 @@ check_dirty_enqueue_in_prio_queue(Process *c_p, return -1*queue; } - *newp |= ERTS_PSFLG_IN_RUNQ; + /* + * Enqueue using process struct. + */ + *newp &= ~ERTS_PSFLGS_PRQ_PRIO_MASK; + *newp |= ERTS_PSFLG_IN_RUNQ | (aprio << ERTS_PSFLGS_PRQ_PRIO_OFFSET); return queue; } -- cgit v1.2.3 From dd2e99bdd726d4322c8e07c81731ad66ae05176e Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 21 Oct 2016 14:05:11 +0200 Subject: Fix send of exit signal to process executing dirty --- erts/emulator/beam/erl_process.c | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 59ee84d115..86a997de65 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -12405,6 +12405,8 @@ send_exit_signal(Process *c_p, /* current process if and only else if (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))) { /* Process not running ... */ ErtsProcLocks need_locks = ~(*rp_locks) & ERTS_PROC_LOCKS_ALL; + ErlHeapFragment *bp = NULL; + Eterm rsn_cpy; if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) { /* ... but we havn't got all locks on it ... */ @@ -12417,12 +12419,32 @@ send_exit_signal(Process *c_p, /* current process if and only } /* ...and we have all locks on it... */ *rp_locks = ERTS_PROC_LOCKS_ALL; - set_proc_exiting(rp, - state, - (is_immed(rsn) - ? rsn - : copy_object(rsn, rp)), - NULL); + + state = erts_smp_atomic32_read_nob(&rp->state); + + if (is_immed(rsn)) + rsn_cpy = rsn; + else { + Eterm *hp; + ErlOffHeap *ohp; + Uint rsn_sz = size_object(rsn); +#ifdef ERTS_DIRTY_SCHEDULERS + if (state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + bp = new_message_buffer(rsn_sz); + ohp = &bp->off_heap; + hp = &bp->mem[0]; + } + else +#endif + { + hp = HAlloc(rp, rsn_sz); + ohp = &rp->off_heap; + } + rsn_cpy = copy_struct(rsn, rsn_sz, &hp, ohp); + } + + set_proc_exiting(rp, state, rsn_cpy, bp); } else { /* Process running... */ -- cgit v1.2.3 From e9ac0c64a78c858d474757d435f3c7dce2a224b1 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 1 Nov 2016 16:08:21 +0100 Subject: Fix call time tracing with dirty schedulers --- erts/emulator/beam/beam_bp.c | 2 +- erts/emulator/beam/beam_bp.h | 2 +- erts/emulator/beam/erl_process.c | 4 ++++ erts/emulator/beam/erl_process.h | 1 + 4 files changed, 7 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 920c8b1ed0..bbb2e4f34f 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -1517,7 +1517,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, ASSERT((bp->flags & ERTS_BPF_TIME_TRACE) == 0); bdt = Alloc(sizeof(BpDataTime)); erts_refc_init(&bdt->refc, 1); - bdt->n = erts_no_schedulers; + bdt->n = erts_no_total_schedulers; bdt->hash = Alloc(sizeof(bp_time_hash_t)*(bdt->n)); for (i = 0; i < bdt->n; i++) { bp_hash_init(&(bdt->hash[i]), 32); diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 541af77211..7206ef471a 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -80,7 +80,7 @@ typedef struct generic_bp { #define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2) #ifdef ERTS_SMP -#define bp_sched2ix_proc(p) (erts_proc_sched_data(p)->no - 1) +#define bp_sched2ix_proc(p) (erts_proc_sched_data(p)->thr_id - 1) #else #define bp_sched2ix_proc(p) (0) #endif diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 86a997de65..adf7779fe5 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -154,6 +154,7 @@ int ERTS_WRITE_UNLIKELY(erts_eager_check_io) = 1; int ERTS_WRITE_UNLIKELY(erts_sched_compact_load); int ERTS_WRITE_UNLIKELY(erts_sched_balance_util) = 0; Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers); +Uint ERTS_WRITE_UNLIKELY(erts_no_total_schedulers); Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers) = 0; Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers) = 0; @@ -5908,9 +5909,12 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online n = (int) no_schedulers; erts_no_schedulers = n; + erts_no_total_schedulers = n; #ifdef ERTS_DIRTY_SCHEDULERS erts_no_dirty_cpu_schedulers = no_dirty_cpu_schedulers; + erts_no_total_schedulers += no_dirty_cpu_schedulers; erts_no_dirty_io_schedulers = no_dirty_io_schedulers; + erts_no_total_schedulers += no_dirty_io_schedulers; #endif /* Create and initialize scheduler sleep info */ diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 435b53fb5f..9214d05456 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -111,6 +111,7 @@ extern int erts_eager_check_io; extern int erts_sched_compact_load; extern int erts_sched_balance_util; extern Uint erts_no_schedulers; +extern Uint erts_no_total_schedulers; #ifdef ERTS_DIRTY_SCHEDULERS extern Uint erts_no_dirty_cpu_schedulers; extern Uint erts_no_dirty_io_schedulers; -- cgit v1.2.3 From ea15a660908c48a2b8cefe0ed90fc5c4a89fe572 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 22 Nov 2016 15:31:44 +0100 Subject: Fix scheduling of system tasks on processes executing dirty --- erts/emulator/beam/erl_process.c | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index adf7779fe5..92b3d9d1a8 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -6691,15 +6691,8 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, erts_aint32_t fail_state, state, a, n, enq_prio; int enqueue; /* < 0 -> use proxy */ unsigned int prof_runnable_procs; - int strict_fail_state; fail_state = *fail_state_p; - /* - * If fail state something other than just exiting process, - * ensure that the task wont be scheduled when the - * receiver is in the failure state. - */ - strict_fail_state = fail_state != ERTS_PSFLG_EXITING; res = 1; /* prepare for success */ st->next = st->prev = st; /* Prep for empty prio queue */ @@ -6781,7 +6774,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, /* Status lock prevents out of order "runnable proc" trace msgs */ ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - if (!prof_runnable_procs && !strict_fail_state) { + if (!prof_runnable_procs) { erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); locked = 0; } @@ -6792,11 +6785,6 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, erts_aint32_t e; n = e = a; - if (strict_fail_state && (a & fail_state)) { - *fail_state_p = (a & fail_state); - goto cleanup; - } - if (a & ERTS_PSFLG_FREE) goto cleanup; /* We don't want to schedule free processes... */ -- cgit v1.2.3 From ab647066f572d9456572a70c53687f9ab52502b8 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 23 Nov 2016 15:33:29 +0100 Subject: erts: Remove one little space in crash dump --- erts/emulator/beam/break.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 33433df68a..dfbe1ced47 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -666,7 +666,7 @@ static Sint64 crash_dump_written = 0; static int crash_dump_limited_writer(void* vfdp, char* buf, size_t len) { - const char stop_msg[] = "\n=abort: CRASH DUMP SIZE LIMIT REACHED\n"; + const char stop_msg[] = "\n=abort:CRASH DUMP SIZE LIMIT REACHED\n"; crash_dump_written += len; if (crash_dump_written <= crash_dump_limit) { -- cgit v1.2.3 From ec776506eac5342da3d45829fb67b669081f6439 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 23 Nov 2016 15:55:43 +0100 Subject: erts: Secure abi backward compat for tracer nifs and simplify the code by handling the compatibility stuff at loading by creating modern struct copies in create_lib(). --- erts/emulator/beam/erl_nif.c | 128 +++++++++++++++++++++++-------------------- erts/emulator/beam/erl_nif.h | 15 +++-- 2 files changed, 77 insertions(+), 66 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 9872a3ab81..9fec9b4939 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -71,10 +71,12 @@ struct erl_module_nif { void* priv_data; void* handle; /* "dlopen" */ - struct enif_entry_t* entry; + struct enif_entry_t entry; erts_refc_t rt_cnt; /* number of resource types */ erts_refc_t rt_dtor_cnt; /* number of resource types with destructors */ Module* mod; /* Can be NULL if orphan with dtor-resources left */ + + ErlNifFunc _funcs_copy_[1]; /* only used for old libs */ }; #ifdef DEBUG @@ -1958,10 +1960,10 @@ static void close_lib(struct erl_module_nif* lib) ASSERT(lib->handle != NULL); ASSERT(erts_refc_read(&lib->rt_dtor_cnt,0) == 0); - if (lib->entry != NULL && lib->entry->unload != NULL) { + if (lib->entry.unload != NULL) { struct enif_msg_environment_t msg_env; pre_nif_noproc(&msg_env, lib, NULL); - lib->entry->unload(&msg_env.env, lib->priv_data); + lib->entry.unload(&msg_env.env, lib->priv_data); post_nif_noproc(&msg_env); } if (!erts_is_static_nif(lib->handle)) @@ -3115,33 +3117,52 @@ static Eterm load_nif_error(Process* p, const char* atom, const char* format, .. return ret; } +#define AT_LEAST_VERSION(E,MAJ,MIN) \ + (((E)->major * 0x100 + (E)->minor) >= ((MAJ) * 0x100 + (MIN))) + /* - * The function below is for looping through ErlNifFunc arrays, helping - * provide backwards compatibility across the version 2.7 change that added - * the "flags" field to ErlNifFunc. + * Allocate erl_module_nif and make a _modern_ copy of the lib entry. */ -static ErlNifFunc* next_func(ErlNifEntry* entry, int* incrp, ErlNifFunc* func) +static struct erl_module_nif* create_lib(const ErlNifEntry* src) { - ASSERT(incrp); - if (!*incrp) { - if (entry->major > 2 || (entry->major == 2 && entry->minor >= 7)) - *incrp = sizeof(ErlNifFunc); - else { - /* - * ErlNifFuncV1 below is what ErlNifFunc was before the - * addition of the flags field for 2.7, and is needed to handle - * backward compatibility. - */ - typedef struct { - const char* name; - unsigned arity; - ERL_NIF_TERM (*fptr)(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); - }ErlNifFuncV1; - *incrp = sizeof(ErlNifFuncV1); - } + struct erl_module_nif* lib; + ErlNifEntry* dst; + Uint bytes = offsetof(struct erl_module_nif, _funcs_copy_); + + if (!AT_LEAST_VERSION(src, 2, 7)) + bytes += src->num_of_funcs * sizeof(ErlNifFunc); + + lib = erts_alloc(ERTS_ALC_T_NIF, bytes); + dst = &lib->entry; + + sys_memcpy(dst, src, offsetof(ErlNifEntry, vm_variant)); + + if (AT_LEAST_VERSION(src, 2, 1)) { + dst->vm_variant = src->vm_variant; + } else { + dst->vm_variant = "beam.vanilla"; } - return (ErlNifFunc*) ((char*)func + *incrp); -} + if (AT_LEAST_VERSION(src, 2, 7)) { + dst->options = src->options; + } else { + /* + * Make a modern copy of the ErlNifFunc array + */ + struct ErlNifFunc_V1 { + const char* name; + unsigned arity; + ERL_NIF_TERM (*fptr)(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + }*src_funcs = (struct ErlNifFunc_V1*) src->funcs; + int i; + for (i = 0; i < src->num_of_funcs; ++i) { + sys_memcpy(&lib->_funcs_copy_[i], &src_funcs[i], sizeof(*src_funcs)); + lib->_funcs_copy_[i].flags = 0; + } + dst->funcs = lib->_funcs_copy_; + dst->options = 0; + } + return lib; +}; BIF_RETTYPE load_nif_2(BIF_ALIST_2) @@ -3261,7 +3282,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ret = load_nif_error(BIF_P, bad_lib, "Library version (%d.%d) not compatible (with %d.%d).", entry->major, entry->minor, ERL_NIF_MAJOR_VERSION, ERL_NIF_MINOR_VERSION); } - else if (entry->minor >= 1 + else if (AT_LEAST_VERSION(entry, 2, 1) && sys_strcmp(entry->vm_variant, ERL_NIF_VM_VARIANT) != 0) { ret = load_nif_error(BIF_P, bad_lib, "Library (%s) not compiled for " "this vm variant (%s).", @@ -3272,20 +3293,25 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) " match calling module '%T'", entry->name, mod_atom); } else { - /*erts_fprintf(stderr, "Found module %T\r\n", mod_atom);*/ + lib = create_lib(entry); + entry = &lib->entry; /* Use a guaranteed modern lib entry from now on */ + + lib->handle = handle; + erts_refc_init(&lib->rt_cnt, 0); + erts_refc_init(&lib->rt_dtor_cnt, 0); + ASSERT(opened_rt_list == NULL); + lib->mod = module_p; - int maybe_dirty_nifs = ((entry->major > 2 || (entry->major == 2 && entry->minor >= 7)) - && (entry->options & ERL_NIF_DIRTY_NIF_OPTION)); - int incr = 0; - ErlNifFunc* f = entry->funcs; - for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) { + for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) { ErtsCodeInfo** ci_pp; + ErlNifFunc* f = &entry->funcs[i]; + if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1) || (ci_pp = get_func_pp(this_mi->code_hdr, f_atom, f->arity))==NULL) { ret = load_nif_error(BIF_P,bad_lib,"Function not found %T:%s/%u", mod_atom, f->name, f->arity); } - else if (maybe_dirty_nifs && f->flags) { + else if (f->flags) { /* * If the flags field is non-zero and this emulator was * built with dirty scheduler support, check that the flags @@ -3314,7 +3340,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) } /*erts_fprintf(stderr, "Found NIF %T:%s/%u\r\n", mod_atom, f->name, f->arity);*/ - f = next_func(entry, &incr, f); } } @@ -3325,14 +3350,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) /* Call load or upgrade: */ - - lib = erts_alloc(ERTS_ALC_T_NIF, sizeof(struct erl_module_nif)); - lib->handle = handle; - lib->entry = entry; - erts_refc_init(&lib->rt_cnt, 0); - erts_refc_init(&lib->rt_dtor_cnt, 0); - ASSERT(opened_rt_list == NULL); - lib->mod = module_p; env.mod_nif = lib; lib->priv_data = NULL; @@ -3349,8 +3366,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) prev_mi->nif->priv_data = prev_old_data; ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful (%d).", veto); } - else - commit_opened_resource_types(lib); } else if (entry->load != NULL) { /********* Initial load ***********/ erts_pre_nif(&env, BIF_P, lib, NULL); @@ -3359,22 +3374,21 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) if (veto) { ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful (%d).", veto); } - else - commit_opened_resource_types(lib); } if (ret == am_ok) { + commit_opened_resource_types(lib); + /* ** Everything ok, patch the beam code with op_call_nif */ - int incr = 0; - ErlNifFunc* f = entry->funcs; - this_mi->nif = lib; for (i=0; i < entry->num_of_funcs; i++) { + ErlNifFunc* f = &entry->funcs[i]; ErtsCodeInfo* ci; BeamInstr *code_ptr; + erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1); ci = *get_func_pp(this_mi->code_hdr, f_atom, f->arity); code_ptr = erts_codeinfo_to_code(ci); @@ -3389,8 +3403,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) g->orig_instr = (BeamInstr) BeamOp(op_call_nif); } #ifdef ERTS_DIRTY_SCHEDULERS - if ((entry->major > 2 || (entry->major == 2 && entry->minor >= 7)) - && (entry->options & ERL_NIF_DIRTY_NIF_OPTION) && f->flags) { + if (f->flags) { code_ptr[3] = (BeamInstr) f->fptr; code_ptr[1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ? (BeamInstr) schedule_dirty_io_nif : @@ -3400,7 +3413,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) #endif code_ptr[1] = (BeamInstr) f->fptr; code_ptr[2] = (BeamInstr) lib; - f = next_func(entry, &incr, f); } } else { @@ -3484,8 +3496,8 @@ void erl_nif_init() int erts_nif_get_funcs(struct erl_module_nif* mod, ErlNifFunc **funcs) { - *funcs = mod->entry->funcs; - return mod->entry->num_of_funcs; + *funcs = mod->entry.funcs; + return mod->entry.num_of_funcs; } Eterm erts_nif_call_function(Process *p, Process *tracee, @@ -3496,10 +3508,10 @@ Eterm erts_nif_call_function(Process *p, Process *tracee, #ifdef DEBUG /* Verify that function is part of this module */ int i; - for (i = 0; i < mod->entry->num_of_funcs; i++) - if (fun == &(mod->entry->funcs[i])) + for (i = 0; i < mod->entry.num_of_funcs; i++) + if (fun == &(mod->entry.funcs[i])) break; - ASSERT(i < mod->entry->num_of_funcs); + ASSERT(i < mod->entry.num_of_funcs); if (p) ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) & ERTS_PROC_LOCK_MAIN || erts_smp_thr_progress_is_blocking()); diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 494971e118..413b4f7343 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -49,7 +49,7 @@ ** 2.8: 18.0 add enif_has_pending_exception ** 2.9: 18.2 enif_getenv ** 2.10: Time API -** 2.11: 19.0 enif_snprintf +** 2.11: 19.0 enif_snprintf */ #define ERL_NIF_MAJOR_VERSION 2 #define ERL_NIF_MINOR_VERSION 11 @@ -116,12 +116,13 @@ typedef struct enif_entry_t int (*reload) (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info); int (*upgrade)(ErlNifEnv*, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info); void (*unload) (ErlNifEnv*, void* priv_data); + + /* Added in 2.1 */ const char* vm_variant; - unsigned options; -}ErlNifEntry; -/* Field bits for ErlNifEntry options */ -#define ERL_NIF_DIRTY_NIF_OPTION 1 + /* Added in 2.7 */ + unsigned options; /* Unused. Can be set to 0 or 1 (dirty sched config) */ +}ErlNifEntry; typedef struct @@ -266,8 +267,6 @@ extern TWinDynNifCallbacks WinDynNifCallbacks; # define ERL_NIF_INIT_DECL(MODNAME) ERL_NIF_INIT_EXPORT ErlNifEntry* nif_init(ERL_NIF_INIT_ARGS) #endif -#define ERL_NIF_ENTRY_OPTIONS ERL_NIF_DIRTY_NIF_OPTION - #ifdef __cplusplus } # define ERL_NIF_INIT_PROLOGUE extern "C" { @@ -293,7 +292,7 @@ ERL_NIF_INIT_DECL(NAME) \ FUNCS, \ LOAD, RELOAD, UPGRADE, UNLOAD, \ ERL_NIF_VM_VARIANT, \ - ERL_NIF_ENTRY_OPTIONS \ + 1 \ }; \ ERL_NIF_INIT_BODY; \ return &entry; \ -- cgit v1.2.3 From 9a88fc02527063be5096bdf4d49a48b144471553 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 14 Oct 2016 18:48:00 +0200 Subject: erts: Add erts internal secret atom --- erts/emulator/beam/atom.c | 5 ++++- erts/emulator/beam/atom.h | 2 -- erts/emulator/beam/atom.names | 3 +++ 3 files changed, 7 insertions(+), 3 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index b47739059b..2b5ad097a0 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -199,7 +199,7 @@ atom_alloc(Atom* tmpl) static void atom_free(Atom* obj) { - erts_free(ERTS_ALC_T_ATOM, (void*) obj); + ASSERT(obj->slot.index == atom_val(am_ErtsSecretAtom)); } static void latin1_to_utf8(byte* conv_buf, const byte** srcp, int* lenp) @@ -467,6 +467,9 @@ init_atom_table(void) atom_space -= a.len; atom_tab(ix)->name = (byte*)erl_atom_names[i]; } + + /* Hide am_ErtsSecretAtom */ + hash_erase(&erts_atom_table.htable, atom_tab(atom_val(am_ErtsSecretAtom))); } void diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h index f3b21e1687..abd3b44993 100644 --- a/erts/emulator/beam/atom.h +++ b/erts/emulator/beam/atom.h @@ -133,8 +133,6 @@ int atom_table_sz(void); /* table size in bytes, excluding stored objects */ Eterm am_atom_put(const char*, int); /* ONLY 7-bit ascii! */ Eterm erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc); -int atom_erase(byte*, int); -int atom_static_put(byte*, int); void init_atom_table(void); void atom_info(fmtfn_t, void *); void dump_atoms(fmtfn_t, void *); diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 66af05c1f2..b1aeed7889 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -59,6 +59,9 @@ atom nocatch atom undefined_function atom undefined_lambda +# Secret internal atom that can never be found by string lookup +# and should never leak out to be seen by the user. +atom ErtsSecretAtom='3RT$' # All other atoms. Try to keep the order alphabetic. # -- cgit v1.2.3 From f7a3650b6827242772f957d624dbb91b5e472744 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 29 Nov 2016 18:21:17 +0100 Subject: erts: Use secret atom as sysname for ETS compressed --- erts/emulator/beam/external.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index a49b242d7c..10e452fa25 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2163,12 +2163,8 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) * We use this atom as sysname in local pid/port/refs * for the ETS compressed format (DFLAG_INTERNAL_TAGS). * - * We used atom '' earlier but that turned out to cause problems - * for buggy erl_interface/ic usage of c-nodes with empty node names. - * A long atom reduces risk of nodes actually called this and the length - * does not matter anyway as it's encoded with atom index (ATOM_INTERNAL_REF2). */ -#define INTERNAL_LOCAL_SYSNAME am_await_microstate_accounting_modifications +#define INTERNAL_LOCAL_SYSNAME am_ErtsSecretAtom static byte* enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags) -- cgit v1.2.3 From 6c76bfa96c12b9f29e66baf3a2df7e84bb9b05ef Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 29 Nov 2016 20:54:14 +0100 Subject: erts: Refactor BEAM_NIF_MIN_FUNC_SZ to be declared once in beam_load.h and get rid of #ifdef kludge. --- erts/emulator/beam/beam_load.c | 11 +++-------- erts/emulator/beam/beam_load.h | 6 ++++++ erts/emulator/beam/erl_nif.c | 7 ++----- 3 files changed, 11 insertions(+), 13 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 3cd395c2c1..8f1faa6719 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -2546,15 +2546,10 @@ load_code(LoaderState* stp) if (stp->may_load_nif) { const int finfo_ix = ci - FUNC_INFO_SZ; -#ifdef ERTS_DIRTY_SCHEDULERS - enum { MIN_FUNC_SZ = 4 }; -#else - enum { MIN_FUNC_SZ = 3 }; -#endif - if (finfo_ix - last_func_start < MIN_FUNC_SZ && last_func_start) { + if (finfo_ix - last_func_start < BEAM_NIF_MIN_FUNC_SZ && last_func_start) { /* Must make room for call_nif op */ - int pad = MIN_FUNC_SZ - (finfo_ix - last_func_start); - ASSERT(pad > 0 && pad < MIN_FUNC_SZ); + int pad = BEAM_NIF_MIN_FUNC_SZ - (finfo_ix - last_func_start); + ASSERT(pad > 0 && pad < BEAM_NIF_MIN_FUNC_SZ); CodeNeed(pad); sys_memmove(&code[finfo_ix+pad], &code[finfo_ix], FUNC_INFO_SZ*sizeof(BeamInstr)); diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index d420a4346c..659b9c303f 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -111,6 +111,12 @@ typedef struct beam_code_header { }BeamCodeHeader; +#ifdef ERTS_DIRTY_SCHEDULERS +# define BEAM_NIF_MIN_FUNC_SZ 4 +#else +# define BEAM_NIF_MIN_FUNC_SZ 3 +#endif + void erts_release_literal_area(struct ErtsLiteralArea_* literal_area); int erts_is_module_native(BeamCodeHeader* code); void erts_beam_bif_load_init(void); diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 202f713f67..5499512dd1 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -3341,11 +3341,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) mod_atom, f->name, f->arity); #endif } -#ifdef ERTS_DIRTY_SCHEDULERS - else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) < (4)) -#else - else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) < (3)) -#endif + else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) + < BEAM_NIF_MIN_FUNC_SZ) { ret = load_nif_error(BIF_P,bad_lib,"No explicit call to load_nif" " in module (%T:%s/%u too small)", -- cgit v1.2.3 From 0763a36867a702e3075b682973a079e0390144ce Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 23 Nov 2016 15:58:15 +0100 Subject: erts: Add enif_select & enif_open_resource_type_x --- erts/emulator/beam/erl_alloc.c | 2 + erts/emulator/beam/erl_alloc.types | 1 + erts/emulator/beam/erl_driver.h | 9 ++-- erts/emulator/beam/erl_drv_nif.h | 7 +++ erts/emulator/beam/erl_nif.c | 90 ++++++++++++++++++++-------------- erts/emulator/beam/erl_nif.h | 20 +++++++- erts/emulator/beam/erl_nif_api_funcs.h | 4 ++ erts/emulator/beam/global.h | 29 +++++++++++ 8 files changed, 117 insertions(+), 45 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 40a45c961f..3ddf7a53e2 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -663,6 +663,8 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) = sizeof(ErtsDrvEventDataState); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_SEL_D_STATE)] = sizeof(ErtsDrvSelectDataState); + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_SEL_D_STATE)] + = sizeof(ErtsNifSelectDataState); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MSG_REF)] = sizeof(ErtsMessageRef); #ifdef ERTS_SMP diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 6e8710eb8a..7ea8c98008 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -396,6 +396,7 @@ type DRV_TAB LONG_LIVED SYSTEM drv_tab type DRV_EV_STATE LONG_LIVED SYSTEM driver_event_state type DRV_EV_D_STATE FIXED_SIZE SYSTEM driver_event_data_state type DRV_SEL_D_STATE FIXED_SIZE SYSTEM driver_select_data_state +type NIF_SEL_D_STATE FIXED_SIZE SYSTEM enif_select_data_state type FD_LIST SHORT_LIVED SYSTEM fd_list type ACTIVE_FD_ARR SHORT_LIVED SYSTEM active_fd_array type POLLSET LONG_LIVED SYSTEM pollset diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 97a69140c3..5bea92e198 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -76,11 +76,10 @@ typedef struct { # endif #endif -/* Values for mode arg to driver_select() */ -#define ERL_DRV_READ (1 << 0) -#define ERL_DRV_WRITE (1 << 1) -#define ERL_DRV_USE (1 << 2) -#define ERL_DRV_USE_NO_CALLBACK (ERL_DRV_USE | (1 << 3)) +#define ERL_DRV_READ ((int)ERL_NIF_SELECT_READ) +#define ERL_DRV_WRITE ((int)ERL_NIF_SELECT_WRITE) +#define ERL_DRV_USE ((int)ERL_NIF_SELECT_STOP) +#define ERL_DRV_USE_NO_CALLBACK (ERL_DRV_USE | (ERL_DRV_USE << 1)) /* Old deprecated */ #define DO_READ ERL_DRV_READ diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h index 2489099b5c..e4ebcdb1d4 100644 --- a/erts/emulator/beam/erl_drv_nif.h +++ b/erts/emulator/beam/erl_drv_nif.h @@ -49,6 +49,13 @@ typedef enum { ERL_DIRTY_JOB_IO_BOUND = 2 } ErlDirtyJobFlags; +/* Values for enif_select AND mode arg for driver_select() */ +enum ErlNifSelectFlags { + ERL_NIF_SELECT_READ = (1 << 0), + ERL_NIF_SELECT_WRITE = (1 << 1), + ERL_NIF_SELECT_STOP = (1 << 2) +}; + #ifdef SIZEOF_CHAR # define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR # undef SIZEOF_CHAR diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 202f713f67..27abba7cfd 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1904,37 +1904,10 @@ int enif_snprintf(char *buffer, size_t size, const char* format, ...) ** Memory managed (GC'ed) "resource" objects ** ***********************************************************/ - -struct enif_resource_type_t -{ - struct enif_resource_type_t* next; /* list of all resource types */ - struct enif_resource_type_t* prev; - struct erl_module_nif* owner; /* that created this type and thus implements the destructor*/ - ErlNifResourceDtor* dtor; /* user destructor function */ - erts_refc_t refc; /* num of resources of this type (HOTSPOT warning) - +1 for active erl_module_nif */ - Eterm module; - Eterm name; -}; - /* dummy node in circular list */ struct enif_resource_type_t resource_type_list; -typedef struct enif_resource_t -{ - struct enif_resource_type_t* type; -#ifdef DEBUG - erts_refc_t nif_refc; -# ifdef ARCH_32 - byte align__[4]; -# endif -#endif - - char data[1]; -}ErlNifResource; - #define SIZEOF_ErlNifResource(SIZE) (offsetof(ErlNifResource,data) + (SIZE)) -#define DATA_TO_RESOURCE(PTR) ((ErlNifResource*)((char*)(PTR) - offsetof(ErlNifResource,data))) static ErlNifResourceType* find_resource_type(Eterm module, Eterm name) { @@ -1997,24 +1970,23 @@ struct opened_resource_type ErlNifResourceFlags op; ErlNifResourceType* type; - ErlNifResourceDtor* new_dtor; + ErlNifResourceTypeInit new_callbacks; }; static struct opened_resource_type* opened_rt_list = NULL; -ErlNifResourceType* -enif_open_resource_type(ErlNifEnv* env, - const char* module_str, - const char* name_str, - ErlNifResourceDtor* dtor, - ErlNifResourceFlags flags, - ErlNifResourceFlags* tried) +static +ErlNifResourceType* open_resource_type(ErlNifEnv* env, + const char* name_str, + const ErlNifResourceTypeInit* init, + ErlNifResourceFlags flags, + ErlNifResourceFlags* tried, + size_t sizeof_init) { ErlNifResourceType* type = NULL; ErlNifResourceFlags op = flags; Eterm module_am, name_am; ASSERT(erts_smp_thr_progress_is_blocking()); - ASSERT(module_str == NULL); /* for now... */ module_am = make_atom(env->mod_nif->mod->module); name_am = enif_make_atom(env, name_str); @@ -2048,7 +2020,9 @@ enif_open_resource_type(ErlNifEnv* env, sizeof(struct opened_resource_type)); ort->op = op; ort->type = type; - ort->new_dtor = dtor; + sys_memzero(&ort->new_callbacks, sizeof(ErlNifResourceTypeInit)); + ASSERT(sizeof_init > 0 && sizeof_init <= sizeof(ErlNifResourceTypeInit)); + sys_memcpy(&ort->new_callbacks, init, sizeof_init); ort->next = opened_rt_list; opened_rt_list = ort; } @@ -2058,6 +2032,31 @@ enif_open_resource_type(ErlNifEnv* env, return type; } +ErlNifResourceType* +enif_open_resource_type(ErlNifEnv* env, + const char* module_str, + const char* name_str, + ErlNifResourceDtor* dtor, + ErlNifResourceFlags flags, + ErlNifResourceFlags* tried) +{ + ErlNifResourceTypeInit init = {dtor, NULL}; + ASSERT(module_str == NULL); /* for now... */ + return open_resource_type(env, name_str, &init, flags, tried, + sizeof(init)); +} + +ErlNifResourceType* +enif_open_resource_type_x(ErlNifEnv* env, + const char* name_str, + const ErlNifResourceTypeInit* init, + ErlNifResourceFlags flags, + ErlNifResourceFlags* tried) +{ + return open_resource_type(env, name_str, init, flags, tried, + env->mod_nif->entry.sizeof_ErlNifResourceTypeInit); +} + static void commit_opened_resource_types(struct erl_module_nif* lib) { while (opened_rt_list) { @@ -2076,7 +2075,8 @@ static void commit_opened_resource_types(struct erl_module_nif* lib) } type->owner = lib; - type->dtor = ort->new_dtor; + type->dtor = ort->new_callbacks.dtor; + type->stop = ort->new_callbacks.stop; if (type->dtor != NULL) { erts_refc_inc(&lib->rt_dtor_cnt, 1); @@ -2124,6 +2124,15 @@ static void nif_resource_dtor(Binary* bin) } } +void erts_resource_stop(ErlNifResource* resource) +{ + struct enif_msg_environment_t msg_env; + ASSERT(resource->type->stop); + pre_nif_noproc(&msg_env, resource->type->owner, NULL); + resource->type->stop(&msg_env.env, resource->data); + post_nif_noproc(&msg_env); +} + void* enif_alloc_resource(ErlNifResourceType* type, size_t size) { Binary* bin = erts_create_magic_binary_x(SIZEOF_ErlNifResource(size), @@ -3174,6 +3183,11 @@ static struct erl_module_nif* create_lib(const ErlNifEntry* src) dst->funcs = lib->_funcs_copy_; dst->options = 0; } + if (AT_LEAST_VERSION(src, 2, 12)) { + dst->sizeof_ErlNifResourceTypeInit = src->sizeof_ErlNifResourceTypeInit; + } else { + dst->sizeof_ErlNifResourceTypeInit = 0; + } return lib; }; diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 413b4f7343..ce8caaf729 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -52,7 +52,7 @@ ** 2.11: 19.0 enif_snprintf */ #define ERL_NIF_MAJOR_VERSION 2 -#define ERL_NIF_MINOR_VERSION 11 +#define ERL_NIF_MINOR_VERSION 12 /* * The emulator will refuse to load a nif-lib with a major version @@ -122,6 +122,9 @@ typedef struct enif_entry_t /* Added in 2.7 */ unsigned options; /* Unused. Can be set to 0 or 1 (dirty sched config) */ + + /* Added in 2.12 */ + size_t sizeof_ErlNifResourceTypeInit; }ErlNifEntry; @@ -135,8 +138,20 @@ typedef struct void* ref_bin; }ErlNifBinary; +typedef struct { + void (*dtor)(ErlNifEnv* env, void* obj); + void (*stop)(ErlNifEnv* env, void* obj); /* at ERL_NIF_SELECT_STOP event */ +} ErlNifResourceTypeInit; + typedef struct enif_resource_type_t ErlNifResourceType; typedef void ErlNifResourceDtor(ErlNifEnv*, void*); +typedef void ErlNifResourceStop(ErlNifEnv*, void*); +typedef void ErlNifResourceExit(ErlNifEnv*, void*); + +//#ifndef ERL_SYS_DRV +typedef int ErlNifEvent; /* An event to be selected on. */ +//#endif + typedef enum { ERL_NIF_RT_CREATE = 1, @@ -292,7 +307,8 @@ ERL_NIF_INIT_DECL(NAME) \ FUNCS, \ LOAD, RELOAD, UPGRADE, UNLOAD, \ ERL_NIF_VM_VARIANT, \ - 1 \ + 1, \ + sizeof(ErlNifResourceTypeInit) \ }; \ ERL_NIF_INIT_BODY; \ return &entry; \ diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index 9a8f216773..9163ce25eb 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -175,6 +175,8 @@ ERL_NIF_API_FUNC_DECL(size_t, enif_binary_to_term, (ErlNifEnv *env, const unsign ERL_NIF_API_FUNC_DECL(int, enif_port_command, (ErlNifEnv *env, const ErlNifPort* to_port, ErlNifEnv *msg_env, ERL_NIF_TERM msg)); ERL_NIF_API_FUNC_DECL(int,enif_thread_type,(void)); ERL_NIF_API_FUNC_DECL(int,enif_snprintf,(char * buffer, size_t size, const char *format, ...)); +ERL_NIF_API_FUNC_DECL(int,enif_select,(ErlNifEnv* env, ErlNifEvent e, enum ErlNifSelectFlags flags, void* obj, ERL_NIF_TERM ref)); +ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type_x,(ErlNifEnv*, const char* name_str, const ErlNifResourceTypeInit*, ErlNifResourceFlags flags, ErlNifResourceFlags* tried)); /* ** ADD NEW ENTRIES HERE (before this comment) !!! @@ -332,6 +334,8 @@ ERL_NIF_API_FUNC_DECL(int,enif_snprintf,(char * buffer, size_t size, const char # define enif_port_command ERL_NIF_API_FUNC_MACRO(enif_port_command) # define enif_thread_type ERL_NIF_API_FUNC_MACRO(enif_thread_type) # define enif_snprintf ERL_NIF_API_FUNC_MACRO(enif_snprintf) +# define enif_select ERL_NIF_API_FUNC_MACRO(enif_select) +# define enif_open_resource_type_x ERL_NIF_API_FUNC_MACRO(enif_open_resource_type_x) /* ** ADD NEW ENTRIES HERE (before this comment) diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 2b2f3c5cdc..5c5693a315 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -42,6 +42,7 @@ #include "erl_utils.h" #include "erl_port.h" #include "erl_gc.h" +#include "erl_nif.h" struct enif_func_t; @@ -58,6 +59,33 @@ struct enif_environment_t /* ErlNifEnv */ Process *tracee; int exiting; /* boolean (dirty nifs might return in exiting state) */ }; +struct enif_resource_type_t +{ + struct enif_resource_type_t* next; /* list of all resource types */ + struct enif_resource_type_t* prev; + struct erl_module_nif* owner; /* that created this type and thus implements the destructor*/ + ErlNifResourceDtor* dtor; /* user destructor function */ + ErlNifResourceStop* stop; + erts_refc_t refc; /* num of resources of this type (HOTSPOT warning) + +1 for active erl_module_nif */ + Eterm module; + Eterm name; +}; +typedef struct enif_resource_t +{ + struct enif_resource_type_t* type; +#ifdef DEBUG + erts_refc_t nif_refc; +# ifdef ARCH_32 + byte align__[4]; +# endif +#endif + + char data[1]; +}ErlNifResource; + +#define DATA_TO_RESOURCE(PTR) ((ErlNifResource*)((char*)(PTR) - offsetof(ErlNifResource,data))) + extern void erts_pre_nif(struct enif_environment_t*, Process*, struct erl_module_nif*, Process* tracee); extern void erts_post_nif(struct enif_environment_t* env); @@ -67,6 +95,7 @@ extern void erts_pre_dirty_nif(ErtsSchedulerData *, struct erl_module_nif*); extern void erts_post_dirty_nif(struct enif_environment_t* env); #endif +extern void erts_resource_stop(ErlNifResource* resource); extern Eterm erts_nif_taints(Process* p); extern void erts_print_nif_taints(fmtfn_t to, void* to_arg); void erts_unload_nif(struct erl_module_nif* nif); -- cgit v1.2.3 From 73e6fc31e8ef1808fb079a4bac2f1ccff44efa3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Wed, 30 Nov 2016 15:16:17 +0100 Subject: erts: Add driver and nif locks to lcnt --- erts/emulator/beam/erl_drv_thread.c | 98 +++++++++++++++++++++++++++++++------ 1 file changed, 84 insertions(+), 14 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index 92edce5176..0e6aadf568 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -54,6 +54,9 @@ fatal_error(int err, char *func) struct ErlDrvMutex_ { ethr_mutex mtx; +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock_t lcnt; +#endif char *name; }; @@ -64,6 +67,9 @@ struct ErlDrvCond_ { struct ErlDrvRWLock_ { ethr_rwmutex rwmtx; +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock_t lcnt; +#endif char *name; }; @@ -161,14 +167,17 @@ erl_drv_mutex_create(char *name) opt.posix_compliant = 1; if (ethr_mutex_init_opt(&dmtx->mtx, &opt) != 0) { erts_free(ERTS_ALC_T_DRV_MTX, (void *) dmtx); - dmtx = NULL; + return NULL; } - else if (!name) - dmtx->name = no_name; - else { + if (name) { dmtx->name = ((char *) dmtx) + sizeof(ErlDrvMutex); sys_strcpy(dmtx->name, name); + } else { + dmtx->name = no_name; } +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_init_lock(&dmtx->lcnt, dmtx->name, ERTS_LCNT_LT_MUTEX); +#endif } return dmtx; #else @@ -180,7 +189,11 @@ void erl_drv_mutex_destroy(ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res = dmtx ? ethr_mutex_destroy(&dmtx->mtx) : EINVAL; + int res; +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_destroy_lock(&dmtx->lcnt); +#endif + res = dmtx ? ethr_mutex_destroy(&dmtx->mtx) : EINVAL; if (res != 0) fatal_error(res, "erl_drv_mutex_destroy()"); erts_free(ERTS_ALC_T_DRV_MTX, (void *) dmtx); @@ -202,9 +215,14 @@ int erl_drv_mutex_trylock(ErlDrvMutex *dmtx) { #ifdef USE_THREADS + int res; if (!dmtx) fatal_error(EINVAL, "erl_drv_mutex_trylock()"); - return ethr_mutex_trylock(&dmtx->mtx); + res = ethr_mutex_trylock(&dmtx->mtx); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_trylock(&dmtx->lcnt, res); +#endif + return res; #else return 0; #endif @@ -216,8 +234,14 @@ erl_drv_mutex_lock(ErlDrvMutex *dmtx) #ifdef USE_THREADS if (!dmtx) fatal_error(EINVAL, "erl_drv_mutex_lock()"); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock(&dmtx->lcnt); +#endif ethr_mutex_lock(&dmtx->mtx); #endif +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock_post(&dmtx->lcnt); +#endif } void @@ -226,6 +250,9 @@ erl_drv_mutex_unlock(ErlDrvMutex *dmtx) #ifdef USE_THREADS if (!dmtx) fatal_error(EINVAL, "erl_drv_mutex_unlock()"); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_unlock(&dmtx->lcnt); +#endif ethr_mutex_unlock(&dmtx->mtx); #endif } @@ -306,10 +333,18 @@ erl_drv_cond_wait(ErlDrvCond *dcnd, ErlDrvMutex *dmtx) if (!dcnd || !dmtx) { fatal_error(EINVAL, "erl_drv_cond_wait()"); } +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_unlock(&dmtx->lcnt); +#endif while (1) { int res = ethr_cond_wait(&dcnd->cnd, &dmtx->mtx); - if (res == 0) + if (res == 0) { +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock(&dmtx->lcnt); + erts_lcnt_lock_post(&dmtx->lcnt); +#endif break; + } } #endif } @@ -324,14 +359,17 @@ erl_drv_rwlock_create(char *name) if (drwlck) { if (ethr_rwmutex_init(&drwlck->rwmtx) != 0) { erts_free(ERTS_ALC_T_DRV_RWLCK, (void *) drwlck); - drwlck = NULL; + return NULL; } - else if (!name) - drwlck->name = no_name; - else { + if (name) { drwlck->name = ((char *) drwlck) + sizeof(ErlDrvRWLock); sys_strcpy(drwlck->name, name); + } else { + drwlck->name = no_name; } +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_init_lock(&drwlck->lcnt, drwlck->name, ERTS_LCNT_LT_RWMUTEX); +#endif } return drwlck; #else @@ -343,7 +381,11 @@ void erl_drv_rwlock_destroy(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_destroy(&drwlck->rwmtx) : EINVAL; + int res; +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_destroy_lock(&drwlck->lcnt); +#endif + res = drwlck ? ethr_rwmutex_destroy(&drwlck->rwmtx) : EINVAL; if (res != 0) fatal_error(res, "erl_drv_rwlock_destroy()"); erts_free(ERTS_ALC_T_DRV_RWLCK, (void *) drwlck); @@ -364,9 +406,14 @@ int erl_drv_rwlock_tryrlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS + int res; if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_tryrlock()"); - return ethr_rwmutex_tryrlock(&drwlck->rwmtx); + res = ethr_rwmutex_tryrlock(&drwlck->rwmtx); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LCNT_LO_READ); +#endif + return res; #else return 0; #endif @@ -378,7 +425,13 @@ erl_drv_rwlock_rlock(ErlDrvRWLock *drwlck) #ifdef USE_THREADS if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_rlock()"); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ); +#endif ethr_rwmutex_rlock(&drwlck->rwmtx); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock_post(&drwlck->lcnt); +#endif #endif } @@ -388,6 +441,9 @@ erl_drv_rwlock_runlock(ErlDrvRWLock *drwlck) #ifdef USE_THREADS if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_runlock()"); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ); +#endif ethr_rwmutex_runlock(&drwlck->rwmtx); #endif } @@ -396,9 +452,14 @@ int erl_drv_rwlock_tryrwlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS + int res; if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_tryrwlock()"); - return ethr_rwmutex_tryrwlock(&drwlck->rwmtx); + res = ethr_rwmutex_tryrwlock(&drwlck->rwmtx); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LCNT_LO_READ_WRITE); +#endif + return res; #else return 0; #endif @@ -410,7 +471,13 @@ erl_drv_rwlock_rwlock(ErlDrvRWLock *drwlck) #ifdef USE_THREADS if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_rwlock()"); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ_WRITE); +#endif ethr_rwmutex_rwlock(&drwlck->rwmtx); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock_post(&drwlck->lcnt); +#endif #endif } @@ -420,6 +487,9 @@ erl_drv_rwlock_rwunlock(ErlDrvRWLock *drwlck) #ifdef USE_THREADS if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_rwunlock()"); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ_WRITE); +#endif ethr_rwmutex_rwunlock(&drwlck->rwmtx); #endif } -- cgit v1.2.3 From 8e5c762d597b9a3ba4d78bcf57ebe379ad666599 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 2 Dec 2016 16:44:43 +0100 Subject: erts: Fix whitespace errors in lock counter --- erts/emulator/beam/erl_lock_count.c | 25 ++++++++++++++----------- erts/emulator/beam/erl_lock_count.h | 4 ++-- 2 files changed, 16 insertions(+), 13 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index 481e92b2cd..6354fc8663 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -55,11 +55,11 @@ static erts_lcnt_thread_data_t *lcnt_thread_data[2048]; /* local functions */ static ERTS_INLINE void lcnt_lock(void) { - ethr_mutex_lock(&lcnt_data_lock); + ethr_mutex_lock(&lcnt_data_lock); } static ERTS_INLINE void lcnt_unlock(void) { - ethr_mutex_unlock(&lcnt_data_lock); + ethr_mutex_unlock(&lcnt_data_lock); } const int log2_tab64[64] = { @@ -159,7 +159,7 @@ static erts_lcnt_thread_data_t *lcnt_thread_data_alloc(void) { lcnt_thread_data[eltd->id] = eltd; return eltd; -} +} static erts_lcnt_thread_data_t *lcnt_get_thread_data(void) { return (erts_lcnt_thread_data_t *)ethr_tsd_get(lcnt_thr_data_key); @@ -254,9 +254,9 @@ void erts_lcnt_init() { /* init lock */ if (ethr_mutex_init(&lcnt_data_lock) != 0) abort(); - /* init tsd */ + /* init tsd */ lcnt_n_thr = 0; - ethr_tsd_key_create(&lcnt_thr_data_key,"lcnt_data"); + ethr_tsd_key_create(&lcnt_thr_data_key, "lcnt_data"); lcnt_lock(); @@ -352,11 +352,11 @@ void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) /* interface to erl_threads.h */ /* only lock on init and destroy, all others should use atomics */ -void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) { +void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) { erts_lcnt_init_lock_x(lock, name, flag, NIL); } -void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) { +void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) { int i; if (name == NULL) { ERTS_LCNT_CLEAR_FLAG(lock); return; } lcnt_lock(); @@ -382,7 +382,10 @@ void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eter erts_lcnt_list_insert(erts_lcnt_data->current_locks, lock); lcnt_unlock(); } -/* init empty, instead of zero struct */ + +/* init empty, instead of zero struct + * used by process locks probes + */ void erts_lcnt_init_lock_empty(erts_lcnt_lock_t *lock) { lock->next = NULL; lock->prev = NULL; @@ -444,7 +447,7 @@ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { } /* we cannot acquire w_lock if either w or r are taken */ - /* we cannot acquire r_lock if w_lock is taken */ + /* we cannot acquire r_lock if w_lock is taken */ if ((w_state > 0) || (r_state > 0)) { eltd->lock_in_conflict = 1; @@ -561,7 +564,7 @@ void erts_lcnt_unlock(erts_lcnt_lock_t *lock) { if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; #ifdef DEBUG { - erts_aint_t w_state; + erts_aint_t w_state; erts_aint_t flowstate; /* flowstate */ @@ -647,7 +650,7 @@ Uint16 erts_lcnt_set_rt_opt(Uint16 opt) { return prev; } -Uint16 erts_lcnt_clear_rt_opt(Uint16 opt) { +Uint16 erts_lcnt_clear_rt_opt(Uint16 opt) { Uint16 prev; prev = (erts_lcnt_rt_options & opt); erts_lcnt_rt_options &= ~opt; diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h index 3e8dcefe69..4f838f7faa 100644 --- a/erts/emulator/beam/erl_lock_count.h +++ b/erts/emulator/beam/erl_lock_count.h @@ -148,13 +148,13 @@ typedef struct erts_lcnt_lock_stats_s { typedef struct erts_lcnt_lock_s { char *name; /* lock name */ Uint16 flag; /* lock type */ - Eterm id; /* id if possible */ + Eterm id; /* id if possible */ #ifdef DEBUG ethr_atomic_t flowstate; #endif - /* lock states */ + /* lock states */ ethr_atomic_t w_state; /* 0 not taken, otherwise n threads waiting */ ethr_atomic_t r_state; /* 0 not taken, > 0 -> writes will wait */ -- cgit v1.2.3 From 7d161f5b475575bd79bd90977b3a79334a8ec658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 2 Dec 2016 17:20:00 +0100 Subject: erts: Remove unnecessary 'enable_lcnt' option in locks --- erts/emulator/beam/erl_alloc_util.c | 8 ++++---- erts/emulator/beam/erl_port_task.h | 8 +++----- erts/emulator/beam/erl_process_lock.c | 21 +++++++-------------- erts/emulator/beam/erl_smp.h | 4 ++-- erts/emulator/beam/erl_threads.h | 33 +++++++++------------------------ erts/emulator/beam/io.c | 21 +++++++++------------ 6 files changed, 34 insertions(+), 61 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 61cce224dd..d346f8c8b6 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -6134,18 +6134,18 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) #ifdef USE_THREADS if (init->ts) { allctr->thread_safe = 1; - + #ifdef ERTS_ENABLE_LOCK_COUNT erts_mtx_init_x_opt(&allctr->mutex, "alcu_allocator", make_small(allctr->alloc_no), - ERTS_LCNT_LT_ALLOC,1); + ERTS_LCNT_LT_ALLOC); #else erts_mtx_init_x(&allctr->mutex, "alcu_allocator", - make_small(allctr->alloc_no),1); + make_small(allctr->alloc_no)); #endif /*ERTS_ENABLE_LOCK_COUNT*/ - + #ifdef DEBUG allctr->debug.saved_tid = 0; #endif diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index 2a6bd165a3..e3550e878e 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -188,13 +188,11 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp, Eterm instr_id) ptsp->taskq.in.last = NULL; erts_smp_atomic32_init_nob(&ptsp->flags, 0); #ifdef ERTS_SMP - erts_mtx_init_x(&ptsp->mtx, lock_str, instr_id, #ifdef ERTS_ENABLE_LOCK_COUNT - (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK) -#else - 1 + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)) + lock_str = NULL; #endif - ); + erts_mtx_init_x(&ptsp->mtx, lock_str, instr_id); #endif } diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index a69185bc5c..180df229eb 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -123,7 +123,7 @@ erts_init_proc_lock(int cpus) for (i = 0; i < ERTS_NO_OF_PIX_LOCKS; i++) { #ifdef ERTS_ENABLE_LOCK_COUNT erts_mtx_init_x(&erts_pix_locks[i].u.mtx, - "pix_lock", make_small(i), 1); + "pix_lock", make_small(i)); #else erts_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock"); #endif @@ -1027,39 +1027,32 @@ erts_proc_lock_init(Process *p) #endif #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL -#ifdef ERTS_ENABLE_LOCK_COUNT - int do_lock_count = 1; -#else - int do_lock_count = 0; -#endif - - erts_mtx_init_x(&p->lock.main, "proc_main", p->common.id, do_lock_count); + erts_mtx_init_x(&p->lock.main, "proc_main", p->common.id); ethr_mutex_lock(&p->lock.main.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.main.lc); #endif - erts_mtx_init_x(&p->lock.link, "proc_link", p->common.id, do_lock_count); + erts_mtx_init_x(&p->lock.link, "proc_link", p->common.id); ethr_mutex_lock(&p->lock.link.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.link.lc); #endif - erts_mtx_init_x(&p->lock.msgq, "proc_msgq", p->common.id, do_lock_count); + erts_mtx_init_x(&p->lock.msgq, "proc_msgq", p->common.id); ethr_mutex_lock(&p->lock.msgq.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.msgq.lc); #endif - erts_mtx_init_x(&p->lock.btm, "proc_btm", p->common.id, do_lock_count); + erts_mtx_init_x(&p->lock.btm, "proc_btm", p->common.id); ethr_mutex_lock(&p->lock.btm.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.btm.lc); #endif - erts_mtx_init_x(&p->lock.status, "proc_status", p->common.id, - do_lock_count); + erts_mtx_init_x(&p->lock.status, "proc_status", p->common.id); ethr_mutex_lock(&p->lock.status.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.status.lc); #endif - erts_mtx_init_x(&p->lock.trace, "proc_trace", p->common.id, do_lock_count); + erts_mtx_init_x(&p->lock.trace, "proc_trace", p->common.id); ethr_mutex_lock(&p->lock.trace.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.trace.lc); diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index 713ed50b86..14be511f86 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -1065,7 +1065,7 @@ ERTS_GLB_INLINE void erts_smp_mtx_init_x(erts_smp_mtx_t *mtx, char *name, Eterm extra) { #ifdef ERTS_SMP - erts_mtx_init_x(mtx, name, extra, 1); + erts_mtx_init_x(mtx, name, extra); #endif } @@ -1073,7 +1073,7 @@ ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, char *name, Eterm extra) { #ifdef ERTS_SMP - erts_mtx_init_locked_x(mtx, name, extra, 1); + erts_mtx_init_locked_x(mtx, name, extra); #endif } diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index eccd49f2a9..9e75f6fee5 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -479,14 +479,9 @@ ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void)); ERTS_GLB_INLINE erts_tid_t erts_thr_self(void); ERTS_GLB_INLINE int erts_thr_getname(erts_tid_t tid, char *buf, size_t len); ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y); -ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra, - int enable_lcnt); -ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, - Uint16 opt, int enable_lcnt); -ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx, - char *name, - Eterm extra, - int enable_lcnt); +ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra); +ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt); +ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra); ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx); @@ -2164,7 +2159,7 @@ erts_equal_tids(erts_tid_t x, erts_tid_t y) } ERTS_GLB_INLINE void -erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt) +erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra) { #ifdef USE_THREADS int res = ethr_mutex_init(&mtx->mtx); @@ -2174,17 +2169,13 @@ erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt) erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - if (enable_lcnt) - erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); - else - erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra); + erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); #endif #endif } ERTS_GLB_INLINE void -erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt, - int enable_lcnt) +erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt) { #ifdef USE_THREADS int res = ethr_mutex_init(&mtx->mtx); @@ -2194,17 +2185,14 @@ erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt, erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - if (enable_lcnt) - erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra); - else - erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX | opt, extra); + erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra); #endif #endif } ERTS_GLB_INLINE void -erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt) +erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra) { #ifdef USE_THREADS int res = ethr_mutex_init(&mtx->mtx); @@ -2214,10 +2202,7 @@ erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - if (enable_lcnt) - erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); - else - erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra); + erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); #endif ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_CHECK diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 4f131c74de..33b74f30b7 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -259,13 +259,11 @@ static ERTS_INLINE void port_init_instr(Port *prt ASSERT(prt->drv_ptr && prt->lock); if (!prt->drv_ptr->lock) { char *lock_str = "port_lock"; - erts_mtx_init_locked_x(prt->lock, lock_str, id, #ifdef ERTS_ENABLE_LOCK_COUNT - (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK) -#else - 0 + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)) + lock_str = NULL; #endif - ); + erts_mtx_init_locked_x(prt->lock, lock_str, id); } #endif erts_port_task_init_sched(&prt->sched, id); @@ -7107,7 +7105,7 @@ driver_pdl_create(ErlDrvPort dp) return NULL; pdl = erts_alloc(ERTS_ALC_T_PORT_DATA_LOCK, sizeof(struct erl_drv_port_data_lock)); - erts_mtx_init_x(&pdl->mtx, "port_data_lock", pp->common.id, 1); + erts_mtx_init_x(&pdl->mtx, "port_data_lock", pp->common.id); pdl_init_refc(pdl); erts_port_inc_refc(pp); pdl->prt = pp; @@ -8290,14 +8288,13 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle) erts_mtx_init_x(drv->lock, "driver_lock", #if defined(ERTS_ENABLE_LOCK_CHECK) || defined(ERTS_ENABLE_LOCK_COUNT) - erts_atom_put((byte *) drv->name, - sys_strlen(drv->name), - ERTS_ATOM_ENC_LATIN1, - 1), + erts_atom_put((byte *) drv->name, + sys_strlen(drv->name), + ERTS_ATOM_ENC_LATIN1, + 1) #else - NIL, + NIL #endif - 1 ); } #endif -- cgit v1.2.3 From 3eddb0f762de248d3230b38bc9d478bfbc8e7331 Mon Sep 17 00:00:00 2001 From: Erlang/OTP Date: Wed, 7 Dec 2016 13:15:31 +0100 Subject: Update copyright-year --- erts/emulator/beam/erl_hl_timer.h | 2 +- erts/emulator/beam/erl_msacc.c | 2 +- erts/emulator/beam/erl_msacc.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h index 705be94532..9cdcd581a0 100644 --- a/erts/emulator/beam/erl_hl_timer.h +++ b/erts/emulator/beam/erl_hl_timer.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2015. All Rights Reserved. + * Copyright Ericsson AB 2015-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c index 421445fbad..7ddf49937f 100644 --- a/erts/emulator/beam/erl_msacc.c +++ b/erts/emulator/beam/erl_msacc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2014-2015. All Rights Reserved. + * Copyright Ericsson AB 2014-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h index ad7c8c5eee..4c8e1c8e22 100644 --- a/erts/emulator/beam/erl_msacc.h +++ b/erts/emulator/beam/erl_msacc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2014-2015. All Rights Reserved. + * Copyright Ericsson AB 2014-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. -- cgit v1.2.3 From 2f8d59aa9e8a96d094172db339fd94aae45a90b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kan=20Mattsson?= Date: Wed, 2 Nov 2016 12:56:39 +0100 Subject: erts: Make depth of current_stacktrace configurable The BIF process_info(Pid, current_stacktrace) truncates the stacktrace. The old behavior was to truncate long stacktraces to max 8 items. And this was hard coded. Now it is truncated to the value of system_flag(backtrace_depth) instead. The backtrace_depth defaults to 8, but is configurable. --- erts/emulator/beam/erl_bif_info.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 735aabbee3..88a052cad7 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1672,11 +1672,11 @@ current_stacktrace(Process* p, Process* rp, Eterm** hpp) Eterm mfa; Eterm res = NIL; - depth = 8; + depth = erts_backtrace_depth; sz = offsetof(struct StackTrace, trace) + sizeof(BeamInstr *)*depth; s = (struct StackTrace *) erts_alloc(ERTS_ALC_T_TMP, sz); s->depth = 0; - if (rp->i) { + if (depth > 0 && rp->i) { s->trace[s->depth++] = rp->i; depth--; } -- cgit v1.2.3 From 064a2630129caadbca58aaebe83f78735c1e374d Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 19 Dec 2016 14:06:12 +0100 Subject: Fix printout of timer data in crash dump --- erts/emulator/beam/erl_hl_timer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index d29d079fc5..cc61f9abf1 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -2865,7 +2865,8 @@ btm_print(ErtsHLTimer *tmr, void *vbtmp) if (tmr->timeout <= btmp->now) left = 0; - left = ERTS_CLKTCKS_TO_MSEC(tmr->timeout - btmp->now); + else + left = ERTS_CLKTCKS_TO_MSEC(tmr->timeout - btmp->now); receiver = ((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME) ? tmr->receiver.name -- cgit v1.2.3 From d44f2e85c058e10082427a354a7d65e9af607501 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 25 Nov 2016 15:03:33 +0100 Subject: Fix stack-trace generated by a traced process --- erts/emulator/beam/beam_emu.c | 45 ++++++++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 11 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 59a9ea1417..8c9bc1fa65 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -5844,12 +5844,13 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, s->depth = 0; /* - * If the failure was in a BIF other than 'error', 'exit' or - * 'throw', find the bif-table index and save the argument - * registers by consing up an arglist. + * If the failure was in a BIF other than 'error/1', 'error/2', + * 'exit/1' or 'throw/1', find the bif-table index and save the + * argument registers by consing up an arglist. */ - if (bf != NULL && bf != error_1 && bf != error_2 && - bf != exit_1 && bf != throw_1) { + if (bf != NULL && bf != error_1 && bf != error_2 && bf != exit_1 + && bf != throw_1 && bf != wrap_error_1 && bf != wrap_error_2 + && bf != wrap_exit_1 && bf != wrap_throw_1) { int i; int a = 0; for (i = 0; i < BIF_SIZE; i++) { @@ -5945,12 +5946,30 @@ erts_save_stacktrace(Process* p, struct StackTrace* s, int depth) p->cp) { /* Cannot follow cp here - code may be unloaded */ BeamInstr *cpp = p->cp; + int trace_cp; if (cpp == beam_exception_trace || cpp == beam_return_trace) { /* Skip return_trace parameters */ ptr += 2; + trace_cp = 1; } else if (cpp == beam_return_to_trace) { /* Skip return_to_trace parameters */ ptr += 1; + trace_cp = 1; + } + else { + trace_cp = 0; + } + if (trace_cp && s->pc == cpp) { + /* + * If process 'cp' points to a return/exception trace + * instruction and 'cp' has been saved as 'pc' in + * stacktrace, we need to update 'pc' in stacktrace + * with the actual 'cp' located on the top of the + * stack; otherwise, we will lose the top stackframe + * when building the stack trace. + */ + ASSERT(is_CP(p->stop[0])); + s->pc = cp_val(p->stop[0]); } } while (ptr < STACK_START(p) && depth > 0) { @@ -6070,12 +6089,15 @@ build_stacktrace(Process* c_p, Eterm exc) { erts_set_current_function(&fi, s->current); } + depth = s->depth; /* - * If fi.current is still NULL, default to the initial function + * If fi.current is still NULL, and we have no + * stack at all, default to the initial function * (e.g. spawn_link(erlang, abs, [1])). */ if (fi.current == NULL) { - erts_set_current_function(&fi, c_p->u.initial); + if (depth <= 0) + erts_set_current_function(&fi, c_p->u.initial); args = am_true; /* Just in case */ } else { args = get_args_from_exc(exc); @@ -6085,10 +6107,9 @@ build_stacktrace(Process* c_p, Eterm exc) { * Look up all saved continuation pointers and calculate * needed heap space. */ - depth = s->depth; stk = stkp = (FunctionInfo *) erts_alloc(ERTS_ALC_T_TMP, depth*sizeof(FunctionInfo)); - heap_size = fi.needed + 2; + heap_size = fi.current ? fi.needed + 2 : 0; for (i = 0; i < depth; i++) { erts_lookup_function_info(stkp, s->trace[i], 1); if (stkp->current) { @@ -6107,8 +6128,10 @@ build_stacktrace(Process* c_p, Eterm exc) { res = CONS(hp, mfa, res); hp += 2; } - hp = erts_build_mfa_item(&fi, hp, args, &mfa); - res = CONS(hp, mfa, res); + if (fi.current) { + hp = erts_build_mfa_item(&fi, hp, args, &mfa); + res = CONS(hp, mfa, res); + } erts_free(ERTS_ALC_T_TMP, (void *) stk); return res; -- cgit v1.2.3 From 0612bcedf520388d58a9441e9e6490bf86bf5a4b Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 28 Nov 2016 18:13:38 +0100 Subject: Fix stactrace for apply on error/[1,2], exit/1, or throw/1 --- erts/emulator/beam/beam_emu.c | 185 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 163 insertions(+), 22 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 8c9bc1fa65..66bccedd94 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1047,9 +1047,11 @@ static BeamInstr* handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) NOINLINE; static BeamInstr* call_error_handler(Process* p, BeamInstr* ip, Eterm* reg, Eterm func) NOINLINE; -static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity) NOINLINE; +static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity, + BeamInstr *I, Uint offs) NOINLINE; static BeamInstr* apply(Process* p, Eterm module, Eterm function, - Eterm args, Eterm* reg) NOINLINE; + Eterm args, Eterm* reg, + BeamInstr *I, Uint offs) NOINLINE; static BeamInstr* call_fun(Process* p, int arity, Eterm* reg, Eterm args) NOINLINE; static BeamInstr* apply_fun(Process* p, Eterm fun, @@ -3194,7 +3196,7 @@ do { \ OpCase(i_apply): { BeamInstr *next; HEAVY_SWAPOUT; - next = apply(c_p, r(0), x(1), x(2), reg); + next = apply(c_p, r(0), x(1), x(2), reg, NULL, 0); HEAVY_SWAPIN; if (next != NULL) { SET_CP(c_p, I+1); @@ -3208,7 +3210,7 @@ do { \ OpCase(i_apply_last_P): { BeamInstr *next; HEAVY_SWAPOUT; - next = apply(c_p, r(0), x(1), x(2), reg); + next = apply(c_p, r(0), x(1), x(2), reg, I, Arg(0)); HEAVY_SWAPIN; if (next != NULL) { SET_CP(c_p, (BeamInstr *) E[0]); @@ -3223,7 +3225,7 @@ do { \ OpCase(i_apply_only): { BeamInstr *next; HEAVY_SWAPOUT; - next = apply(c_p, r(0), x(1), x(2), reg); + next = apply(c_p, r(0), x(1), x(2), reg, I, 0); HEAVY_SWAPIN; if (next != NULL) { SET_I(next); @@ -3237,7 +3239,7 @@ do { \ BeamInstr *next; HEAVY_SWAPOUT; - next = fixed_apply(c_p, reg, Arg(0)); + next = fixed_apply(c_p, reg, Arg(0), NULL, 0); HEAVY_SWAPIN; if (next != NULL) { SET_CP(c_p, I+2); @@ -3252,7 +3254,7 @@ do { \ BeamInstr *next; HEAVY_SWAPOUT; - next = fixed_apply(c_p, reg, Arg(0)); + next = fixed_apply(c_p, reg, Arg(0), I, Arg(1)); HEAVY_SWAPIN; if (next != NULL) { SET_CP(c_p, (BeamInstr *) E[0]); @@ -6228,8 +6230,107 @@ apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity, return ep; } +static ERTS_INLINE void +apply_bif_error_adjustment(Process *p, Export *ep, + Eterm *reg, Uint arity, + BeamInstr *I, Uint stack_offset) +{ + /* + * I is only set when the apply is a tail call, i.e., + * from the instructions i_apply_only, i_apply_last_P, + * and apply_last_IP. + */ + if (I + && ep->code[3] == (BeamInstr) em_apply_bif + && (ep == bif_export[BIF_error_1] + || ep == bif_export[BIF_error_2] + || ep == bif_export[BIF_exit_1] + || ep == bif_export[BIF_throw_1])) { + /* + * We are about to tail apply one of the BIFs + * erlang:error/1, erlang:error/2, erlang:exit/1, + * or erlang:throw/1. Error handling of these BIFs is + * special! + * + * We need 'p->cp' to point into the calling + * function when handling the error after the BIF has + * been applied. This in order to get the topmost + * stackframe correct. Without the following adjustment, + * 'p->cp' will point into the function that called + * current function when handling the error. We add a + * dummy stackframe in order to achive this. + * + * Note that these BIFs unconditionally will cause + * an exception to be raised. That is, our modifications + * of 'p->cp' as well as the stack will be corrected by + * the error handling code. + * + * If we find an exception/return-to trace continuation + * pointer as the topmost continuation pointer, we do not + * need to do anything since the information already will + * be available for generation of the stacktrace. + */ + int apply_only = stack_offset == 0; + BeamInstr *cpp; + + if (apply_only) { + ASSERT(p->cp != NULL); + cpp = p->cp; + } + else { + ASSERT(is_CP(p->stop[0])); + cpp = cp_val(p->stop[0]); + } + + if (cpp != beam_exception_trace + && cpp != beam_return_trace + && cpp != beam_return_to_trace) { + Uint need = stack_offset /* bytes */ / sizeof(Eterm); + if (need == 0) + need = 1; /* i_apply_only */ + if (p->stop - p->htop < need) + erts_garbage_collect(p, (int) need, reg, arity+1); + p->stop -= need; + + if (apply_only) { + /* + * Called from the i_apply_only instruction. + * + * 'p->cp' contains continuation pointer pointing + * into the function that called current function. + * We push that continuation pointer onto the stack, + * and set 'p->cp' to point into current function. + */ + + p->stop[0] = make_cp(p->cp); + p->cp = I; + } + else { + /* + * Called from an i_apply_last_p, or apply_last_IP, + * instruction. + * + * Calling instruction will after we return read + * a continuation pointer from the stack and write + * it to 'p->cp', and then remove the topmost + * stackframe of size 'stack_offset'. + * + * We have sized the dummy-stackframe so that it + * will be removed by the instruction we currently + * are executing, and leave the stackframe that + * normally would have been removed intact. + * + */ + p->stop[0] = make_cp(I); + } + } + } +} + static BeamInstr* -apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) +apply( +Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg, +BeamInstr *I, Uint stack_offset) { int arity; Export* ep; @@ -6254,21 +6355,54 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) return 0; } - /* The module argument may be either an atom or an abstract module - * (currently implemented using tuples, but this might change). - */ - this = THE_NON_VALUE; - if (is_not_atom(module)) { - Eterm* tp; + while (1) { + Eterm m, f, a; + /* The module argument may be either an atom or an abstract module + * (currently implemented using tuples, but this might change). + */ + this = THE_NON_VALUE; + if (is_not_atom(module)) { + Eterm* tp; + + if (is_not_tuple(module)) goto error; + tp = tuple_val(module); + if (arityval(tp[0]) < 1) goto error; + this = module; + module = tp[1]; + if (is_not_atom(module)) goto error; + } - if (is_not_tuple(module)) goto error; - tp = tuple_val(module); - if (arityval(tp[0]) < 1) goto error; - this = module; - module = tp[1]; - if (is_not_atom(module)) goto error; + if (module != am_erlang || function != am_apply) + break; + + /* Adjust for multiple apply of apply/3... */ + + a = args; + if (is_list(a)) { + Eterm *consp = list_val(a); + m = CAR(consp); + a = CDR(consp); + if (is_list(a)) { + consp = list_val(a); + f = CAR(consp); + a = CDR(consp); + if (is_list(a)) { + consp = list_val(a); + a = CAR(consp); + if (is_nil(CDR(consp))) { + /* erlang:apply/3 */ + module = m; + function = f; + args = a; + if (is_not_atom(f)) + goto error; + continue; + } + } + } + } + break; /* != erlang:apply/3 */ } - /* * Walk down the 3rd parameter of apply (the argument list) and copy * the parameters to the x registers (reg[]). If the module argument @@ -6306,12 +6440,14 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) { save_calls(p, ep); } + apply_bif_error_adjustment(p, ep, reg, arity, I, stack_offset); DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep); return ep->addressv[erts_active_code_ix()]; } static BeamInstr* -fixed_apply(Process* p, Eterm* reg, Uint arity) +fixed_apply(Process* p, Eterm* reg, Uint arity, + BeamInstr *I, Uint stack_offset) { Export* ep; Eterm module; @@ -6341,6 +6477,10 @@ fixed_apply(Process* p, Eterm* reg, Uint arity) if (is_not_atom(module)) goto error; ++arity; } + + /* Handle apply of apply/3... */ + if (module == am_erlang && function == am_apply && arity == 3) + return apply(p, reg[0], reg[1], reg[2], reg, I, stack_offset); /* * Get the index into the export table, or failing that the export @@ -6355,6 +6495,7 @@ fixed_apply(Process* p, Eterm* reg, Uint arity) } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) { save_calls(p, ep); } + apply_bif_error_adjustment(p, ep, reg, arity, I, stack_offset); DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep); return ep->addressv[erts_active_code_ix()]; } -- cgit v1.2.3 From 9ffe2d285943b661317cee2b00d779a2e75a3374 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 1 Dec 2016 19:54:22 +0100 Subject: erts: Add testing + some minor fixes --- erts/emulator/beam/erl_nif.h | 1 - 1 file changed, 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index ce8caaf729..bf0f7b1f15 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -146,7 +146,6 @@ typedef struct { typedef struct enif_resource_type_t ErlNifResourceType; typedef void ErlNifResourceDtor(ErlNifEnv*, void*); typedef void ErlNifResourceStop(ErlNifEnv*, void*); -typedef void ErlNifResourceExit(ErlNifEnv*, void*); //#ifndef ERL_SYS_DRV typedef int ErlNifEvent; /* An event to be selected on. */ -- cgit v1.2.3 From 387ff8e3347d21e9ca5ad3d8c3a694bc79d38bca Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 8 Dec 2016 19:26:14 +0100 Subject: Add stop arguments: fd and is_direct_call --- erts/emulator/beam/erl_nif.c | 5 +++-- erts/emulator/beam/erl_nif.h | 16 ++++++++-------- erts/emulator/beam/global.h | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 27abba7cfd..4e41944ccb 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2124,12 +2124,13 @@ static void nif_resource_dtor(Binary* bin) } } -void erts_resource_stop(ErlNifResource* resource) +void erts_resource_stop(ErlNifResource* resource, ErlNifEvent e, + int is_direct_call) { struct enif_msg_environment_t msg_env; ASSERT(resource->type->stop); pre_nif_noproc(&msg_env, resource->type->owner, NULL); - resource->type->stop(&msg_env.env, resource->data); + resource->type->stop(&msg_env.env, resource->data, e, is_direct_call); post_nif_noproc(&msg_env); } diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index bf0f7b1f15..78e0fa1864 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -138,18 +138,18 @@ typedef struct void* ref_bin; }ErlNifBinary; -typedef struct { - void (*dtor)(ErlNifEnv* env, void* obj); - void (*stop)(ErlNifEnv* env, void* obj); /* at ERL_NIF_SELECT_STOP event */ -} ErlNifResourceTypeInit; +//#ifndef ERL_SYS_DRV +typedef int ErlNifEvent; /* An event to be selected on. */ +//#endif typedef struct enif_resource_type_t ErlNifResourceType; typedef void ErlNifResourceDtor(ErlNifEnv*, void*); -typedef void ErlNifResourceStop(ErlNifEnv*, void*); +typedef void ErlNifResourceStop(ErlNifEnv*, void*, ErlNifEvent, int is_direct_call); -//#ifndef ERL_SYS_DRV -typedef int ErlNifEvent; /* An event to be selected on. */ -//#endif +typedef struct { + ErlNifResourceDtor* dtor; + ErlNifResourceStop* stop; /* at ERL_NIF_SELECT_STOP event */ +} ErlNifResourceTypeInit; typedef enum { diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 5c5693a315..2decb56544 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -95,7 +95,7 @@ extern void erts_pre_dirty_nif(ErtsSchedulerData *, struct erl_module_nif*); extern void erts_post_dirty_nif(struct enif_environment_t* env); #endif -extern void erts_resource_stop(ErlNifResource* resource); +extern void erts_resource_stop(ErlNifResource*, ErlNifEvent, int is_direct_call); extern Eterm erts_nif_taints(Process* p); extern void erts_print_nif_taints(fmtfn_t to, void* to_arg); void erts_unload_nif(struct erl_module_nif* nif); -- cgit v1.2.3 From 4c5d33114edea787833d6aa1b0d51ea9d98b3321 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 16 Dec 2016 19:30:22 +0100 Subject: Add ErlNifSelectReturn --- erts/emulator/beam/erl_drv_nif.h | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h index e4ebcdb1d4..46bb06d642 100644 --- a/erts/emulator/beam/erl_drv_nif.h +++ b/erts/emulator/beam/erl_drv_nif.h @@ -56,6 +56,14 @@ enum ErlNifSelectFlags { ERL_NIF_SELECT_STOP = (1 << 2) }; +enum ErlNifSelectReturn { + ERL_NIF_SELECT_ERROR = (1 << 0), + ERL_NIF_SELECT_STOP_CALLED = (1 << 1), + ERL_NIF_SELECT_STOP_SCHEDULED = (1 << 2), + ERL_NIF_SELECT_INVALID_EVENT = (1 << 3), + ERL_NIF_SELECT_FAILED = (1 << 4) +}; + #ifdef SIZEOF_CHAR # define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR # undef SIZEOF_CHAR -- cgit v1.2.3 From ca7e946af9c2fdc86c1c74259ee7b6881c5aec1e Mon Sep 17 00:00:00 2001 From: Mikael Pettersson Date: Mon, 19 Dec 2016 11:17:05 +0100 Subject: erts: add erlang:system_info(atom_count) --- erts/emulator/beam/erl_bif_info.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 0e0842e139..9a3b78ae8d 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -2863,6 +2863,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("atom_limit",BIF_ARG_1)) { BIF_RET(make_small(erts_get_atom_limit())); } + else if (ERTS_IS_ATOM_STR("atom_count",BIF_ARG_1)) { + BIF_RET(make_small(atom_table_size())); + } else if (ERTS_IS_ATOM_STR("tolerant_timeofday",BIF_ARG_1)) { if (erts_has_time_correction() && erts_time_offset_state() == ERTS_TIME_OFFSET_FINAL) { -- cgit v1.2.3 From 3eef3cd38c6beb48bfe4c2e26ede2ab2d107c146 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 28 Dec 2016 18:21:44 +0100 Subject: Fix crash due to GC of node entry on dirty scheduler The dirty scheduler failed to set a timer. Dispatch this job to an ordinary scheduler. --- erts/emulator/beam/erl_hl_timer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index d29d079fc5..38bebc7576 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -2666,7 +2666,7 @@ erts_start_timer_callback(ErtsMonotonicTime tmo, tmo); twt = tmo < ERTS_TIMER_WHEEL_MSEC; - if (esdp) + if (esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)) start_callback_timer(esdp, twt, timeout_pos, -- cgit v1.2.3 From 32c71f85bd2d17a6c58f6d1fa450dfb8cf3de6cb Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 28 Dec 2016 18:27:54 +0100 Subject: Fix premature removal of process struct Refc on process struct could reach zero while it was still referred by dirty scheduler --- erts/emulator/beam/erl_process.c | 119 +++++++++++++++++++++++++++------------ 1 file changed, 82 insertions(+), 37 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index b345c35a7e..6810210e4d 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -2060,6 +2060,7 @@ handle_thr_prgr_later_op(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int wait #endif for (lops = 0; lops < ERTS_MAX_THR_PRGR_LATER_OPS; lops++) { ErtsThrPrgrLaterOp *lop = awdp->later_op.first; + if (!erts_thr_progress_has_reached_this(current, lop->later)) return aux_work & ~ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP; awdp->later_op.first = lop->next; @@ -6234,6 +6235,12 @@ check_dirty_enqueue_in_prio_queue(Process *c_p, int queue; erts_aint32_t dact, max_qbit; + /* Do not enqueue free process... */ + if (actual & ERTS_PSFLG_FREE) { + *newp &= ~ERTS_PSFLGS_DIRTY_WORK; + return ERTS_ENQUEUE_NOT; + } + /* Termination should be done on an ordinary scheduler */ if ((*newp) & ERTS_PSFLG_EXITING) { *newp &= ~ERTS_PSFLGS_DIRTY_WORK; @@ -6426,6 +6433,9 @@ select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t st /* * schedule_out_process() return with c_rq locked. + * + * Return non-zero value if caller should decrease + * reference count on the process when done with it... */ static ERTS_INLINE int schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, @@ -6480,12 +6490,18 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, erts_smp_runq_lock(c_rq); - return 0; - +#if !defined(ERTS_SMP) + /* Decrement refc if process struct is free... */ + return !!(n & ERTS_PSFLG_FREE); +#else + /* Decrement refc if scheduled out from dirty scheduler... */ + return !is_normal_sched; +#endif } else { Process* sched_p; + ASSERT(!(n & ERTS_PSFLG_FREE)); ASSERT(!(n & ERTS_PSFLG_SUSPENDED) || (n & (ERTS_PSFLG_ACTIVE_SYS | ERTS_PSFLG_DIRTY_ACTIVE_SYS))); @@ -6501,11 +6517,14 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, erts_smp_runq_lock(runq); + if (is_normal_sched && sched_p == p && ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) + erts_proc_inc_refc(p); /* Needs to be done before enqueue_process() */ + /* Enqueue the process */ enqueue_process(runq, (int) enq_prio, sched_p); if (runq == c_rq) - return 1; + return 0; erts_smp_runq_unlock(runq); @@ -6513,9 +6532,15 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, erts_smp_runq_lock(c_rq); - return 1; + /* + * Decrement refc if process is scheduled out by a + * dirty scheduler, and we have not just scheduled + * the process using the ordinary process struct + * on a dirty run-queue again... + */ + return !is_normal_sched && (sched_p != p + || !ERTS_RUNQ_IX_IS_DIRTY(runq->ix)); } - } static ERTS_INLINE void @@ -6530,8 +6555,17 @@ add2runq(int enqueue, erts_aint32_t prio, if (runq) { Process *sched_p; - if (enqueue > 0) + if (enqueue > 0) { sched_p = proc; + /* + * Refc on process struct (i.e. true struct, + * not proxy-struct) increased while in a + * dirty run-queue or executing on a dirty + * scheduler. + */ + if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) + erts_proc_inc_refc(proc); + } else { Process *pxy; @@ -9587,40 +9621,45 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) esdp->reductions += reds; - /* schedule_out_process() returns with rq locked! */ - schedule_out_process(rq, state, p, proxy_p, is_normal_sched); - proxy_p = NULL; + { + int dec_refc; - ERTS_PROC_REDUCTIONS_EXECUTED(esdp, rq, - (int) ERTS_PSFLGS_GET_USR_PRIO(state), - reds, - actual_reds); + /* schedule_out_process() returns with rq locked! */ + dec_refc = schedule_out_process(rq, state, p, + proxy_p, is_normal_sched); + proxy_p = NULL; - esdp->current_process = NULL; + ERTS_PROC_REDUCTIONS_EXECUTED(esdp, rq, + (int) ERTS_PSFLGS_GET_USR_PRIO(state), + reds, + actual_reds); + + esdp->current_process = NULL; #ifdef ERTS_SMP - p->scheduler_data = NULL; + p->scheduler_data = NULL; #endif - erts_smp_proc_unlock(p, (ERTS_PROC_LOCK_MAIN - | ERTS_PROC_LOCK_STATUS - | ERTS_PROC_LOCK_TRACE)); + erts_smp_proc_unlock(p, (ERTS_PROC_LOCK_MAIN + | ERTS_PROC_LOCK_STATUS + | ERTS_PROC_LOCK_TRACE)); - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER); + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER); - if (state & ERTS_PSFLG_FREE) { - if (!is_normal_sched) { - ASSERT(p->flags & F_DELAYED_DEL_PROC); - erts_proc_dec_refc(p); - } - else { #ifdef ERTS_SMP - ASSERT(esdp->free_process == p); - esdp->free_process = NULL; -#else - erts_proc_dec_refc(p); + if (state & ERTS_PSFLG_FREE) { + if (!is_normal_sched) { + ASSERT(p->flags & F_DELAYED_DEL_PROC); + } + else { + ASSERT(esdp->free_process == p); + esdp->free_process = NULL; + } + } #endif - } - } + + if (dec_refc) + erts_proc_dec_refc(p); + } #ifdef ERTS_SMP ASSERT(!esdp->free_process); @@ -9896,8 +9935,12 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) if (!(state & ERTS_PSFLG_PROXY)) psflg_band_mask &= ~ERTS_PSFLG_IN_RUNQ; else { + Eterm pid = p->common.id; proxy_p = p; - p = erts_proc_lookup_raw(proxy_p->common.id); + p = (is_normal_sched + ? erts_proc_lookup_raw(pid) + : erts_pid2proc_opt(NULL, 0, pid, 0, + ERTS_P2P_FLG_INC_REFC)); if (!p) { free_proxy_proc(proxy_p); proxy_p = NULL; @@ -9929,6 +9972,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) | ERTS_PSFLG_FREE))) #ifdef ERTS_DIRTY_SCHEDULERS | (((state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_FREE | ERTS_PSFLG_RUNNING_SYS | ERTS_PSFLG_EXITING)) @@ -9961,6 +10005,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) /* free and not queued by proxy */ erts_proc_dec_refc(p); } + if (!is_normal_sched) + erts_proc_dec_refc(p); goto pick_next_process; } state = new; @@ -13091,15 +13137,14 @@ erts_continue_exit_process(Process *p) } #ifdef ERTS_DIRTY_SCHEDULERS - if (a & (ERTS_PSFLG_DIRTY_RUNNING - | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + if (a & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { p->flags |= F_DELAYED_DEL_PROC; delay_del_proc = 1; /* - * The dirty scheduler will also decrease - * refc when done... + * The dirty scheduler decrease refc + * when done with the process... */ - erts_proc_inc_refc(p); } #endif -- cgit v1.2.3 From 88ae8fd6c952cd1f1edba1321de0dc97d8eafaf2 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 28 Dec 2016 18:56:18 +0100 Subject: Leave dirty work in dirty run-queues on multi scheduling block --- erts/emulator/beam/erl_process.c | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 6810210e4d..fba81db1cd 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -7165,16 +7165,8 @@ suspend_scheduler(ErtsSchedulerData *esdp) ASSERT(sched_type != ERTS_SCHED_NORMAL || no != 1); - if (sched_type != ERTS_SCHED_NORMAL) { - if (erts_smp_mtx_trylock(&schdlr_sspnd.mtx) == EBUSY) { - erts_smp_runq_unlock(esdp->run_queue); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - erts_smp_runq_lock(esdp->run_queue); - } - if (schdlr_sspnd.msb.ongoing) - evacuate_run_queue(esdp->run_queue, &sbp); + if (sched_type != ERTS_SCHED_NORMAL) erts_smp_runq_unlock(esdp->run_queue); - } else { evacuate_run_queue(esdp->run_queue, &sbp); @@ -7187,8 +7179,8 @@ suspend_scheduler(ErtsSchedulerData *esdp) sched_wall_time_change(esdp, 0); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); } + erts_smp_mtx_lock(&schdlr_sspnd.mtx); flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); if (flgs & ERTS_SSI_FLG_SUSPENDED) { @@ -7304,24 +7296,14 @@ suspend_scheduler(ErtsSchedulerData *esdp) while (1) { ErtsMonotonicTime current_time; - erts_aint32_t qmask; erts_aint32_t flgs; - qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue) - & ERTS_RUNQ_FLGS_QMASK); - - if (sched_type != ERTS_SCHED_NORMAL) { - if (qmask) { - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - erts_smp_runq_lock(esdp->run_queue); - if (schdlr_sspnd.msb.ongoing) - evacuate_run_queue(esdp->run_queue, &sbp); - erts_smp_runq_unlock(esdp->run_queue); - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - } + if (sched_type != ERTS_SCHED_NORMAL) aux_work = 0; - } else { + erts_aint32_t qmask; + qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue) + & ERTS_RUNQ_FLGS_QMASK); aux_work = erts_atomic32_read_acqb(&ssi->aux_work); -- cgit v1.2.3 From efc331a04191a124ee7428c25f2ec05ded748faf Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 28 Dec 2016 19:20:29 +0100 Subject: Fix VM global GC info for dirty schedulers --- erts/emulator/beam/erl_gc.c | 39 +++++++++++++++++++++++++++++++++++-- erts/emulator/beam/erl_lock_check.c | 1 + 2 files changed, 38 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index af799d09da..cb48b31b7e 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -176,6 +176,13 @@ typedef struct { erts_smp_atomic32_t refc; } ErtsGCInfoReq; +#ifdef ERTS_DIRTY_SCHEDULERS +static struct { + erts_mtx_t mtx; + ErtsGCInfo info; +} dirty_gc; +#endif + static ERTS_INLINE int gc_cost(Uint gc_moved_live_words, Uint resize_moved_words) { @@ -259,6 +266,11 @@ erts_init_gc(void) init_gc_info(&esdp->gc_info); } +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_mtx_init(&dirty_gc.mtx, "dirty_gc_info"); + init_gc_info(&dirty_gc.info); +#endif + init_gcireq_alloc(); } @@ -735,8 +747,19 @@ do_major_collection: monitor_large_heap(p); } - esdp->gc_info.garbage_cols++; - esdp->gc_info.reclaimed += reclaimed_now; +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { + erts_mtx_lock(&dirty_gc.mtx); + dirty_gc.info.garbage_cols++; + dirty_gc.info.reclaimed += reclaimed_now; + erts_mtx_unlock(&dirty_gc.mtx); + } + else +#endif + { + esdp->gc_info.garbage_cols++; + esdp->gc_info.reclaimed += reclaimed_now; + } FLAGS(p) &= ~F_FORCE_GC; p->live_hf_end = ERTS_INVALID_HFRAG_PTR; @@ -3017,6 +3040,18 @@ reply_gc_info(void *vgcirp) reclaimed = esdp->gc_info.reclaimed; garbage_cols = esdp->gc_info.garbage_cols; +#ifdef ERTS_DIRTY_SCHEDULERS + /* + * Add dirty schedulers info on requesting + * schedulers info + */ + if (gcirp->req_sched == esdp->no) { + erts_mtx_lock(&dirty_gc.mtx); + reclaimed += dirty_gc.info.reclaimed; + garbage_cols += dirty_gc.info.garbage_cols; + erts_mtx_unlock(&dirty_gc.mtx); + } +#endif sz = 0; hpp = NULL; diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 06266363b5..13a4b2cd93 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -129,6 +129,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "run_queue", "address" }, #ifdef ERTS_DIRTY_SCHEDULERS { "dirty_run_queue_sleep_list", "address" }, + { "dirty_gc_info", NULL }, #endif { "process_table", NULL }, { "cpu_info", NULL }, -- cgit v1.2.3 From 765e63dbffc4ec7f8cc3a7a1463151d6d863746f Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 30 Dec 2016 17:15:53 +0100 Subject: Multi scheduling block bug-fixes --- erts/emulator/beam/erl_process.c | 80 +++++++++++++++++++++++++--------------- erts/emulator/beam/erl_process.h | 2 +- 2 files changed, 52 insertions(+), 30 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index b345c35a7e..58097ad42f 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -7055,12 +7055,18 @@ typedef struct { } ErtsSchdlrSspndResume; static void -schdlr_sspnd_resume_proc(Eterm pid) +schdlr_sspnd_resume_proc(ErtsSchedType sched_type, Eterm pid) { - Process *p = erts_pid2proc(NULL, 0, pid, ERTS_PROC_LOCK_STATUS); + Process *p; + p = erts_pid2proc_opt(NULL, 0, pid, ERTS_PROC_LOCK_STATUS, + (sched_type != ERTS_SCHED_NORMAL + ? ERTS_P2P_FLG_INC_REFC + : 0)); if (p) { resume_process(p, ERTS_PROC_LOCK_STATUS); erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + if (sched_type != ERTS_SCHED_NORMAL) + erts_proc_dec_refc(p); } } @@ -7069,17 +7075,20 @@ schdlr_sspnd_resume_procs(ErtsSchedType sched_type, ErtsSchdlrSspndResume *resume) { if (is_internal_pid(resume->onln.chngr)) { - schdlr_sspnd_resume_proc(resume->onln.chngr); + schdlr_sspnd_resume_proc(sched_type, + resume->onln.chngr); resume->onln.chngr = NIL; } if (is_internal_pid(resume->onln.nxt)) { - schdlr_sspnd_resume_proc(resume->onln.nxt); + schdlr_sspnd_resume_proc(sched_type, + resume->onln.nxt); resume->onln.nxt = NIL; } while (resume->msb.chngrs) { ErtsProcList *plp = resume->msb.chngrs; resume->msb.chngrs = plp->next; - schdlr_sspnd_resume_proc(plp->pid); + schdlr_sspnd_resume_proc(sched_type, + plp->pid); proclist_destroy(plp); } } @@ -7198,15 +7207,18 @@ suspend_scheduler(ErtsSchedulerData *esdp) (void) erts_proclist_fetch(&msb[i]->chngq, &end_plp); /* resume processes that initiated the multi scheduling block... */ plp = msb[i]->chngq; - while (plp) { - erts_proclist_store_last(&msb[i]->blckrs, - proclist_copy(plp)); - plp = plp->next; - } - if (end_plp) + if (plp) { + ASSERT(end_plp); + ASSERT(msb[i]->ongoing); + do { + erts_proclist_store_last(&msb[i]->blckrs, + proclist_copy(plp)); + plp = plp->next; + } while (plp); end_plp->next = resume.msb.chngrs; - resume.msb.chngrs = msb[i]->chngq; - msb[i]->chngq = NULL; + resume.msb.chngrs = msb[i]->chngq; + msb[i]->chngq = NULL; + } } } } @@ -7415,12 +7427,23 @@ suspend_scheduler(ErtsSchedulerData *esdp) schdlr_sspnd_inc_nscheds(&schdlr_sspnd.active, sched_type); changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); - if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) - && schdlr_sspnd.online == schdlr_sspnd.active) { - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_MSB); - } - + if (changing) { + if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) + && !schdlr_sspnd.msb.ongoing + && schdlr_sspnd.online == schdlr_sspnd.active) { + erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_MSB); + } + if ((changing & ERTS_SCHDLR_SSPND_CHNG_NMSB) + && !schdlr_sspnd.nmsb.ongoing + && (schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL) + == schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL))) { + erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_NMSB); + } + } ASSERT(no <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, sched_type)); ASSERT((sched_type == ERTS_SCHED_NORMAL ? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) @@ -7553,7 +7576,7 @@ abort_sched_onln_chng_waitq(Process *p) erts_smp_mtx_unlock(&schdlr_sspnd.mtx); if (is_internal_pid(resume)) - schdlr_sspnd_resume_proc(resume); + schdlr_sspnd_resume_proc(ERTS_SCHED_NORMAL, resume); } ErtsSchedSuspendResult @@ -7951,21 +7974,20 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal } else { /* ------ UNBLOCK ------ */ if (p->flags & have_blckd_flg) { - ErtsProcList *plps[2]; + ErtsProcList **plpps[3] = {0}; ErtsProcList *plp; - int limit = 0; - plps[limit++] = erts_proclist_peek_first(msbp->blckrs); - if (all) - plps[limit++] = erts_proclist_peek_first(msbp->chngq); + plpps[0] = &msbp->blckrs; + if (all) + plpps[1] = &msbp->chngq; - for (ix = 0; ix < limit; ix++) { - plp = plps[ix]; + for (ix = 0; plpps[ix]; ix++) { + plp = erts_proclist_peek_first(*plpps[ix]); while (plp) { ErtsProcList *tmp_plp = plp; - plp = erts_proclist_peek_next(msbp->blckrs, plp); + plp = erts_proclist_peek_next(*plpps[ix], plp); if (erts_proclist_same(tmp_plp, p)) { - erts_proclist_remove(&msbp->blckrs, tmp_plp); + erts_proclist_remove(plpps[ix], tmp_plp); proclist_destroy(tmp_plp); if (!all) break; diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 9f7084c127..68fbb10602 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1684,7 +1684,7 @@ ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **list) return NULL; else { ErtsProcList *res = *list; - if (res == *list) + if (res->next == *list) *list = NULL; else *list = res->next; -- cgit v1.2.3 From 67355d1f2f4efb61ef9345751c15c0263cdc97a0 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 2 Jan 2017 13:09:47 +0100 Subject: Fix faulty assertion --- erts/emulator/beam/erl_process.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index b345c35a7e..d15430b902 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -8604,8 +8604,15 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, * from being selected for normal execution regardless * of locks held or not held on it... */ - ASSERT(!((ERTS_PSFLG_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS) - & erts_smp_atomic32_read_nob(&rp->state))); +#ifdef DEBUG + { + erts_aint32_t state; + state = erts_smp_atomic32_read_nob(&rp->state); + ASSERT((state & ERTS_PSFLG_PENDING_EXIT) + || !(state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS))); + } +#endif if (!suspend) resume_process(rp, pid_locks|ERTS_PROC_LOCK_STATUS); -- cgit v1.2.3 From fb4e0f8726d2e3033164fe6e44c49351c6b641de Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 21 Dec 2016 17:41:43 +0100 Subject: erts: Make erts_dbg_within_proc available for debug assertions. --- erts/emulator/beam/erl_gc.c | 12 +++--------- erts/emulator/beam/erl_gc.h | 7 +++---- 2 files changed, 6 insertions(+), 13 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 818f89f0e3..d907e7b351 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -3317,8 +3317,8 @@ erts_max_heap_size(Eterm arg, Uint *max_heap_size, Uint *max_heap_flags) #if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG) -static int -within2(Eterm *ptr, Process *p, Eterm *real_htop) +int +erts_dbg_within_proc(Eterm *ptr, Process *p, Eterm *real_htop) { ErlHeapFragment* bp; ErtsMessage* mp; @@ -3362,12 +3362,6 @@ within2(Eterm *ptr, Process *p, Eterm *real_htop) return 0; } -int -within(Eterm *ptr, Process *p) -{ - return within2(ptr, p, NULL); -} - #endif #ifdef ERTS_OFFHEAP_DEBUG @@ -3422,7 +3416,7 @@ erts_check_off_heap2(Process *p, Eterm *htop) else if (oheap <= u.ep && u.ep < ohtop) old = 1; else { - ERTS_CHK_OFFHEAP_ASSERT(within2(u.ep, p, htop)); + ERTS_CHK_OFFHEAP_ASSERT(erts_dbg_within_proc(u.ep, p, htop)); } } diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 54ea9ca3c0..9684478c1b 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -69,10 +69,6 @@ do { \ while (nelts--) *HTOP++ = *PTR++; \ } while(0) -#if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG) -int within(Eterm *ptr, Process *p); -#endif - #define ErtsInYoungGen(TPtr, Ptr, OldHeap, OldHeapSz) \ (!erts_is_literal((TPtr), (Ptr)) \ & !ErtsInArea((Ptr), (OldHeap), (OldHeapSz))) @@ -157,5 +153,8 @@ void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*); void erts_free_heap_frags(struct process* p); Eterm erts_max_heap_size_map(Sint, Uint, Eterm **, Uint *); int erts_max_heap_size(Eterm, Uint *, Uint *); +#if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG) +int erts_dbg_within_proc(Eterm *ptr, Process *p, Eterm* real_htop); +#endif #endif /* __ERL_GC_H__ */ -- cgit v1.2.3 From c9060f9b4289bf9a669ac651a1a7aeb97a28652c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Mon, 13 Apr 2015 18:03:33 +0200 Subject: erts: Add SIGHUP signal handler A received SIGHUP signal to beam will generate a '{notify, sighup}' message to the registered process 'erl_signal_server'. 'erl_signal_server' is a gen_event process. --- erts/emulator/beam/atom.names | 2 ++ erts/emulator/beam/register.c | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index b1aeed7889..363b17e27c 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -239,6 +239,7 @@ atom Eq='=:=' atom Eqeq='==' atom erl_tracer atom erlang +atom erl_signal_server atom ERROR='ERROR' atom error_handler atom error_logger @@ -590,6 +591,7 @@ atom set_tcw atom set_tcw_fake atom separate atom shared +atom sighup atom silent atom size atom sl_alloc diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c index acda51c9fc..7f60710124 100644 --- a/erts/emulator/beam/register.c +++ b/erts/emulator/beam/register.c @@ -272,13 +272,13 @@ erts_whereis_name_to_id(Process *c_p, Eterm name) int ix; HashBucket* b; #ifdef ERTS_SMP - ErtsProcLocks c_p_locks = c_p ? ERTS_PROC_LOCK_MAIN : 0; - -#ifdef ERTS_ENABLE_LOCK_CHECK - if (c_p) ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p); -#endif - + ErtsProcLocks c_p_locks = 0; + if (c_p) { + c_p_locks = ERTS_PROC_LOCK_MAIN; + ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p); + } reg_safe_read_lock(c_p, &c_p_locks); + if (c_p && !c_p_locks) erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); #endif -- cgit v1.2.3 From 5f4d538b2abfd1ab3f495879996edaa004309623 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 5 Jan 2017 20:06:04 +0100 Subject: erts: Assert sufficient space need after GC --- erts/emulator/beam/erl_gc.c | 1 + 1 file changed, 1 insertion(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index af799d09da..50f09277fa 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -765,6 +765,7 @@ do_major_collection: ASSERT(!p->mbuf); ASSERT(!ERTS_IS_GC_DESIRED(p)); + ASSERT(need <= HEAP_LIMIT(p) - HEAP_TOP(p)); return reds; } -- cgit v1.2.3 From 287edce0090a14fdd88a9e10d13327e74e52831a Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 5 Jan 2017 20:43:49 +0100 Subject: erts: Fix GC tracing to use temp heap Can't use HAlloc as it might consume part of callers 'need'. --- erts/emulator/beam/erl_gc.c | 3 ++- erts/emulator/beam/erl_nif.c | 4 ++-- erts/emulator/beam/erl_trace.c | 5 ++++- 3 files changed, 8 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 50f09277fa..b05209402d 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -602,7 +602,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, Uint reclaimed_now = 0; Eterm gc_trace_end_tag; int reds; - ErtsMonotonicTime start_time = 0; /* Shut up faulty warning... */ + ErtsMonotonicTime start_time; ErtsSchedulerData *esdp; erts_aint32_t state; ERTS_MSACC_PUSH_STATE_M(); @@ -610,6 +610,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); #endif + ERTS_UNDEF(start_time, 0); ERTS_CHK_MBUF_SZ(p); ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls) diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 6b265a8b80..19ce0f6965 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -3561,8 +3561,8 @@ Eterm erts_nif_call_function(Process *p, Process *tracee, #endif if (p) { /* This is almost a normal nif call like in beam_emu, - except that any heap fragment created in the nif will be - discarded without checking if anything in it is live. + except that any heap consumed by the nif will be + released without checking if anything in it is live. This is because we cannot do a GC here as we don't know the number of live registers that have to be preserved. This means that any heap part of the returned term may diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 8c84303997..ac9e91e31f 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1436,6 +1436,7 @@ void trace_gc(Process *p, Eterm what, Uint size, Eterm msg) { ErtsTracerNif *tnif = NULL; + Eterm* o_hp = NULL; Eterm* hp; Uint sz = 0; Eterm tup; @@ -1446,7 +1447,7 @@ trace_gc(Process *p, Eterm what, Uint size, Eterm msg) if (is_non_value(msg)) { (void) erts_process_gc_info(p, &sz, NULL, 0, 0); - hp = HAlloc(p, sz + 3 + 2); + o_hp = hp = erts_alloc(ERTS_ALC_T_TMP, (sz + 3 + 2) * sizeof(Eterm)); msg = erts_process_gc_info(p, NULL, &hp, 0, 0); tup = TUPLE2(hp, am_wordsize, make_small(size)); hp += 3; @@ -1455,6 +1456,8 @@ trace_gc(Process *p, Eterm what, Uint size, Eterm msg) send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_GC, what, msg, THE_NON_VALUE, am_true); + if (o_hp) + erts_free(ERTS_ALC_T_TMP, o_hp); } } -- cgit v1.2.3 From 5bf38ca930b75189cc83ed01e4a34eeb6a0892d2 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 9 Jan 2017 13:57:34 +0100 Subject: erts: Fix error report message for erlang:delete/1 --- erts/emulator/beam/beam_bif_load.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index ea1323d651..644fd9fbf9 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -658,7 +658,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) } else if (modp->old.code_hdr) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, "Module %T must be purged before loading\n", + erts_dsprintf(dsbufp, "Module %T must be purged before deleting\n", BIF_ARG_1); erts_send_error_to_logger(BIF_P->group_leader, dsbufp); ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); -- cgit v1.2.3 From 8df00f51a4a2c0c6788b3dcf3c6be56c50e44461 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 4 Jan 2017 17:51:54 +0100 Subject: Fix issues with abandoned heap --- erts/emulator/beam/beam_bif_load.c | 11 +++ erts/emulator/beam/erl_gc.c | 158 +++++++++++++++++++--------------- erts/emulator/beam/erl_message.c | 2 +- erts/emulator/beam/erl_process_dump.c | 3 + 4 files changed, 103 insertions(+), 71 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index ea1323d651..1307fe22b5 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -995,6 +995,12 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed if (any_heap_refs(c_p->heap, c_p->htop, literals, lit_bsize)) goto literal_gc; *redsp += 1; + if (c_p->abandoned_heap) { + if (any_heap_refs(c_p->abandoned_heap, c_p->abandoned_heap + c_p->heap_sz, + literals, lit_bsize)) + goto literal_gc; + *redsp += 1; + } if (any_heap_refs(c_p->old_heap, c_p->old_htop, literals, lit_bsize)) goto literal_gc; @@ -1295,6 +1301,11 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls #endif if (any_heap_refs(rp->heap, rp->htop, literals, lit_bsize)) goto try_literal_gc; + if (rp->abandoned_heap) { + if (any_heap_refs(rp->abandoned_heap, rp->abandoned_heap + rp->heap_sz, + literals, lit_bsize)) + goto try_literal_gc; + } if (any_heap_refs(rp->old_heap, rp->old_htop, literals, lit_bsize)) goto try_literal_gc; diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index af799d09da..561e7fbdee 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -110,7 +110,7 @@ typedef struct { static Uint setup_rootset(Process*, Eterm*, int, Rootset*); static void cleanup_rootset(Rootset *rootset); -static void remove_message_buffers(Process* p); +static void deallocate_previous_young_generation(Process *c_p); static Eterm *full_sweep_heaps(Process *p, int hibernate, Eterm *n_heap, Eterm* n_htop, @@ -153,7 +153,7 @@ static void init_gc_info(ErtsGCInfo *gcip); static Uint64 next_vheap_size(Process* p, Uint64 vheap, Uint64 vheap_sz); #ifdef HARDDEBUG -static void disallow_heap_frag_ref_in_heap(Process* p); +static void disallow_heap_frag_ref_in_heap(Process *p, Eterm *heap, Eterm *htop); static void disallow_heap_frag_ref_in_old_heap(Process* p); #endif @@ -477,24 +477,26 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int hsz = ssz + need + ERTS_DELAY_GC_EXTRA_FREE; hfrag = new_message_buffer(hsz); - hfrag->next = p->mbuf; - p->mbuf = hfrag; - p->mbuf_sz += hsz; p->heap = p->htop = &hfrag->mem[0]; p->hend = hend = &hfrag->mem[hsz]; p->stop = stop = hend - ssz; sys_memcpy((void *) stop, (void *) orig_stop, ssz * sizeof(Eterm)); if (p->abandoned_heap) { - /* Active heap already in a fragment; adjust it... */ - ErlHeapFragment *hfrag = ((ErlHeapFragment *) - (((char *) orig_heap) - - offsetof(ErlHeapFragment, mem))); - Uint unused = orig_hend - orig_htop; - ASSERT(hfrag->used_size == hfrag->alloc_size); - ASSERT(hfrag->used_size >= unused); - hfrag->used_size -= unused; - p->mbuf_sz -= unused; + /* + * Active heap already in a fragment; adjust it and + * save it into mbuf list... + */ + ErlHeapFragment *hfrag = ((ErlHeapFragment *) + (((char *) orig_heap) + - offsetof(ErlHeapFragment, mem))); + Uint used = orig_htop - orig_heap; + hfrag->used_size = used; + p->mbuf_sz += used; + ASSERT(hfrag->used_size <= hfrag->alloc_size); + ASSERT(!hfrag->off_heap.first && !hfrag->off_heap.overhead); + hfrag->next = p->mbuf; + p->mbuf = hfrag; } else { /* Do not leave a hole in the abandoned heap... */ @@ -556,17 +558,14 @@ young_gen_usage(Process *p) } } + hsz += p->htop - p->heap; aheap = p->abandoned_heap; - if (!aheap) - hsz += p->htop - p->heap; - else { + if (aheap) { /* used in orig heap */ if (p->flags & F_ABANDONED_HEAP_USE) hsz += aheap[p->heap_sz-1]; else hsz += p->heap_sz; - /* Remove unused part in latest fragment */ - hsz -= p->hend - p->htop; } return hsz; } @@ -789,6 +788,7 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) >= erts_proc_sched_data(p)->virtual_reds); } + /* * Place all living data on a the new heap; deallocate any old heap. * Meant to be used by hibernate/3. @@ -835,11 +835,11 @@ erts_garbage_collect_hibernate(Process* p) p->arg_reg, p->arity); - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, - (p->abandoned_heap - ? p->abandoned_heap - : p->heap), - p->heap_sz * sizeof(Eterm)); +#ifdef HARDDEBUG + disallow_heap_frag_ref_in_heap(p, heap, htop); +#endif + + deallocate_previous_young_generation(p); p->heap = heap; p->high_water = htop; @@ -874,8 +874,6 @@ erts_garbage_collect_hibernate(Process* p) sys_memcpy((void *) heap, (void *) p->heap, actual_size*sizeof(Eterm)); ERTS_HEAP_FREE(ERTS_ALC_T_TMP_HEAP, p->heap, p->heap_sz*sizeof(Eterm)); - remove_message_buffers(p); - p->stop = p->hend = heap + heap_size; offs = heap - p->heap; @@ -1494,22 +1492,16 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, } #endif - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, - (p->abandoned_heap - ? p->abandoned_heap - : HEAP_START(p)), - HEAP_SIZE(p) * sizeof(Eterm)); - p->abandoned_heap = NULL; - p->flags &= ~F_ABANDONED_HEAP_USE; +#ifdef HARDDEBUG + disallow_heap_frag_ref_in_heap(p, n_heap, n_htop); +#endif + + deallocate_previous_young_generation(p); + HEAP_START(p) = n_heap; HEAP_TOP(p) = n_htop; HEAP_SIZE(p) = new_sz; HEAP_END(p) = n_heap + new_sz; - -#ifdef HARDDEBUG - disallow_heap_frag_ref_in_heap(p); -#endif - remove_message_buffers(p); } /* @@ -1598,13 +1590,12 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, } #endif - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, - (p->abandoned_heap - ? p->abandoned_heap - : HEAP_START(p)), - p->heap_sz * sizeof(Eterm)); - p->abandoned_heap = NULL; - p->flags &= ~F_ABANDONED_HEAP_USE; +#ifdef HARDDEBUG + disallow_heap_frag_ref_in_heap(p, n_heap, n_htop); +#endif + + deallocate_previous_young_generation(p); + HEAP_START(p) = n_heap; HEAP_TOP(p) = n_htop; HEAP_SIZE(p) = new_sz; @@ -1613,11 +1604,6 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, HIGH_WATER(p) = HEAP_TOP(p); -#ifdef HARDDEBUG - disallow_heap_frag_ref_in_heap(p); -#endif - remove_message_buffers(p); - if (p->flags & F_ON_HEAP_MSGQ) move_msgq_to_heap(p); @@ -1770,22 +1756,56 @@ adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj) return adjusted; } -/* - * Remove all message buffers. - */ static void -remove_message_buffers(Process* p) +deallocate_previous_young_generation(Process *c_p) { - if (MBUF(p) != NULL) { - free_message_buffer(MBUF(p)); - MBUF(p) = NULL; + Eterm *orig_heap; + + if (!c_p->abandoned_heap) { + orig_heap = c_p->heap; + ASSERT(!(c_p->flags & F_ABANDONED_HEAP_USE)); + } + else { + ErlHeapFragment *hfrag; + + orig_heap = c_p->abandoned_heap; + c_p->abandoned_heap = NULL; + c_p->flags &= ~F_ABANDONED_HEAP_USE; + + /* + * Temporary heap located in heap fragment + * only referred to by 'c_p->heap'. Add it to + * 'c_p->mbuf' list and deallocate it as any + * other heap fragment... + */ + hfrag = ((ErlHeapFragment *) + (((char *) c_p->heap) + - offsetof(ErlHeapFragment, mem))); + + ASSERT(!hfrag->off_heap.first); + ASSERT(!hfrag->off_heap.overhead); + ASSERT(!hfrag->next); + ASSERT(c_p->htop - c_p->heap <= hfrag->alloc_size); + + hfrag->next = c_p->mbuf; + c_p->mbuf = hfrag; + } + + if (c_p->mbuf) { + free_message_buffer(c_p->mbuf); + c_p->mbuf = NULL; } - if (p->msg_frag) { - erts_cleanup_messages(p->msg_frag); - p->msg_frag = NULL; + if (c_p->msg_frag) { + erts_cleanup_messages(c_p->msg_frag); + c_p->msg_frag = NULL; } - MBUF_SIZE(p) = 0; + c_p->mbuf_sz = 0; + + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, + orig_heap, + c_p->heap_sz * sizeof(Eterm)); } + #ifdef HARDDEBUG /* @@ -1797,19 +1817,15 @@ remove_message_buffers(Process* p) */ static void -disallow_heap_frag_ref_in_heap(Process* p) +disallow_heap_frag_ref_in_heap(Process *p, Eterm *heap, Eterm *htop) { Eterm* hp; - Eterm* htop; - Eterm* heap; Uint heap_size; if (p->mbuf == 0) { return; } - htop = p->htop; - heap = p->heap; heap_size = (htop - heap)*sizeof(Eterm); hp = heap; @@ -3297,13 +3313,15 @@ within2(Eterm *ptr, Process *p, Eterm *real_htop) ErtsMessage* mp; Eterm *htop, *heap; - if (p->abandoned_heap) + if (p->abandoned_heap) { ERTS_GET_ORIG_HEAP(p, heap, htop); - else { - heap = p->heap; - htop = real_htop ? real_htop : HEAP_TOP(p); + if (heap <= ptr && ptr < htop) + return 1; } + heap = p->heap; + htop = real_htop ? real_htop : HEAP_TOP(p); + if (OLD_HEAP(p) && (OLD_HEAP(p) <= ptr && ptr < OLD_HEND(p))) { return 1; } diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index e4c696ae3b..f45e6974cd 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -193,7 +193,7 @@ free_message_buffer(ErlHeapFragment* bp) erts_cleanup_offheap(&bp->off_heap); ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp, - ERTS_HEAP_FRAG_SIZE(bp->size)); + ERTS_HEAP_FRAG_SIZE(bp->alloc_size)); bp = next_bp; }while (bp != NULL); } diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index d8bb00e8c6..a19db74763 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -90,9 +90,12 @@ Uint erts_process_memory(Process *p, int incl_msg_inq) { erts_doforall_links(ERTS_P_LINKS(p), &erts_one_link_size, &size); erts_doforall_monitors(ERTS_P_MONITORS(p), &erts_one_mon_size, &size); size += (p->heap_sz + p->mbuf_sz) * sizeof(Eterm); + if (p->abandoned_heap) + size += (p->hend - p->heap) * sizeof(Eterm); if (p->old_hend && p->old_heap) size += (p->old_hend - p->old_heap) * sizeof(Eterm); + size += p->msg.len * sizeof(ErtsMessage); for (mp = p->msg.first; mp; mp = mp->next) -- cgit v1.2.3 From a0d79ee6eeca9d99fbb644717ee2df57b824a527 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 10 Jan 2017 20:40:56 +0100 Subject: Fix call_time trace for NIFs --- erts/emulator/beam/beam_emu.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 59a9ea1417..6ac156a15d 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -3550,18 +3550,11 @@ do { \ BifFunction vbf; ErlHeapFragment *live_hf_end; - if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) { - /* If we have run out of reductions, we do a context - switch before calling the nif */ - goto context_switch; - } - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); c_p->current = I-3; /* current and vbf set to please handle_error */ - SWAPOUT; - c_p->fcalls = FCALLS - 1; + HEAVY_SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); bif_nif_arity = I[-1]; ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); -- cgit v1.2.3 From c057b468ee7535f199aa01301ff93e059fad4b07 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 11 Jan 2017 18:18:11 +0100 Subject: Fix call time tracing with dirty schedulers --- erts/emulator/beam/beam_bp.c | 60 +++++++++++++++++++++++++++++++++---- erts/emulator/beam/beam_bp.h | 6 ++-- erts/emulator/beam/erl_bif_trace.c | 7 +++++ erts/emulator/beam/erl_lock_check.c | 1 + 4 files changed, 64 insertions(+), 10 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index bbb2e4f34f..0df2df0eaa 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -74,6 +74,9 @@ extern BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ erts_smp_atomic32_t erts_active_bp_index; erts_smp_atomic32_t erts_staging_bp_index; +#ifdef ERTS_DIRTY_SCHEDULERS +erts_smp_mtx_t erts_dirty_bp_ix_mtx; +#endif /* * Inlined helpers @@ -85,6 +88,31 @@ get_mtime(Process *c_p) return erts_get_monotonic_time(erts_proc_sched_data(c_p)); } +static ERTS_INLINE Uint32 +acquire_bp_sched_ix(Process *c_p) +{ + ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); + ASSERT(esdp); +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { + erts_smp_mtx_lock(&erts_dirty_bp_ix_mtx); + return (Uint32) erts_no_schedulers; + } +#endif + return (Uint32) esdp->no - 1; +} + +static ERTS_INLINE void +release_bp_sched_ix(Uint32 ix) +{ +#ifdef ERTS_DIRTY_SCHEDULERS + if (ix == (Uint32) erts_no_schedulers) + erts_smp_mtx_unlock(&erts_dirty_bp_ix_mtx); +#endif +} + + + /* ************************************************************************* ** Local prototypes */ @@ -135,6 +163,9 @@ void erts_bp_init(void) { erts_smp_atomic32_init_nob(&erts_active_bp_index, 0); erts_smp_atomic32_init_nob(&erts_staging_bp_index, 1); +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_mtx_init(&erts_dirty_bp_ix_mtx, "dirty_break_point_index"); +#endif } @@ -973,6 +1004,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) bp_data_time_item_t sitem, *item = NULL; bp_time_hash_t *h = NULL; BpDataTime *pbdt = NULL; + Uint32 six = acquire_bp_sched_ix(c_p); ASSERT(c_p); ASSERT(erts_smp_atomic32_read_acqb(&c_p->state) & (ERTS_PSFLG_RUNNING @@ -980,7 +1012,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) /* get previous timestamp and breakpoint * from the process psd */ - + pbt = ERTS_PROC_GET_CALL_TIME(c_p); time = get_mtime(c_p); @@ -1006,7 +1038,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) /* if null then the breakpoint was removed */ if (pbdt) { - h = &(pbdt->hash[bp_sched2ix_proc(c_p)]); + h = &(pbdt->hash[six]); ASSERT(h); ASSERT(h->item); @@ -1027,7 +1059,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) /* this breakpoint */ ASSERT(bdt); - h = &(bdt->hash[bp_sched2ix_proc(c_p)]); + h = &(bdt->hash[six]); ASSERT(h); ASSERT(h->item); @@ -1041,6 +1073,8 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) pbt->pc = I; pbt->time = time; + + release_bp_sched_ix(six); } void @@ -1051,6 +1085,7 @@ erts_trace_time_return(Process *p, BeamInstr *pc) bp_data_time_item_t sitem, *item = NULL; bp_time_hash_t *h = NULL; BpDataTime *pbdt = NULL; + Uint32 six = acquire_bp_sched_ix(p); ASSERT(p); ASSERT(erts_smp_atomic32_read_acqb(&p->state) & (ERTS_PSFLG_RUNNING @@ -1071,6 +1106,7 @@ erts_trace_time_return(Process *p, BeamInstr *pc) */ if (pbt) { + /* might have been removed due to * trace_pattern(false) */ @@ -1085,7 +1121,8 @@ erts_trace_time_return(Process *p, BeamInstr *pc) /* beware, the trace_pattern might have been removed */ if (pbdt) { - h = &(pbdt->hash[bp_sched2ix_proc(p)]); + + h = &(pbdt->hash[six]); ASSERT(h); ASSERT(h->item); @@ -1096,11 +1133,15 @@ erts_trace_time_return(Process *p, BeamInstr *pc) } else { BP_TIME_ADD(item, &sitem); } + } pbt->pc = pc; pbt->time = time; + } + + release_bp_sched_ix(six); } int @@ -1353,6 +1394,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) { bp_data_time_item_t sitem, *item = NULL; bp_time_hash_t *h = NULL; BpDataTime *pbdt = NULL; + Uint32 six = acquire_bp_sched_ix(p); ASSERT(p); @@ -1375,7 +1417,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) { sitem.pid = p->common.id; sitem.count = 0; - h = &(pbdt->hash[bp_sched2ix_proc(p)]); + h = &(pbdt->hash[six]); ASSERT(h); ASSERT(h->item); @@ -1401,6 +1443,8 @@ void erts_schedule_time_break(Process *p, Uint schedule) { break; } } /* pbt */ + + release_bp_sched_ix(six); } /* ************************************************************************* @@ -1517,7 +1561,11 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, ASSERT((bp->flags & ERTS_BPF_TIME_TRACE) == 0); bdt = Alloc(sizeof(BpDataTime)); erts_refc_init(&bdt->refc, 1); - bdt->n = erts_no_total_schedulers; +#ifdef ERTS_DIRTY_SCHEDULERS + bdt->n = erts_no_schedulers + 1; +#else + bdt->n = erts_no_schedulers; +#endif bdt->hash = Alloc(sizeof(bp_time_hash_t)*(bdt->n)); for (i = 0; i < bdt->n; i++) { bp_hash_init(&(bdt->hash[i]), 32); diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 7206ef471a..4743e4fc2f 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -79,10 +79,8 @@ typedef struct generic_bp { #define ERTS_BP_CALL_TIME_SCHEDULE_OUT (1) #define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2) -#ifdef ERTS_SMP -#define bp_sched2ix_proc(p) (erts_proc_sched_data(p)->thr_id - 1) -#else -#define bp_sched2ix_proc(p) (0) +#ifdef ERTS_DIRTY_SCHEDULERS +extern erts_smp_mtx_t erts_dirty_bp_ix_mtx; #endif enum erts_break_op{ diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 96275eb228..0627526d7e 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -1062,9 +1062,16 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) erts_smp_thr_progress_block(); } #endif +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_mtx_lock(&erts_dirty_bp_ix_mtx); +#endif + r = function_is_traced(p, mfa, &ms, &ms_meta, &meta, &count, &call_time); +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_mtx_unlock(&erts_dirty_bp_ix_mtx); +#endif #ifdef ERTS_SMP if ( (key == am_call_time) || (key == am_all)) { erts_smp_thr_progress_unblock(); diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 13a4b2cd93..5d4823c077 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -130,6 +130,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { #ifdef ERTS_DIRTY_SCHEDULERS { "dirty_run_queue_sleep_list", "address" }, { "dirty_gc_info", NULL }, + { "dirty_break_point_index", NULL }, #endif { "process_table", NULL }, { "cpu_info", NULL }, -- cgit v1.2.3 From af5169d85fcd545e3c857a219db081a62f33404d Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 12 Jan 2017 13:58:26 +0100 Subject: erts: Fix race bug between export fun creation and code loading Symptom: SEGV crash on ARM in delete_code() -> export_list(). Could probably happen on other machines as well. Problem: Staging export table was iterated in an unsafe way while an entry was added for a new export fun. Solution: Correct write order and some memory barriers. --- erts/emulator/beam/beam_bif_load.c | 14 ++++++++------ erts/emulator/beam/beam_load.c | 11 ++++++----- erts/emulator/beam/export.c | 2 +- erts/emulator/beam/index.c | 9 ++++++++- erts/emulator/beam/index.h | 14 ++++++++++++++ 5 files changed, 37 insertions(+), 13 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 5969197168..93f5ed500b 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -786,7 +786,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) } if (BIF_ARG_2 == am_true) { - int i; + int i, num_exps; /* * Make the code with the on_load function current. @@ -802,7 +802,8 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) /* * The on_load function succeded. Fix up export entries. */ - for (i = 0; i < export_list_size(code_ix); i++) { + num_exps = export_list_size(code_ix); + for (i = 0; i < num_exps; i++) { Export *ep = export_list(i,code_ix); if (ep == NULL || ep->code[0] != BIF_ARG_1) { continue; @@ -822,14 +823,15 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) modp->curr.code_hdr->on_load_function_ptr = NULL; set_default_trace_pattern(BIF_ARG_1); } else if (BIF_ARG_2 == am_false) { - int i; + int i, num_exps; /* * The on_load function failed. Remove references to the * code that is about to be purged from the export entries. */ - for (i = 0; i < export_list_size(code_ix); i++) { + num_exps = export_list_size(code_ix); + for (i = 0; i < num_exps; i++) { Export *ep = export_list(i,code_ix); if (ep == NULL || ep->code[0] != BIF_ARG_1) { continue; @@ -2011,9 +2013,9 @@ delete_code(Module* modp) { ErtsCodeIndex code_ix = erts_staging_code_ix(); Eterm module = make_atom(modp->module); - int i; + int i, num_exps = export_list_size(code_ix); - for (i = 0; i < export_list_size(code_ix); i++) { + for (i = 0; i < num_exps; i++) { Export *ep = export_list(i, code_ix); if (ep != NULL && (ep->code[0] == module)) { if (ep->addressv[code_ix] == ep->code+3) { diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 0afdedf6c2..3f2bdf3f9d 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -797,14 +797,14 @@ erts_finish_loading(Binary* magic, Process* c_p, } else { ErtsCodeIndex code_ix = erts_staging_code_ix(); Eterm module = stp->module; - int i; + int i, num_exps; /* * There is an -on_load() function. We will keep the current * code, but we must turn off any tracing. */ - - for (i = 0; i < export_list_size(code_ix); i++) { + num_exps = export_list_size(code_ix); + for (i = 0; i < num_exps; i++) { Export *ep = export_list(i, code_ix); if (ep == NULL || ep->code[0] != module) { continue; @@ -5754,12 +5754,13 @@ exported_from_module(Process* p, /* Process whose heap to use. */ ErtsCodeIndex code_ix, Eterm mod) /* Tagged atom for module. */ { - int i; + int i, num_exps; Eterm* hp = NULL; Eterm* hend = NULL; Eterm result = NIL; - for (i = 0; i < export_list_size(code_ix); i++) { + num_exps = export_list_size(code_ix); + for (i = 0; i < num_exps; i++) { Export* ep = export_list(i,code_ix); if (ep->code[0] == mod) { diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 2a19211987..2a007ce860 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -348,7 +348,7 @@ Export *export_list(int i, ErtsCodeIndex code_ix) int export_list_size(ErtsCodeIndex code_ix) { - return export_tables[code_ix].entries; + return erts_index_num_entries(&export_tables[code_ix]); } int export_table_sz(void) diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c index 26d6c04ea0..4cebb4d8fa 100644 --- a/erts/emulator/beam/index.c +++ b/erts/emulator/beam/index.c @@ -91,9 +91,16 @@ index_put_entry(IndexTable* t, void* tmpl) t->seg_table[ix>>INDEX_PAGE_SHIFT] = erts_alloc(t->type, sz); t->size += INDEX_PAGE_SIZE; } - t->entries++; p->index = ix; t->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK] = p; + + /* + * Do a write barrier here to allow readers to do lock free iteration. + * erts_index_num_entries() does matching read barrier. + */ + ERTS_SMP_WRITE_MEMORY_BARRIER; + t->entries++; + return p; } diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h index 0a109d8699..532dec0168 100644 --- a/erts/emulator/beam/index.h +++ b/erts/emulator/beam/index.h @@ -65,6 +65,7 @@ void index_erase_latest_from(IndexTable*, Uint ix); ERTS_GLB_INLINE int index_put(IndexTable*, void*); ERTS_GLB_INLINE IndexSlot* erts_index_lookup(IndexTable*, Uint); +ERTS_GLB_INLINE int erts_index_num_entries(IndexTable* t); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -78,6 +79,19 @@ erts_index_lookup(IndexTable* t, Uint ix) { return t->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK]; } + +ERTS_GLB_INLINE int erts_index_num_entries(IndexTable* t) +{ + int ret = t->entries; + /* + * Do a read barrier here to allow lock free iteration + * on tables where entries are never erased. + * index_put_entry() does matching write barrier. + */ + ERTS_SMP_READ_MEMORY_BARRIER; + return ret; +} + #endif #endif -- cgit v1.2.3 From 6bcdd45abd97134fddfb5b0307b1d256337b0c67 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 28 Dec 2016 18:23:06 +0100 Subject: Reduction counting on non-tail return --- erts/emulator/beam/beam_emu.c | 21 +++++++++++++++++---- erts/emulator/beam/erl_vm.h | 2 +- 2 files changed, 18 insertions(+), 5 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index c554bd73b6..bb77dbd955 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -633,21 +633,34 @@ void** beam_ops; y[4] = xt4; \ } while (0) +#define DispatchReturn \ +do { \ + if (FCALLS > 0 || FCALLS > neg_o_reds) { \ + FCALLS--; \ + Goto(*I); \ + } \ + else { \ + c_p->current = NULL; \ + c_p->arity = 1; \ + goto context_switch3; \ + } \ +} while (0) + #define MoveReturn(Src) \ x(0) = (Src); \ I = c_p->cp; \ ASSERT(VALID_INSTR(*c_p->cp)); \ c_p->cp = 0; \ CHECK_TERM(r(0)); \ - Goto(*I) + DispatchReturn #define DeallocateReturn(Deallocate) \ do { \ int words_to_pop = (Deallocate); \ - SET_I((BeamInstr *) cp_val(*E)); \ + SET_I((BeamInstr *) cp_val(*E)); \ E = ADD_BYTE_OFFSET(E, words_to_pop); \ CHECK_TERM(r(0)); \ - Goto(*I); \ + DispatchReturn; \ } while (0) #define MoveDeallocateReturn(Src, Deallocate) \ @@ -1681,7 +1694,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) c_p->cp = 0; CHECK_TERM(r(0)); HEAP_SPACE_VERIFIED(0); - Goto(*I); + DispatchReturn; } /* diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index 93cfe08105..d88cafc5bf 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -36,7 +36,7 @@ #define EMULATOR "BEAM" #define SEQ_TRACE 1 -#define CONTEXT_REDS 2000 /* Swap process out after this number */ +#define CONTEXT_REDS 4000 /* Swap process out after this number */ #define MAX_ARG 255 /* Max number of arguments allowed */ #define MAX_REG 1024 /* Max number of x(N) registers used */ -- cgit v1.2.3 From 5aff60d96efac96a41b514ed167f13eb787a415f Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 26 Sep 2016 17:05:50 +0200 Subject: Support for dirty BIFs --- erts/emulator/beam/atom.names | 9 + erts/emulator/beam/beam_bif_load.c | 6 + erts/emulator/beam/beam_debug.c | 347 +++++++++++++++++ erts/emulator/beam/beam_emu.c | 117 +++--- erts/emulator/beam/bif.c | 317 +++++++++++++++- erts/emulator/beam/bif.h | 87 ++++- erts/emulator/beam/bif.tab | 3 + erts/emulator/beam/erl_bif_info.c | 3 + erts/emulator/beam/erl_dirty_bif.tab | 82 ++++ erts/emulator/beam/erl_gc.c | 24 +- erts/emulator/beam/erl_msacc.c | 4 +- erts/emulator/beam/erl_msacc.h | 2 +- erts/emulator/beam/erl_nfunc_sched.c | 144 +++++++ erts/emulator/beam/erl_nfunc_sched.h | 289 ++++++++++++++ erts/emulator/beam/erl_nif.c | 707 +++++++++++++---------------------- erts/emulator/beam/erl_process.c | 18 +- erts/emulator/beam/erl_process.h | 8 +- erts/emulator/beam/error.h | 88 +++-- erts/emulator/beam/global.h | 14 +- erts/emulator/beam/utils.c | 2 + 20 files changed, 1663 insertions(+), 608 deletions(-) create mode 100644 erts/emulator/beam/erl_dirty_bif.tab create mode 100644 erts/emulator/beam/erl_nfunc_sched.c create mode 100644 erts/emulator/beam/erl_nfunc_sched.h (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index b1aeed7889..7df350116a 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -76,6 +76,7 @@ atom ac atom accessor atom active atom active_tasks +atom alive atom all atom all_but_first atom all_names @@ -199,10 +200,15 @@ atom dexit atom depth atom dgroup_leader atom dictionary +atom dirty_bif_exception +atom dirty_bif_result +atom dirty_bif_trap atom dirty_cpu atom dirty_cpu_schedulers_online atom dirty_execution atom dirty_io +atom dirty_nif_exception +atom dirty_nif_finalizer atom disable_trace atom disabled atom discard @@ -243,6 +249,7 @@ atom ERROR='ERROR' atom error_handler atom error_logger atom erts_code_purger +atom erts_debug atom erts_internal atom ets atom ETS_TRANSFER='ETS-TRANSFER' @@ -573,6 +580,7 @@ atom scientific atom scope atom second atom seconds +atom send atom send_to_non_existing_process atom sensitive atom sequential_tracer @@ -665,6 +673,7 @@ atom value atom values atom version atom visible +atom wait atom waiting atom wall_clock atom warning diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 09331eb8ad..36bbe81ed5 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -36,6 +36,7 @@ #include "erl_nif.h" #include "erl_bits.h" #include "erl_thr_progress.h" +#include "erl_nfunc_sched.h" #ifdef HIPE # include "hipe_bif0.h" # define IF_HIPE(X) (X) @@ -1102,6 +1103,11 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) *redsp += 1; + if (erts_check_nif_export_in_area(rp, mod_start, mod_size)) + return am_true; + + *redsp += 1; + if (erts_check_nif_export_in_area(rp, mod_start, mod_size)) return am_true; diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index e72d7f8de4..3ffbb0364c 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -39,6 +39,7 @@ #include "beam_bp.h" #include "erl_binary.h" #include "erl_thr_progress.h" +#include "erl_nfunc_sched.h" #ifdef ARCH_64 # define HEXF "%016bpX" @@ -764,3 +765,349 @@ static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif) erts_print(to, to_arg, "%T/%u", name, arity); } } + +/* + * Dirty BIF testing. + * + * The erts_debug:dirty_cpu/2, erts_debug:dirty_io/1, and + * erts_debug:dirty/3 BIFs are used by the dirty_bif_SUITE + * test suite. + */ + +#ifdef ERTS_DIRTY_SCHEDULERS +static int ms_wait(Process *c_p, Eterm etimeout, int busy); +static int dirty_send_message(Process *c_p, Eterm to, Eterm tag); +#endif +static BIF_RETTYPE dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I); + +/* + * erts_debug:dirty_cpu/2 is statically determined to execute on + * a dirty CPU scheduler (see erts_dirty_bif.tab). + */ +BIF_RETTYPE +erts_debug_dirty_cpu_2(BIF_ALIST_2) +{ + return dirty_test(BIF_P, am_dirty_cpu, BIF_ARG_1, BIF_ARG_2, BIF_I); +} + +/* + * erts_debug:dirty_io/2 is statically determined to execute on + * a dirty I/O scheduler (see erts_dirty_bif.tab). + */ +BIF_RETTYPE +erts_debug_dirty_io_2(BIF_ALIST_2) +{ + return dirty_test(BIF_P, am_dirty_io, BIF_ARG_1, BIF_ARG_2, BIF_I); +} + +/* + * erts_debug:dirty/3 executes on a normal scheduler. + */ +BIF_RETTYPE +erts_debug_dirty_3(BIF_ALIST_3) +{ +#ifdef ERTS_DIRTY_SCHEDULERS + Eterm argv[2]; + switch (BIF_ARG_1) { + case am_normal: + return dirty_test(BIF_P, am_normal, BIF_ARG_2, BIF_ARG_3, BIF_I); + case am_dirty_cpu: + argv[0] = BIF_ARG_2; + argv[1] = BIF_ARG_3; + return erts_schedule_bif(BIF_P, + argv, + BIF_I, + erts_debug_dirty_cpu_2, + ERTS_SCHED_DIRTY_CPU, + am_erts_debug, + am_dirty_cpu, + 2); + case am_dirty_io: + argv[0] = BIF_ARG_2; + argv[1] = BIF_ARG_3; + return erts_schedule_bif(BIF_P, + argv, + BIF_I, + erts_debug_dirty_io_2, + ERTS_SCHED_DIRTY_IO, + am_erts_debug, + am_dirty_io, + 2); + default: + BIF_ERROR(BIF_P, EXC_BADARG); + } +#else + BIF_ERROR(BIF_P, EXC_UNDEF); +#endif +} + + +static BIF_RETTYPE +dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I) +{ + BIF_RETTYPE ret; +#ifdef ERTS_DIRTY_SCHEDULERS + if (am_scheduler == arg1) { + ErtsSchedulerData *esdp; + if (arg2 != am_type) + goto badarg; + esdp = erts_proc_sched_data(c_p); + if (!esdp) + ERTS_BIF_PREP_RET(ret, am_error); + else if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) + ERTS_BIF_PREP_RET(ret, am_normal); + else if (ERTS_SCHEDULER_IS_DIRTY_CPU(esdp)) + ERTS_BIF_PREP_RET(ret, am_dirty_cpu); + else if (ERTS_SCHEDULER_IS_DIRTY_IO(esdp)) + ERTS_BIF_PREP_RET(ret, am_dirty_io); + else + ERTS_BIF_PREP_RET(ret, am_error); + } + else if (am_error == arg1) { + switch (arg2) { + case am_notsup: + ERTS_BIF_PREP_ERROR(ret, c_p, EXC_NOTSUP); + break; + case am_undef: + ERTS_BIF_PREP_ERROR(ret, c_p, EXC_UNDEF); + break; + case am_badarith: + ERTS_BIF_PREP_ERROR(ret, c_p, EXC_BADARITH); + break; + case am_noproc: + ERTS_BIF_PREP_ERROR(ret, c_p, EXC_NOPROC); + break; + case am_system_limit: + ERTS_BIF_PREP_ERROR(ret, c_p, SYSTEM_LIMIT); + break; + case am_badarg: + default: + goto badarg; + } + } + else if (am_copy == arg1) { + int i; + Eterm res; + + for (res = NIL, i = 0; i < 1000; i++) { + Eterm *hp, sz; + Eterm cpy; + /* We do not want this to be optimized, + but rather the oposite... */ + sz = size_object(arg2); + hp = HAlloc(c_p, sz); + cpy = copy_struct(arg2, sz, &hp, &c_p->off_heap); + hp = HAlloc(c_p, 2); + res = CONS(hp, cpy, res); + } + + ERTS_BIF_PREP_RET(ret, res); + } + else if (am_send == arg1) { + dirty_send_message(c_p, arg2, am_ok); + ERTS_BIF_PREP_RET(ret, am_ok); + } + else if (ERTS_IS_ATOM_STR("wait", arg1)) { + if (!ms_wait(c_p, arg2, type == am_dirty_cpu)) + goto badarg; + ERTS_BIF_PREP_RET(ret, am_ok); + } + else if (ERTS_IS_ATOM_STR("reschedule", arg1)) { + /* + * Reschedule operation after decrement of two until we reach + * zero. Switch between dirty scheduler types when 'n' is + * evenly divided by 4. If the initial value wasn't evenly + * dividable by 2, throw badarg exception. + */ + Eterm next_type; + Sint n; + if (!term_to_Sint(arg2, &n) || n < 0) + goto badarg; + if (n == 0) + ERTS_BIF_PREP_RET(ret, am_ok); + else { + Eterm argv[3]; + Eterm eint = erts_make_integer((Uint) (n - 2), c_p); + if (n % 4 != 0) + next_type = type; + else { + switch (type) { + case am_dirty_cpu: next_type = am_dirty_io; break; + case am_dirty_io: next_type = am_normal; break; + case am_normal: next_type = am_dirty_cpu; break; + default: goto badarg; + } + } + switch (next_type) { + case am_dirty_io: + argv[0] = arg1; + argv[1] = eint; + ret = erts_schedule_bif(c_p, + argv, + I, + erts_debug_dirty_io_2, + ERTS_SCHED_DIRTY_IO, + am_erts_debug, + am_dirty_io, + 2); + break; + case am_dirty_cpu: + argv[0] = arg1; + argv[1] = eint; + ret = erts_schedule_bif(c_p, + argv, + I, + erts_debug_dirty_cpu_2, + ERTS_SCHED_DIRTY_CPU, + am_erts_debug, + am_dirty_cpu, + 2); + break; + case am_normal: + argv[0] = am_normal; + argv[1] = arg1; + argv[2] = eint; + ret = erts_schedule_bif(c_p, + argv, + I, + erts_debug_dirty_3, + ERTS_SCHED_NORMAL, + am_erts_debug, + am_dirty, + 3); + break; + default: + goto badarg; + } + } + } + else if (ERTS_IS_ATOM_STR("ready_wait6_done", arg1)) { + ERTS_DECL_AM(ready); + ERTS_DECL_AM(done); + dirty_send_message(c_p, arg2, AM_ready); + ms_wait(c_p, make_small(6000), 0); + dirty_send_message(c_p, arg2, AM_done); + ERTS_BIF_PREP_RET(ret, am_ok); + } + else if (ERTS_IS_ATOM_STR("alive_waitexiting", arg1)) { + Process *real_c_p = erts_proc_shadow2real(c_p); + Eterm *hp, *hp2; + Uint sz; + int i; + if (ERTS_PROC_IS_EXITING(real_c_p)) + goto badarg; + dirty_send_message(c_p, arg2, am_alive); + /* Wait until dead */ + + while (!ERTS_PROC_IS_EXITING(real_c_p)) + erts_thr_yield(); + + ms_wait(c_p, make_small(1000), 0); + + /* Should still be able to allocate memory */ + hp = HAlloc(c_p, 3); /* Likely on heap */ + sz = 10000; + hp2 = HAlloc(c_p, sz); /* Likely in heap fragment */ + *hp2 = make_pos_bignum_header(sz); + for (i = 1; i < sz; i++) + hp2[i] = (Eterm) 4711; + ERTS_BIF_PREP_RET(ret, TUPLE2(hp, am_ok, make_big(hp2))); + } + else { + badarg: + ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); + } +#else + ERTS_BIF_PREP_ERROR(ret, c_p, EXC_UNDEF); +#endif + return ret; +} + +#ifdef ERTS_DIRTY_SCHEDULERS + +static int +dirty_send_message(Process *c_p, Eterm to, Eterm tag) +{ + ErtsProcLocks c_p_locks, rp_locks; + Process *rp, *real_c_p; + Eterm msg, *hp; + ErlOffHeap *ohp; + ErtsMessage *mp; + + ASSERT(is_immed(tag)); + + real_c_p = erts_proc_shadow2real(c_p); + if (real_c_p != c_p) + c_p_locks = 0; + else + c_p_locks = ERTS_PROC_LOCK_MAIN; + + ASSERT(real_c_p->common.id == c_p->common.id); + + rp = erts_pid2proc_opt(real_c_p, c_p_locks, + to, 0, + ERTS_P2P_FLG_INC_REFC); + + if (!rp) + return 0; + + rp_locks = 0; + mp = erts_alloc_message_heap(rp, &rp_locks, 3, &hp, &ohp); + + msg = TUPLE2(hp, tag, c_p->common.id); + erts_queue_message(rp, rp_locks, mp, msg, c_p->common.id); + + if (rp == real_c_p) + rp_locks &= ~c_p_locks; + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + + erts_proc_dec_refc(rp); + + return 1; +} + +static int +ms_wait(Process *c_p, Eterm etimeout, int busy) +{ + ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); + ErtsMonotonicTime time, timeout_time; + Sint64 ms; + + if (!term_to_Sint64(etimeout, &ms)) + return 0; + + time = erts_get_monotonic_time(esdp); + + if (ms < 0) + timeout_time = time; + else + timeout_time = time + ERTS_MSEC_TO_MONOTONIC(ms); + + while (time < timeout_time) { + if (busy) + erts_thr_yield(); + else { + ErtsMonotonicTime timeout = timeout_time - time; + +#ifdef __WIN32__ + Sleep((DWORD) ERTS_MONOTONIC_TO_MSEC(timeout)); +#else + { + ErtsMonotonicTime to = ERTS_MONOTONIC_TO_USEC(timeout); + struct timeval tv; + + tv.tv_sec = (long) to / (1000*1000); + tv.tv_usec = (long) to % (1000*1000); + + select(0, NULL, NULL, NULL, &tv); + } +#endif + } + + time = erts_get_monotonic_time(esdp); + } + return 1; +} + +#endif /* ERTS_DIRTY_SCHEDULERS */ diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index bb77dbd955..befd2989f9 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -38,6 +38,7 @@ #include "beam_bp.h" #include "beam_catches.h" #include "erl_thr_progress.h" +#include "erl_nfunc_sched.h" #ifdef HIPE #include "hipe_mode_switch.h" #include "hipe_bif1.h" @@ -214,6 +215,7 @@ BeamInstr beam_continue_exit[1]; BeamInstr* em_call_error_handler; BeamInstr* em_apply_bif; BeamInstr* em_call_nif; +BeamInstr* em_call_bif_e; /* NOTE These should be the only variables containing trace instructions. @@ -2581,7 +2583,7 @@ do { \ OpCase(bif1_fbsd): { - Eterm (*bf)(Process*, Eterm*); + ErtsBifFunc bf; Eterm tmp_reg[1]; Eterm result; @@ -2592,7 +2594,7 @@ do { \ PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, tmp_reg); + result = (*bf)(c_p, tmp_reg, I); ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -2613,19 +2615,19 @@ do { \ OpCase(bif1_body_bsd): { - Eterm (*bf)(Process*, Eterm*); + ErtsBifFunc bf; Eterm tmp_reg[1]; Eterm result; GetArg1(1, tmp_reg[0]); - bf = (BifFunction) Arg(0); + bf = (ErtsBifFunc) Arg(0); ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, tmp_reg); + result = (*bf)(c_p, tmp_reg, I); ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -2775,17 +2777,17 @@ do { \ OpCase(i_bif2_fbssd): { Eterm tmp_reg[2]; - Eterm (*bf)(Process*, Eterm*); + ErtsBifFunc bf; Eterm result; GetArg2(2, tmp_reg[0], tmp_reg[1]); - bf = (BifFunction) Arg(1); + bf = (ErtsBifFunc) Arg(1); ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, tmp_reg); + result = (*bf)(c_p, tmp_reg, I); ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -2806,15 +2808,15 @@ do { \ OpCase(i_bif2_body_bssd): { Eterm tmp_reg[2]; - Eterm (*bf)(Process*, Eterm*); + ErtsBifFunc bf; Eterm result; GetArg2(1, tmp_reg[0], tmp_reg[1]); - bf = (BifFunction) Arg(0); + bf = (ErtsBifFunc) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, tmp_reg); + result = (*bf)(c_p, tmp_reg, I); ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -2837,7 +2839,7 @@ do { \ */ OpCase(call_bif_e): { - Eterm (*bf)(Process*, Eterm*, BeamInstr*); + ErtsBifFunc bf; Eterm result; BeamInstr *next; ErlHeapFragment *live_hf_end; @@ -3644,7 +3646,7 @@ do { \ ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); { - Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf; + ErtsBifFunc bf = vbf; ASSERT(!ERTS_PROC_IS_EXITING(c_p)); live_hf_end = c_p->mbuf; ERTS_CHK_MBUF_SZ(c_p); @@ -5137,6 +5139,7 @@ do { \ em_call_error_handler = OpCode(call_error_handler); em_apply_bif = OpCode(apply_bif); em_call_nif = OpCode(call_nif); + em_call_bif_e = OpCode(call_bif_e); beam_apply[0] = (BeamInstr) OpCode(i_apply); beam_apply[1] = (BeamInstr) OpCode(normal_exit); @@ -5342,8 +5345,6 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) I = c_p->i; - ASSERT(em_call_nif == (BeamInstr *) *I); - /* * Set fcalls even though we ignore it, so we don't * confuse code accessing it... @@ -5379,10 +5380,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) } { -#ifdef DEBUG - Eterm result; -#endif - Eterm arity; + int exiting; { /* @@ -5398,7 +5396,6 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) * * This layout is determined by the NifExport struct */ - BifFunction vbf; ErtsCodeMFA *codemfa; ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); @@ -5406,44 +5403,29 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) codemfa = erts_code_to_codemfa(I); DTRACE_NIF_ENTRY(c_p, codemfa); - /* current and vbf set to please handle_error */ c_p->current = codemfa; SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); - arity = codemfa->arity; ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - { - typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); - NifF* fp = vbf = (NifF*) I[1]; - struct enif_environment_t env; - ASSERT(!c_p->scheduler_data); - - erts_pre_dirty_nif(esdp, &env, c_p, - (struct erl_module_nif*)I[2]); - -#ifdef DEBUG - result = -#else - (void) -#endif - (*fp)(&env, arity, reg); - - erts_post_dirty_nif(&env); + if (em_apply_bif == (BeamInstr *) *I) { + exiting = erts_call_dirty_bif(esdp, c_p, I, reg); + } + else { + ASSERT(em_call_nif == (BeamInstr *) *I); + exiting = erts_call_dirty_nif(esdp, c_p, I, reg); + } - ASSERT(!is_value(result)); - ASSERT(c_p->freason == TRAP); - ASSERT(!(c_p->flags & F_HIBERNATE_SCHED)); + ASSERT(!(c_p->flags & F_HIBERNATE_SCHED)); - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); - if (env.exiting) - goto do_dirty_schedule; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - } + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); + if (exiting) + goto do_dirty_schedule; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); DTRACE_NIF_RETURN(c_p, codemfa); ERTS_HOLE_CHECK(c_p); @@ -5529,10 +5511,15 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) Eterm* hp; Eterm Value = c_p->fvalue; Eterm Args = am_true; - c_p->i = pc; /* In case we call erts_exit(). */ ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */ + if (c_p->freason & EXF_RESTORE_NIF) { + erts_nif_export_restore_error(c_p, &pc, reg, &bf); + } + + c_p->i = pc; /* In case we call erts_exit(). */ + /* * Check if we have an arglist for the top level call. If so, this * is encoded in Value, so we have to dig out the real Value as well @@ -5829,6 +5816,18 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) { * error terms.) */ +static void +save_stacktrace_current(struct StackTrace *s, Process *c_p, ErtsCodeMFA *mfa) +{ + NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + if (!nep || &nep->exp.info.mfa != mfa) + s->current = mfa; + else { + s->mfa = *mfa; + s->current = &s->mfa; + } +} + static void save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, Eterm args) { @@ -5858,12 +5857,10 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, && bf != throw_1 && bf != wrap_error_1 && bf != wrap_error_2 && bf != wrap_exit_1 && bf != wrap_throw_1) { int i; - int a = 0; + int a; for (i = 0; i < BIF_SIZE; i++) { if (bf == bif_table[i].f || bf == bif_table[i].traced) { - Export *ep = bif_export[i]; - s->current = &ep->info.mfa; - a = bif_table[i].arity; + save_stacktrace_current(s, c_p, &bif_export[i]->info.mfa); break; } } @@ -5875,9 +5872,11 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, * OR it is a NIF called by call_nif where current is also set. */ ASSERT(c_p->current); - s->current = c_p->current; - a = s->current->arity; + save_stacktrace_current(s, c_p, c_p->current); } + + a = s->current->arity; + /* Save first stack entry */ ASSERT(pc); if (depth > 0) { @@ -5892,7 +5891,9 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, s->pc = NULL; args = make_arglist(c_p, reg, a); /* Overwrite CAR(c_p->ftrace) */ } else { - s->current = c_p->current; + + save_stacktrace_current(s, c_p, c_p->current); + /* * For a function_clause error, the arguments are in the beam * registers, c_p->cp is valid, and c_p->current is set. diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index d886c2985e..7bcb7c196d 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -4958,7 +4958,7 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, Export bif_return_trap_export; void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, - Eterm (*bif)(BIF_ALIST_0)) + Eterm (*bif)(BIF_ALIST)) { int i; sys_memset((void *) ep, 0, sizeof(Export)); @@ -5019,6 +5019,321 @@ void erts_init_bif(void) erts_smp_atomic32_init_nob(&msacc, ERTS_MSACC_IS_ENABLED()); } +/* + * Scheduling of BIFs via NifExport... + */ +#define ERTS_WANT_NFUNC_SCHED_INTERNALS__ +#include "erl_nfunc_sched.h" + +#define ERTS_SCHED_BIF_TRAP_MARKER ((void *) (UWord) 1) + +static void +schedule(Process *c_p, Process *dirty_shadow_proc, + ErtsCodeMFA *mfa, BifFunction *nif, BeamInstr *pc, + ErtsBifFunc dfunc, void *ifunc, + Eterm module, Eterm function, + int argc, Eterm *argv) +{ + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); + (void) erts_nif_export_schedule(c_p, dirty_shadow_proc, + mfa, nif, pc, + (BeamInstr) em_apply_bif, + dfunc, ifunc, + module, function, + argc, argv); +} + +#ifdef ERTS_DIRTY_SCHEDULERS + +static BIF_RETTYPE dirty_bif_result(BIF_ALIST_1) +{ + NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(BIF_P); + erts_nif_export_restore(BIF_P, nep); + BIF_RET(BIF_ARG_1); +} + +static BIF_RETTYPE dirty_bif_trap(BIF_ALIST) +{ + NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(BIF_P); + + /* + * Arity and argument registers already set + * correct by call to dirty_bif_trap()... + */ + + ASSERT(BIF_P->arity == nep->exp.info.mfa.arity); + + erts_nif_export_restore(BIF_P, nep); + + BIF_P->i = (BeamInstr *) nep->func; + BIF_P->freason = TRAP; + return THE_NON_VALUE; +} + +static BIF_RETTYPE dirty_bif_exception(BIF_ALIST_2) +{ + Eterm freason; + + ASSERT(is_small(BIF_ARG_1)); + + freason = signed_val(BIF_ARG_1); + + /* Restore orig info for error and clear nif export in handle_error() */ + freason |= EXF_RESTORE_NIF; + + BIF_P->fvalue = BIF_ARG_2; + + BIF_ERROR(BIF_P, freason); +} + +#endif /* ERTS_DIRTY_SCHEDULERS */ + +extern BeamInstr* em_call_bif_e; +static BIF_RETTYPE call_bif(Process *c_p, Eterm *reg, BeamInstr *I); + +BIF_RETTYPE +erts_schedule_bif(Process *proc, + Eterm *argv, + BeamInstr *i, + ErtsBifFunc bif, + ErtsSchedType sched_type, + Eterm mod, + Eterm func, + int argc) +{ + Process *c_p, *dirty_shadow_proc; + ErtsCodeMFA *mfa; + +#ifdef ERTS_DIRTY_SCHEDULERS + if (proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { + dirty_shadow_proc = proc; + c_p = proc->next; + ASSERT(c_p->common.id == dirty_shadow_proc->common.id); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + } + else +#endif + { + dirty_shadow_proc = NULL; + c_p = proc; + } + + if (!ERTS_PROC_IS_EXITING(c_p)) { + Export *exp; + BifFunction obif, dbif, ibif; + BeamInstr *pc; + + /* + * obif - original bif + * dbif - direct bif + * ibif - indirect bif + */ + +#ifdef ERTS_DIRTY_SCHEDULERS + erts_aint32_t set, mask; + mask = (ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_IO_PROC); + switch (sched_type) { + case ERTS_SCHED_DIRTY_CPU: + set = ERTS_PSFLG_DIRTY_CPU_PROC; + dbif = bif; + ibif = NULL; + break; + case ERTS_SCHED_DIRTY_IO: + set = ERTS_PSFLG_DIRTY_IO_PROC; + dbif = bif; + ibif = NULL; + break; + case ERTS_SCHED_NORMAL: + default: + set = 0; + dbif = call_bif; + ibif = bif; + break; + } + + (void) erts_smp_atomic32_read_bset_nob(&c_p->state, mask, set); +#else + dbif = call_bif; + ibif = bif; +#endif + + if (i == NULL) { + ERTS_INTERNAL_ERROR("Missing instruction pointer"); + obif = NULL; + } +#ifdef HIPE + else if (proc->flags & F_HIPE_MODE) { + /* Pointer to bif export in i */ + exp = (Export *) i; + pc = c_p->cp; + obif = (BifFunction) exp->beam[1]; + mfa = &exp->info.mfa; + } +#endif + else if (em_call_bif_e == (BeamInstr *) *i) { + /* Pointer to bif export in i+1 */ + exp = (Export *) i[1]; + pc = i; + obif = (BifFunction) exp->beam[1]; + mfa = &exp->info.mfa; + } + else if (em_apply_bif == (BeamInstr *) *i) { + /* Pointer to bif in i+1, and mfa in i-3 */ + obif = (BifFunction) i[1]; + pc = c_p->cp; + mfa = erts_code_to_codemfa(i); + } + else { + ERTS_INTERNAL_ERROR("erts_schedule_bif() called " + "from unexpected instruction"); + obif = NULL; + } + ASSERT(bif); + + if (argc < 0) { /* reschedule original call */ + mod = mfa->module; + func = mfa->function; + argc = (int) mfa->arity; + } + + schedule(c_p, dirty_shadow_proc, mfa, obif, + pc, dbif, ibif, mod, func, argc, argv); + } + + if (dirty_shadow_proc) + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + + return THE_NON_VALUE; +} + +static BIF_RETTYPE +call_bif(Process *c_p, Eterm *reg, BeamInstr *I) +{ + NifExport *nep = ERTS_I_BEAM_OP_TO_NIF_EXPORT(I); + ErtsBifFunc bif = (ErtsBifFunc) nep->func; + BIF_RETTYPE ret; + + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); + + nep->func = ERTS_SCHED_BIF_TRAP_MARKER; + + ASSERT(bif); + + ret = (*bif)(c_p, reg, I); + + if (is_value(ret)) + erts_nif_export_restore(c_p, nep); + else if (c_p->freason != TRAP) + c_p->freason |= EXF_RESTORE_NIF; /* restore in handle_error() */ + else if (nep->func == ERTS_SCHED_BIF_TRAP_MARKER) { + /* BIF did an ordinary trap... */ + erts_nif_export_restore(c_p, nep); + } + /* else: + * BIF rescheduled itself using erts_schedule_bif(). + */ + + return ret; +} + +#ifdef ERTS_DIRTY_SCHEDULERS + +int +erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *reg) +{ + BIF_RETTYPE result; + int exiting; + Process *dirty_shadow_proc; + ErtsBifFunc bf; + NifExport *nep; +#ifdef DEBUG + Eterm *c_p_htop; + erts_aint32_t state; + + ASSERT(!c_p->scheduler_data); + state = erts_smp_atomic32_read_nob(&c_p->state); + ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING) + && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))); + ASSERT(esdp); + +#endif + + nep = ERTS_I_BEAM_OP_TO_NIF_EXPORT(I); + ASSERT(nep == ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p)); + + nep->func = ERTS_SCHED_BIF_TRAP_MARKER; + + bf = (ErtsBifFunc) I[1]; + + erts_smp_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_IO_PROC)); + + dirty_shadow_proc = erts_make_dirty_shadow_proc(esdp, c_p); + + dirty_shadow_proc->freason = c_p->freason; + dirty_shadow_proc->fvalue = c_p->fvalue; + dirty_shadow_proc->ftrace = c_p->ftrace; + dirty_shadow_proc->cp = c_p->cp; + dirty_shadow_proc->i = c_p->i; + +#ifdef DEBUG + c_p_htop = c_p->htop; +#endif + + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + + result = (*bf)(dirty_shadow_proc, reg, I); + + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + + ASSERT(c_p_htop == c_p->htop); + ASSERT(dirty_shadow_proc->static_flags & ERTS_STC_FLG_SHADOW_PROC); + ASSERT(dirty_shadow_proc->next == c_p); + + exiting = ERTS_PROC_IS_EXITING(c_p); + + if (!exiting) { + if (is_value(result)) + schedule(c_p, dirty_shadow_proc, NULL, NULL, NULL, dirty_bif_result, + NULL, am_erts_internal, am_dirty_bif_result, 1, &result); + else if (dirty_shadow_proc->freason != TRAP) { + Eterm argv[2]; + ASSERT(dirty_shadow_proc->freason <= MAX_SMALL); + argv[0] = make_small(dirty_shadow_proc->freason); + argv[1] = dirty_shadow_proc->fvalue; + schedule(c_p, dirty_shadow_proc, NULL, NULL, NULL, + dirty_bif_exception, NULL, am_erts_internal, + am_dirty_bif_exception, 2, argv); + } + else if (nep->func == ERTS_SCHED_BIF_TRAP_MARKER) { + /* Dirty BIF did an ordinary trap... */ + ASSERT(!(erts_smp_atomic32_read_nob(&c_p->state) + & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC))); + schedule(c_p, dirty_shadow_proc, NULL, NULL, NULL, + dirty_bif_trap, (void *) dirty_shadow_proc->i, + am_erts_internal, am_dirty_bif_trap, + dirty_shadow_proc->arity, reg); + } + /* else: + * BIF rescheduled itself using erts_schedule_bif(). + */ + c_p->freason = dirty_shadow_proc->freason; + c_p->fvalue = dirty_shadow_proc->fvalue; + c_p->ftrace = dirty_shadow_proc->ftrace; + c_p->cp = dirty_shadow_proc->cp; + c_p->i = dirty_shadow_proc->i; + c_p->arity = dirty_shadow_proc->arity; + } + + erts_flush_dirty_shadow_proc(dirty_shadow_proc); + + return exiting; +} + +#endif /* ERTS_DIRTY_SCHEDULERS */ + + #ifdef HARDDEBUG /* You'll need this line in bif.tab to be able to use this debug bif diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index 0c85e19ef0..ad55b19d0a 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -29,17 +29,35 @@ extern Export *erts_convert_time_unit_trap; #define BIF_P A__p -#define BIF_ALIST_0 Process* A__p, Eterm* BIF__ARGS -#define BIF_ALIST_1 Process* A__p, Eterm* BIF__ARGS -#define BIF_ALIST_2 Process* A__p, Eterm* BIF__ARGS -#define BIF_ALIST_3 Process* A__p, Eterm* BIF__ARGS -#define BIF_ALIST_4 Process* A__p, Eterm* BIF__ARGS +#define BIF_ALIST Process* A__p, Eterm* BIF__ARGS, BeamInstr *A__I +#define BIF_CALL_ARGS A__p, BIF__ARGS, A__I + +#define BIF_ALIST_0 BIF_ALIST +#define BIF_ALIST_1 BIF_ALIST +#define BIF_ALIST_2 BIF_ALIST +#define BIF_ALIST_3 BIF_ALIST +#define BIF_ALIST_4 BIF_ALIST #define BIF_ARG_1 (BIF__ARGS[0]) #define BIF_ARG_2 (BIF__ARGS[1]) #define BIF_ARG_3 (BIF__ARGS[2]) #define BIF_ARG_4 (BIF__ARGS[3]) +#define BIF_I A__I + +/* NBIF_* is for bif calls from native code... */ + +#define NBIF_ALIST Process* A__p, Eterm* BIF__ARGS +#define NBIF_CALL_ARGS A__p, BIF__ARGS + +#define NBIF_ALIST_0 NBIF_ALIST +#define NBIF_ALIST_1 NBIF_ALIST +#define NBIF_ALIST_2 NBIF_ALIST +#define NBIF_ALIST_3 NBIF_ALIST +#define NBIF_ALIST_4 NBIF_ALIST + +typedef BIF_RETTYPE (*ErtsBifFunc)(BIF_ALIST); + #define ERTS_IS_PROC_OUT_OF_REDS(p) \ ((p)->fcalls > 0 \ ? 0 \ @@ -480,6 +498,49 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, Eterm args[], int nargs); +typedef enum { + ERTS_SCHED_NORMAL, + ERTS_SCHED_DIRTY_CPU, + ERTS_SCHED_DIRTY_IO +} ErtsSchedType; + +#ifdef ERTS_DIRTY_SCHEDULERS +int erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, + BeamInstr *I, Eterm *reg); +#endif + +BIF_RETTYPE +erts_schedule_bif(Process *proc, + Eterm *argv, + BeamInstr *i, + ErtsBifFunc dbf, + ErtsSchedType sched_type, + Eterm mod, + Eterm func, + int argc); + +ERTS_GLB_INLINE BIF_RETTYPE +erts_reschedule_bif(Process *proc, + Eterm *argv, + BeamInstr *i, + ErtsBifFunc dbf, + ErtsSchedType sched_type); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE BIF_RETTYPE +erts_reschedule_bif(Process *proc, + Eterm *argv, + BeamInstr *i, + ErtsBifFunc dbf, + ErtsSchedType sched_type) +{ + return erts_schedule_bif(proc, argv, i, dbf, sched_type, + THE_NON_VALUE, THE_NON_VALUE, -1); +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + #ifdef ERL_WANT_HIPE_BIF_WRAPPER__ #ifndef HIPE @@ -510,16 +571,16 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, #define HIPE_WRAPPER_BIF_DISABLE_GC(BIF_NAME, ARITY) \ -BIF_RETTYPE hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (Process* c_p, \ - Eterm* args); \ -BIF_RETTYPE hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (Process* c_p, \ - Eterm* args) \ +BIF_RETTYPE \ +nbif_impl_hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (NBIF_ALIST); \ +BIF_RETTYPE \ +nbif_impl_hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (NBIF_ALIST) \ { \ BIF_RETTYPE res; \ - hipe_reserve_beam_trap_frame(c_p, args, ARITY); \ - res = BIF_NAME ## _ ## ARITY (c_p, args); \ - if (is_value(res) || c_p->freason != TRAP) { \ - hipe_unreserve_beam_trap_frame(c_p); \ + hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, ARITY); \ + res = nbif_impl_ ## BIF_NAME ## _ ## ARITY (NBIF_CALL_ARGS); \ + if (is_value(res) || BIF_P->freason != TRAP) { \ + hipe_unreserve_beam_trap_frame(BIF_P); \ } \ return res; \ } diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 32600f4338..f461da913e 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -420,6 +420,9 @@ bif erts_debug:set_internal_state/2 bif erts_debug:display/1 bif erts_debug:dist_ext_to_term/2 bif erts_debug:instructions/0 +bif erts_debug:dirty_cpu/2 +bif erts_debug:dirty_io/2 +bif erts_debug:dirty/3 # # Monitor testing bif's... diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 009d2b72d3..4659cb5a7b 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -93,6 +93,9 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE #ifdef USE_THREADS #if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP) " [ds:%beu:%beu:%beu]" +#endif +#if defined(ERTS_DIRTY_SCHEDULERS_TEST) + " [dirty-schedulers-TEST]" #endif " [async-threads:%d]" #endif diff --git a/erts/emulator/beam/erl_dirty_bif.tab b/erts/emulator/beam/erl_dirty_bif.tab new file mode 100644 index 0000000000..69421dcfcc --- /dev/null +++ b/erts/emulator/beam/erl_dirty_bif.tab @@ -0,0 +1,82 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 2016. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# %CopyrightEnd% +# + +# +# Static declaration of BIFs that should execute on dirty schedulers. +# +# ::= +# ::= ":" "/" +# ::= dirty-cpu | dirty-io | dirty-cpu-test | dirty-io-test +# +# When dirty scheduler support is available, a BIF declared with the +# 'dirty-cpu' type will unconditionally execute on a dirty CPU scheduler, +# and a BIF declared with the type 'dirty-io' will unconditionally execute +# on a dirty IO scheduler. When dirty scheduler support is not available +# all BIFs will of course execute on normal schedulers. +# +# When the emulator has been configured with the debug option +# '--enable-dirty-schedulers-test', BIFs with the types 'dirty-cpu-test', +# and 'dirty-io-test' will unconditionally execute on dirty schedulers. +# When this debug option has not been enabled, these BIFs will be executed +# on normal schedulers. +# +# BIFs marked as 'ubif' in ./bif.tab will be ignored, i.e., will always +# execute on normal schedulers. +# + +# --- Dirty BIFs --- + +dirty-cpu erts_debug:dirty_cpu/2 +dirty-io erts_debug:dirty_io/2 + +# --- TEST of Dirty BIF functionality --- +# Functions below will execute on dirty schedulers when emulator has +# been configured for testing dirty schedulers. This is used for test +# and debug purposes only. We really do *not* want to execute these +# on dirty schedulers on a real system. + +dirty-cpu-test erlang:'++'/2 +dirty-cpu-test erlang:append/2 +dirty-cpu-test erlang:'--'/2 +dirty-cpu-test erlang:subtract/2 +dirty-cpu-test erlang:iolist_size/1 +dirty-cpu-test erlang:make_tuple/2 +dirty-cpu-test erlang:make_tuple/3 +dirty-cpu-test erlang:append_element/2 +dirty-cpu-test erlang:insert_element/3 +dirty-cpu-test erlang:delete_element/2 +dirty-cpu-test erlang:atom_to_list/1 +dirty-cpu-test erlang:list_to_atom/1 +dirty-cpu-test erlang:list_to_existing_atom/1 +dirty-cpu-test erlang:integer_to_list/1 +dirty-cpu-test erlang:string_to_integer/1 +dirty-cpu-test erlang:list_to_integer/1 +dirty-cpu-test erlang:list_to_integer/2 +dirty-cpu-test erlang:float_to_list/1 +dirty-cpu-test erlang:float_to_list/2 +dirty-cpu-test erlang:float_to_binary/1 +dirty-cpu-test erlang:float_to_binary/2 +dirty-cpu-test erlang:string_to_float/1 +dirty-cpu-test erlang:list_to_float/1 +dirty-cpu-test erlang:binary_to_float/1 +dirty-cpu-test erlang:tuple_to_list/1 +dirty-cpu-test erlang:list_to_tuple/1 +dirty-cpu-test erlang:display/1 +dirty-cpu-test erlang:display_string/1 diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 643b46c861..6093b0cf39 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -42,6 +42,7 @@ #include "dtrace-wrapper.h" #include "erl_bif_unique.h" #include "dist.h" +#include "erl_nfunc_sched.h" #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20 @@ -2423,17 +2424,10 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) } /* - * If a NIF has saved arguments, they need to be added + * If a NIF or BIF has saved arguments, they need to be added */ - if (ERTS_PROC_GET_NIF_TRAP_EXPORT(p)) { - Eterm* argv; - int argc; - if (erts_setup_nif_gc(p, &argv, &argc)) { - roots[n].v = argv; - roots[n].sz = argc; - n++; - } - } + if (erts_setup_nif_export_rootset(p, &roots[n].v, &roots[n].sz)) + n++; ASSERT(n <= rootset->size); @@ -2985,6 +2979,8 @@ static void ERTS_INLINE offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size, Eterm* objv, int nobj) { + Eterm *v; + Uint sz; if (p->dictionary) { offset_heap(ERTS_PD_START(p->dictionary), ERTS_PD_SIZE(p->dictionary), @@ -3005,12 +3001,8 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size, offset_heap_ptr(objv, nobj, offs, area, area_size); } offset_off_heap(p, offs, area, area_size); - if (ERTS_PROC_GET_NIF_TRAP_EXPORT(p)) { - Eterm* argv; - int argc; - if (erts_setup_nif_gc(p, &argv, &argc)) - offset_heap_ptr(argv, argc, offs, area, area_size); - } + if (erts_setup_nif_export_rootset(p, &v, &sz)) + offset_heap_ptr(v, sz, offs, area, area_size); } static void diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c index 7ddf49937f..66bb55e6c8 100644 --- a/erts/emulator/beam/erl_msacc.c +++ b/erts/emulator/beam/erl_msacc.c @@ -137,8 +137,8 @@ void erts_msacc_init_thread(char *type, int id, int managed) { void erts_msacc_set_bif_state(ErtsMsAcc *__erts_msacc_cache, Eterm mod, void *fn) { #ifdef ERTS_MSACC_EXTENDED_BIFS -#define BIF_LIST(Mod,Func,Arity,FuncAddr,Num) \ - if (fn == &FuncAddr) { \ +#define BIF_LIST(Mod,Func,Arity,BifFuncAddr,FuncAddr,Num) \ + if (fn == &BifFuncAddr) { \ ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATIC_STATE_COUNT + Num); \ } else #include "erl_bif_list.h" diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h index 9ef86a1293..d64ef8c8b9 100644 --- a/erts/emulator/beam/erl_msacc.h +++ b/erts/emulator/beam/erl_msacc.h @@ -122,7 +122,7 @@ static char *erts_msacc_states[] = { "sleep", "timers" #ifdef ERTS_MSACC_EXTENDED_BIFS -#define BIF_LIST(Mod,Func,Arity,FuncAddr,Num) \ +#define BIF_LIST(Mod,Func,Arity,BifFuncAddr,FuncAddr,Num) \ ,"bif_" #Mod "_" #Func "_" #Arity #include "erl_bif_list.h" #undef BIF_LIST diff --git a/erts/emulator/beam/erl_nfunc_sched.c b/erts/emulator/beam/erl_nfunc_sched.c new file mode 100644 index 0000000000..0333545255 --- /dev/null +++ b/erts/emulator/beam/erl_nfunc_sched.c @@ -0,0 +1,144 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#define ERTS_WANT_NFUNC_SCHED_INTERNALS__ + +#include "global.h" +#include "erl_process.h" +#include "bif.h" +#include "erl_nfunc_sched.h" + +NifExport * +erts_new_proc_nif_export(Process *c_p, int argc) +{ + size_t size; + int i; + NifExport *nep, *old_nep; + + size = sizeof(NifExport) + (argc-1)*sizeof(Eterm); + nep = erts_alloc(ERTS_ALC_T_NIF_TRAP_EXPORT, size); + + for (i = 0; i < ERTS_NUM_CODE_IX; i++) + nep->exp.addressv[i] = &nep->exp.beam[0]; + + nep->argc = -1; /* unused marker */ + nep->argv_size = argc; + old_nep = ERTS_PROC_SET_NIF_TRAP_EXPORT(c_p, nep); + if (old_nep) + erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, old_nep); + return nep; +} + +void +erts_destroy_nif_export(Process *p) +{ + NifExport *nep = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL); + if (nep) { + if (nep->m) + erts_nif_export_cleanup_nif_mod(nep); + erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, nep); + } +} + +NifExport * +erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, + ErtsCodeMFA *mfa, void *nif, BeamInstr *pc, + BeamInstr instr, + void *dfunc, void *ifunc, + Eterm mod, Eterm func, + int argc, const Eterm *argv) +{ + Process *used_proc; + ErtsSchedulerData *esdp; + Eterm* reg; + NifExport* nep; + int i; + + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + & ERTS_PROC_LOCK_MAIN); + + if (dirty_shadow_proc) { + esdp = erts_get_scheduler_data(); + ASSERT(esdp && ERTS_SCHEDULER_IS_DIRTY(esdp)); + + used_proc = dirty_shadow_proc; + } + else { + esdp = erts_proc_sched_data(c_p); + ASSERT(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)); + + used_proc = c_p; + ERTS_VBUMP_ALL_REDS(c_p); + } + + reg = esdp->x_reg_array; + + if (mfa) + nep = erts_get_proc_nif_export(c_p, (int) mfa->arity); + else { + /* If no mfa, this is not the first schedule... */ + nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + ASSERT(nep && nep->argc >= 0); + } + + if (nep->argc < 0) { + /* + * First schedule; save things that might + * need to be restored... + */ + for (i = 0; i < (int) mfa->arity; i++) + nep->argv[i] = reg[i]; + nep->pc = pc; + nep->cp = c_p->cp; + ASSERT(nif); + nep->nif = nif; + nep->mfa = mfa; + nep->current = c_p->current; + ASSERT(argc >= 0); + nep->argc = (int) mfa->arity; + nep->m = NULL; + + ASSERT(!erts_check_nif_export_in_area(c_p, + (char *) nep, + (sizeof(NifExport) + + (sizeof(Eterm) + *(nep->argc-1))))); + } + /* Copy new arguments into register array if necessary... */ + if (reg != argv) { + for (i = 0; i < argc; i++) + reg[i] = argv[i]; + } + ASSERT(is_atom(mod) && is_atom(func)); + nep->exp.info.mfa.module = mod; + nep->exp.info.mfa.function = func; + nep->exp.info.mfa.arity = (Uint) argc; + nep->exp.beam[0] = (BeamInstr) instr; /* call_nif || apply_bif */ + nep->exp.beam[1] = (BeamInstr) dfunc; + nep->func = ifunc; + used_proc->arity = argc; + used_proc->freason = TRAP; + used_proc->i = (BeamInstr*) nep->exp.addressv[0]; + return nep; +} diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h new file mode 100644 index 0000000000..72de7f0eb5 --- /dev/null +++ b/erts/emulator/beam/erl_nfunc_sched.h @@ -0,0 +1,289 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef ERL_NFUNC_SCHED_H__ +#define ERL_NFUNC_SCHED_H__ + +#include "erl_process.h" +#include "bif.h" + +/* + * NIF exports need a few more items than the Export struct provides, + * including the erl_module_nif* and a NIF function pointer, so the + * NifExport below adds those. The Export member must be first in the + * struct. A number of values are stored for error handling purposes + * only. + * + * 'argc' is >= 0 when NifExport is in use, and < 0 when not. + */ + +typedef struct { + Export exp; + struct erl_module_nif* m; /* NIF module, or NULL if BIF */ + void *func; /* Indirect NIF or BIF to execute (may be unused) */ + ErtsCodeMFA *current;/* Current as set when originally called */ + /* --- The following is only used on error --- */ + BeamInstr *pc; /* Program counter */ + BeamInstr *cp; /* Continuation pointer */ + void *nif; /* Original NIF/BIF call */ + ErtsCodeMFA *mfa; /* MFA of original call */ + int argc; /* Number of arguments in original call */ + int argv_size; /* Allocated size of argv */ + Eterm argv[1]; /* Saved arguments from the original call */ +} NifExport; + +NifExport *erts_new_proc_nif_export(Process *c_p, int argc); +void erts_destroy_nif_export(Process *p); +NifExport *erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, + ErtsCodeMFA *mfa, void *nif, BeamInstr *pc, + BeamInstr instr, + void *dfunc, void *ifunc, + Eterm mod, Eterm func, + int argc, const Eterm *argv); +void erts_nif_export_cleanup_nif_mod(NifExport *ep); /* erl_nif.c */ +ERTS_GLB_INLINE NifExport *erts_get_proc_nif_export(Process *c_p, int extra); +ERTS_GLB_INLINE int erts_setup_nif_export_rootset(Process* proc, Eterm** objv, + Uint* nobj); +ERTS_GLB_INLINE int erts_check_nif_export_in_area(Process *p, + char *start, Uint size); +ERTS_GLB_INLINE void erts_nif_export_restore(Process *c_p, NifExport *ep); +ERTS_GLB_INLINE void erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, + Eterm *reg, void **nif); +ERTS_GLB_INLINE Process *erts_proc_shadow2real(Process *c_p); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE NifExport * +erts_get_proc_nif_export(Process *c_p, int argc) +{ + NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + if (!nep || (nep->argc < 0 && nep->argv_size < argc)) + return erts_new_proc_nif_export(c_p, argc); + return nep; +} + +/* + * If a process has saved arguments, they need to be part of the GC + * rootset. The function below is called from setup_rootset() in + * erl_gc.c. Any exception term saved in the NifExport is also made + * part of the GC rootset here; it always resides in rootset[0]. + */ +ERTS_GLB_INLINE int +erts_setup_nif_export_rootset(Process* proc, Eterm** objv, Uint* nobj) +{ + NifExport* ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + + if (!ep || ep->argc <= 0) + return 0; + + *objv = ep->argv; + *nobj = ep->argc; + return 1; +} + +/* + * Check if nif export points into code area... + */ +ERTS_GLB_INLINE int +erts_check_nif_export_in_area(Process *p, char *start, Uint size) +{ + NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(p); + if (!nep || nep->argc < 0) + return 0; + if (ErtsInArea(nep->pc, start, size)) + return 1; + if (ErtsInArea(nep->cp, start, size)) + return 1; + if (ErtsInArea(nep->mfa, start, size)) + return 1; + if (ErtsInArea(nep->current, start, size)) + return 1; + return 0; +} + +ERTS_GLB_INLINE void +erts_nif_export_restore(Process *c_p, NifExport *ep) +{ + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); + ERTS_SMP_LC_ASSERT(!(c_p->static_flags + & ERTS_STC_FLG_SHADOW_PROC)); + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + & ERTS_PROC_LOCK_MAIN); + + c_p->current = ep->current; + ep->argc = -1; /* Unused nif-export marker... */ +} + +ERTS_GLB_INLINE void +erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, Eterm *reg, + void **nif) +{ + NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + int ix; + + ASSERT(nep); + *pc = nep->pc; + c_p->cp = nep->cp; + *nif = nep->nif; + for (ix = 0; ix < nep->argc; ix++) + reg[ix] = nep->argv[ix]; + erts_nif_export_restore(c_p, nep); +} + +ERTS_GLB_INLINE Process * +erts_proc_shadow2real(Process *c_p) +{ +#ifdef ERTS_DIRTY_SCHEDULERS + if (c_p->static_flags & ERTS_STC_FLG_SHADOW_PROC) { + Process *real_c_p = c_p->next; + ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); + ASSERT(real_c_p->common.id == c_p->common.id); + return real_c_p; + } + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); +#endif + return c_p; +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* ERL_NFUNC_SCHED_H__ */ + +#if defined(ERTS_WANT_NFUNC_SCHED_INTERNALS__) && !defined(ERTS_NFUNC_SCHED_INTERNALS__) +#define ERTS_NFUNC_SCHED_INTERNALS__ + +#define ERTS_I_BEAM_OP_TO_NIF_EXPORT(I) \ + (ASSERT(BeamOp(op_apply_bif) == (BeamInstr *) (*(I)) \ + || BeamOp(op_call_nif) == (BeamInstr *) (*(I))), \ + ((NifExport *) (((char *) (I)) - offsetof(NifExport, exp.beam[0])))) + +#ifdef ERTS_DIRTY_SCHEDULERS + +#include "erl_message.h" +#include + +ERTS_GLB_INLINE void erts_flush_dirty_shadow_proc(Process *sproc); +ERTS_GLB_INLINE void erts_cache_dirty_shadow_proc(Process *sproc); +ERTS_GLB_INLINE Process *erts_make_dirty_shadow_proc(ErtsSchedulerData *esdp, + Process *c_p); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE void +erts_flush_dirty_shadow_proc(Process *sproc) +{ + Process *c_p = sproc->next; + + ASSERT(sproc->common.id == c_p->common.id); + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + & ERTS_PROC_LOCK_MAIN); + + ASSERT(c_p->stop == sproc->stop); + ASSERT(c_p->hend == sproc->hend); + ASSERT(c_p->heap == sproc->heap); + ASSERT(c_p->abandoned_heap == sproc->abandoned_heap); + ASSERT(c_p->heap_sz == sproc->heap_sz); + ASSERT(c_p->high_water == sproc->high_water); + ASSERT(c_p->old_heap == sproc->old_heap); + ASSERT(c_p->old_htop == sproc->old_htop); + ASSERT(c_p->old_hend == sproc->old_hend); + + ASSERT(c_p->htop <= sproc->htop && sproc->htop <= c_p->stop); + + c_p->htop = sproc->htop; + + if (!c_p->mbuf) + c_p->mbuf = sproc->mbuf; + else if (sproc->mbuf) { + ErlHeapFragment *bp; + for (bp = sproc->mbuf; bp->next; bp = bp->next) + ASSERT(!bp->off_heap.first); + bp->next = c_p->mbuf; + c_p->mbuf = sproc->mbuf; + } + + c_p->mbuf_sz += sproc->mbuf_sz; + + if (!c_p->off_heap.first) + c_p->off_heap.first = sproc->off_heap.first; + else if (sproc->off_heap.first) { + struct erl_off_heap_header *ohhp; + for (ohhp = sproc->off_heap.first; ohhp->next; ohhp = ohhp->next) + ; + ohhp->next = c_p->off_heap.first; + c_p->off_heap.first = sproc->off_heap.first; + } + + c_p->off_heap.overhead += sproc->off_heap.overhead; +} + +ERTS_GLB_INLINE void +erts_cache_dirty_shadow_proc(Process *sproc) +{ + Process *c_p = sproc->next; + ASSERT(c_p); + ASSERT(sproc->common.id == c_p->common.id); + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + & ERTS_PROC_LOCK_MAIN); + + sproc->htop = c_p->htop; + sproc->stop = c_p->stop; + sproc->hend = c_p->hend; + sproc->heap = c_p->heap; + sproc->abandoned_heap = c_p->abandoned_heap; + sproc->heap_sz = c_p->heap_sz; + sproc->high_water = c_p->high_water; + sproc->old_hend = c_p->old_hend; + sproc->old_htop = c_p->old_htop; + sproc->old_heap = c_p->old_heap; + sproc->mbuf = NULL; + sproc->mbuf_sz = 0; + ERTS_INIT_OFF_HEAP(&sproc->off_heap); +} + +ERTS_GLB_INLINE Process * +erts_make_dirty_shadow_proc(ErtsSchedulerData *esdp, Process *c_p) +{ + Process *sproc; + + ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)); + + sproc = esdp->dirty_shadow_process; + ASSERT(sproc); + ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC); + ASSERT(erts_smp_atomic32_read_nob(&sproc->state) + == (ERTS_PSFLG_ACTIVE + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_PROXY)); + + sproc->next = c_p; + sproc->common.id = c_p->common.id; + + erts_cache_dirty_shadow_proc(sproc); + + return sproc; +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* ERTS_DIRTY_SCHEDULERS */ + +#endif /* defined(ERTS_WANT_NFUNC_SCHED_INTERNALS__) && !defined(ERTS_NFUNC_SCHED_INTERNALS__) */ + diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 5499512dd1..ee4b22f7ca 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -55,6 +55,9 @@ #include "dtrace-wrapper.h" #include "erl_process.h" #include "erl_bif_unique.h" +#undef ERTS_WANT_NFUNC_SCHED_INTERNALS__ +#define ERTS_WANT_NFUNC_SCHED_INTERNALS__ +#include "erl_nfunc_sched.h" #if defined(USE_DYNAMIC_TRACE) && (defined(USE_DTRACE) || defined(USE_SYSTEMTAP)) #define HAVE_USE_DTRACE 1 #endif @@ -79,8 +82,11 @@ struct erl_module_nif { ErlNifFunc _funcs_copy_[1]; /* only used for old libs */ }; +typedef ERL_NIF_TERM (*NativeFunPtr)(ErlNifEnv*, int, const ERL_NIF_TERM[]); + #ifdef DEBUG # define READONLY_CHECK +# define ERTS_DBG_NIF_NOT_SCHED_MARKER ((void *) (UWord) 1) #endif #ifdef READONLY_CHECK # define ADD_READONLY_CHECK(ENV,PTR,SIZE) add_readonly_check(ENV,PTR,SIZE) @@ -219,38 +225,6 @@ static void cache_env(ErlNifEnv* env); static void full_flush_env(ErlNifEnv *env); static void flush_env(ErlNifEnv* env); -#ifdef ERTS_DIRTY_SCHEDULERS -void erts_pre_dirty_nif(ErtsSchedulerData *esdp, - ErlNifEnv* env, Process* p, - struct erl_module_nif* mod_nif) -{ - Process *sproc; -#ifdef DEBUG - erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); - - ASSERT(!p->scheduler_data); - ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING) - && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))); - ASSERT(esdp); -#endif - - erts_pre_nif(env, p, mod_nif, NULL); - - sproc = esdp->dirty_shadow_process; - ASSERT(sproc); - ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC); - ASSERT(erts_smp_atomic32_read_nob(&sproc->state) - == (ERTS_PSFLG_ACTIVE - | ERTS_PSFLG_DIRTY_RUNNING - | ERTS_PSFLG_PROXY)); - - sproc->next = p; - sproc->common.id = p->common.id; - env->proc = sproc; - full_cache_env(env); -} -#endif - /* Temporary object header, auto-deallocated when NIF returns * or when independent environment is cleared. */ @@ -278,115 +252,158 @@ void erts_post_nif(ErlNifEnv* env) env->exiting = ERTS_PROC_IS_EXITING(env->proc); } -#ifdef ERTS_DIRTY_SCHEDULERS -void erts_post_dirty_nif(ErlNifEnv* env) + +/* + * Initialize a NifExport struct. Create it if needed and store it in the + * proc. The direct_fp function is what will be invoked by op_call_nif, and + * the indirect_fp function, if not NULL, is what the direct_fp function + * will call. If the allocated NifExport isn't enough to hold all of argv, + * allocate a larger one. Save 'current' and registers if first time this + * call is scheduled. + */ + +static ERTS_INLINE ERL_NIF_TERM +schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp, + Eterm mod, Eterm func_name, int argc, const ERL_NIF_TERM argv[]) { - Process *c_p; - ASSERT(env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC); - ASSERT(env->proc->next); - erts_unblock_fpe(env->fpe_was_unmasked); - full_flush_env(env); - free_tmp_objs(env); - c_p = env->proc->next; - env->exiting = ERTS_PROC_IS_EXITING(c_p); - ERTS_VBUMP_ALL_REDS(c_p); + Export *exp; + NifExport *ep; + Process *c_p, *dirty_shadow_proc; + + execution_state(env, &c_p, NULL); + if (c_p == env->proc) + dirty_shadow_proc = NULL; + else + dirty_shadow_proc = env->proc; + + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); + + exp = ErtsContainerStruct(c_p->current, Export, info.mfa); + + ep = erts_nif_export_schedule(c_p, dirty_shadow_proc, + c_p->current, + (BifFunction) exp->beam[1], + c_p->cp, + (BeamInstr) em_call_nif, + direct_fp, indirect_fp, + mod, func_name, + argc, (const Eterm *) argv); + if (!ep->m) { + /* First time this call is scheduled... */ + erts_refc_inc(&env->mod_nif->rt_dtor_cnt, 1); + ep->m = env->mod_nif; + } + return (ERL_NIF_TERM) THE_NON_VALUE; } -#endif -static void full_flush_env(ErlNifEnv* env) -{ #ifdef ERTS_DIRTY_SCHEDULERS - if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { - /* Dirty nif call using shadow process struct */ - Process *c_p = env->proc->next; - - ASSERT(is_scheduler() < 0); - ASSERT(env->proc->common.id == c_p->common.id); - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) - & ERTS_PROC_LOCK_MAIN); - - if (!env->heap_frag) { - ASSERT(env->hp_end == HEAP_LIMIT(c_p)); - ASSERT(env->hp >= HEAP_TOP(c_p)); - ASSERT(env->hp <= HEAP_LIMIT(c_p)); - HEAP_TOP(c_p) = env->hp; + +static ERL_NIF_TERM dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +int +erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *reg) +{ + int exiting; + ERL_NIF_TERM *argv = (ERL_NIF_TERM *) reg; + NifExport *nep = ERTS_I_BEAM_OP_TO_NIF_EXPORT(I); + ErtsCodeMFA *codemfa = erts_code_to_codemfa(I); + NativeFunPtr dirty_nif = (NativeFunPtr) I[1]; + ErlNifEnv env; + ERL_NIF_TERM result; +#ifdef DEBUG + erts_aint32_t state = erts_smp_atomic32_read_nob(&c_p->state); + + ASSERT(nep == ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p)); + + ASSERT(!c_p->scheduler_data); + ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING) + && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))); + ASSERT(esdp); + + nep->func = ERTS_DBG_NIF_NOT_SCHED_MARKER; +#endif + + erts_pre_nif(&env, c_p, nep->m, NULL); + + env.proc = erts_make_dirty_shadow_proc(esdp, c_p); + + env.proc->freason = EXC_NULL; + env.proc->fvalue = NIL; + env.proc->ftrace = NIL; + env.proc->i = c_p->i; + + ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p))); + + erts_smp_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_IO_PROC)); + + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + + result = (*dirty_nif)(&env, codemfa->arity, argv); /* Call dirty NIF */ + + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + + ASSERT(env.proc->static_flags & ERTS_STC_FLG_SHADOW_PROC); + ASSERT(env.proc->next == c_p); + + exiting = ERTS_PROC_IS_EXITING(c_p); + + if (!exiting) { + if (env.exception_thrown) { + schedule_exception: + schedule(&env, dirty_nif_exception, NULL, + am_erts_internal, am_dirty_nif_exception, + 1, &env.proc->fvalue); + } + else if (is_value(result)) { + schedule(&env, dirty_nif_finalizer, NULL, + am_erts_internal, am_dirty_nif_finalizer, + 1, &result); + } + else if (env.proc->freason != TRAP) { /* user returned garbage... */ + ERTS_DECL_AM(badreturn); + (void) enif_raise_exception(&env, AM_badreturn); + goto schedule_exception; } else { - Uint usz; - ASSERT(env->hp_end != HEAP_LIMIT(c_p)); - ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); - - HEAP_TOP(c_p) = HEAP_TOP(env->proc); - usz = env->hp - env->heap_frag->mem; - env->proc->mbuf_sz += usz - env->heap_frag->used_size; - env->heap_frag->used_size = usz; - - ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); - - if (c_p->mbuf) { - ErlHeapFragment *bp; - for (bp = env->proc->mbuf; bp->next; bp = bp->next) - ; - bp->next = c_p->mbuf; - } + /* Rescheduled by dirty NIF call... */ + ASSERT(nep->func != ERTS_DBG_NIF_NOT_SCHED_MARKER); + } + c_p->i = env.proc->i; + c_p->arity = env.proc->arity; + } - c_p->mbuf = env->proc->mbuf; - c_p->mbuf_sz += env->proc->mbuf_sz; +#ifdef DEBUG + if (nep->func == ERTS_DBG_NIF_NOT_SCHED_MARKER) + nep->func = NULL; +#endif - } + erts_unblock_fpe(env.fpe_was_unmasked); + full_flush_env(&env); + free_tmp_objs(&env); - if (!c_p->off_heap.first) - c_p->off_heap.first = env->proc->off_heap.first; - else if (env->proc->off_heap.first) { - struct erl_off_heap_header *ohhp; - for (ohhp = env->proc->off_heap.first; ohhp->next; ohhp = ohhp->next) - ; - ohhp->next = c_p->off_heap.first; - c_p->off_heap.first = env->proc->off_heap.first; - } - c_p->off_heap.overhead += env->proc->off_heap.overhead; + return exiting; +} - return; - } #endif +static void full_flush_env(ErlNifEnv* env) +{ flush_env(env); +#ifdef ERTS_DIRTY_SCHEDULERS + if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) + /* Dirty nif call using shadow process struct */ + erts_flush_dirty_shadow_proc(env->proc); +#endif } static void full_cache_env(ErlNifEnv* env) { #ifdef ERTS_DIRTY_SCHEDULERS - if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { - /* Dirty nif call using shadow process struct */ - Process *sproc = env->proc; - Process *c_p = sproc->next; - ASSERT(c_p); - ASSERT(is_scheduler() < 0); - ASSERT(env->proc->common.id == c_p->common.id); - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) - & ERTS_PROC_LOCK_MAIN); - - sproc->htop = c_p->htop; - sproc->stop = c_p->stop; - sproc->hend = c_p->hend; - sproc->heap = c_p->heap; - sproc->abandoned_heap = c_p->abandoned_heap; - sproc->heap_sz = c_p->heap_sz; - sproc->high_water = c_p->high_water; - sproc->old_hend = c_p->old_hend; - sproc->old_htop = c_p->old_htop; - sproc->old_heap = c_p->old_heap; - sproc->mbuf = NULL; - sproc->mbuf_sz = 0; - ERTS_INIT_OFF_HEAP(&sproc->off_heap); - - env->hp_end = HEAP_LIMIT(c_p); - env->hp = HEAP_TOP(c_p); - env->heap_frag = NULL; - return; - } + if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) + erts_cache_dirty_shadow_proc(env->proc); #endif - cache_env(env); } @@ -1303,22 +1320,15 @@ Eterm enif_make_badarg(ErlNifEnv* env) Eterm enif_raise_exception(ErlNifEnv* env, ERL_NIF_TERM reason) { - Process *c_p; - - execution_state(env, &c_p, NULL); - env->exception_thrown = 1; - c_p->fvalue = reason; - BIF_ERROR(c_p, EXC_ERROR); + env->proc->fvalue = reason; + BIF_ERROR(env->proc, EXC_ERROR); } int enif_has_pending_exception(ErlNifEnv* env, ERL_NIF_TERM* reason) { - if (env->exception_thrown && reason != NULL) { - Process *c_p; - execution_state(env, &c_p, NULL); - *reason = c_p->fvalue; - } + if (env->exception_thrown && reason != NULL) + *reason = env->proc->fvalue; return env->exception_thrown; } @@ -2269,188 +2279,28 @@ int enif_consume_timeslice(ErlNifEnv* env, int percent) return ERTS_BIF_REDS_LEFT(proc) == 0; } -/* - * NIF exports need a few more items than the Export struct provides, - * including the erl_module_nif* and a NIF function pointer, so the - * NifExport below adds those. The Export member must be first in the - * struct. The saved_current, exception_thrown, saved_argc, rootset_extra, and - * rootset members are used to track the MFA, any pending exception, and - * arguments of the top NIF in case a chain of one or more - * enif_schedule_nif() calls results in an exception, since in that case - * the original MFA and registers have to be restored before returning to - * Erlang to ensure stacktrace information associated with the exception is - * correct. - */ -typedef ERL_NIF_TERM (*NativeFunPtr)(ErlNifEnv*, int, const ERL_NIF_TERM[]); - -typedef struct { - Export exp; - struct erl_module_nif* m; - NativeFunPtr fp; - ErtsCodeMFA *saved_current; - int exception_thrown; - int saved_argc; - int rootset_extra; - Eterm rootset[1]; -} NifExport; - -/* - * If a process has saved arguments, they need to be part of the GC - * rootset. The function below is called from setup_rootset() in - * erl_gc.c. This function is declared in erl_process.h. Any exception term - * saved in the NifExport is also made part of the GC rootset here; it - * always resides in rootset[0]. - */ -int -erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj) -{ - NifExport* ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - int gc = ep && (ep->saved_argc > 0 || ep->rootset[0] != NIL); - - if (gc) { - *objv = ep->rootset; - *nobj = 1 + ep->saved_argc; - } - return gc; -} - -int -erts_check_nif_export_in_area(Process *p, char *start, Uint size) -{ - NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(p); - if (!nep || !nep->saved_current) - return 0; - if (ErtsInArea(nep->saved_current, start, size)) - return 1; - return 0; -} - -/* - * Allocate a NifExport and set it in proc specific data - */ -static NifExport* -allocate_nif_sched_data(Process* proc, int argc) -{ - NifExport* ep; - size_t total; - int i; - - total = sizeof(NifExport) + argc*sizeof(Eterm); - ep = erts_alloc(ERTS_ALC_T_NIF_TRAP_EXPORT, total); - sys_memset((void*) ep, 0, total); - ep->rootset_extra = argc; - ep->rootset[0] = NIL; - for (i=0; iexp.addressv[i] = &ep->exp.beam[0]; - } - ep->exp.beam[0] = (BeamInstr) em_call_nif; - (void) ERTS_PROC_SET_NIF_TRAP_EXPORT(proc, ep); - return ep; -} - static ERTS_INLINE void -destroy_nif_export(NifExport *nif_export) +nif_export_cleanup_nif_mod(NifExport *ep) { - erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, (void *) nif_export); + if (erts_refc_dectest(&ep->m->rt_dtor_cnt, 0) == 0 && ep->m->mod == NULL) + close_lib(ep->m); + ep->m = NULL; } void -erts_destroy_nif_export(void *nif_export) +erts_nif_export_cleanup_nif_mod(NifExport *ep) { - destroy_nif_export((NifExport *) nif_export); + nif_export_cleanup_nif_mod(ep); } -/* - * Initialize a NifExport struct. Create it if needed and store it in the - * proc. The direct_fp function is what will be invoked by op_call_nif, and - * the indirect_fp function, if not NULL, is what the direct_fp function - * will call. If the allocated NifExport isn't enough to hold all of argv, - * allocate a larger one. Save MFA and registers only if the need_save - * parameter is true. - */ -static ERL_NIF_TERM -init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp, - int need_save, int argc, const ERL_NIF_TERM argv[]) +static ERTS_INLINE void +nif_export_restore(Process *c_p, NifExport *ep) { - Process* proc; - Eterm* reg; - NifExport* ep; - int i, scheduler; - int orig_argc; - - execution_state(env, &proc, &scheduler); - - ASSERT(scheduler); - - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(proc) - & ERTS_PROC_LOCK_MAIN); - - reg = erts_proc_sched_data(proc)->x_reg_array; - - ASSERT(!need_save || proc->current); - orig_argc = need_save ? (int) proc->current->arity : 0; - - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - if (!ep) - ep = allocate_nif_sched_data(proc, orig_argc); - else if (need_save && ep->rootset_extra < orig_argc) { - NifExport* new_ep = allocate_nif_sched_data(proc, orig_argc); - destroy_nif_export(ep); - ep = new_ep; - } - if (env->exception_thrown) { - ep->exception_thrown = 1; - ep->rootset[0] = proc->fvalue; - } else { - ep->exception_thrown = 0; - ep->rootset[0] = NIL; - } - if (scheduler > 0) - ERTS_VBUMP_ALL_REDS(proc); - if (need_save) { - ep->saved_current = proc->current; - ep->saved_argc = orig_argc; - for (i = 0; i < orig_argc; i++) - ep->rootset[i+1] = reg[i]; - } - for (i = 0; i < argc; i++) - reg[i] = (Eterm) argv[i]; - proc->i = (BeamInstr*) ep->exp.addressv[0]; - ep->exp.info.mfa.module = proc->current->module; - ep->exp.info.mfa.function = proc->current->function; - ep->exp.info.mfa.arity = argc; - ep->exp.beam[1] = (BeamInstr) direct_fp; - ep->m = env->mod_nif; - ep->fp = indirect_fp; - proc->freason = TRAP; - proc->arity = argc; - return THE_NON_VALUE; + erts_nif_export_restore(c_p, ep); + ASSERT(ep->m); + nif_export_cleanup_nif_mod(ep); } -/* - * Restore saved MFA and registers. Registers are restored only when the - * exception flag is true. - */ -static void -restore_nif_mfa(Process* proc, NifExport* ep, int exception) -{ - int i; - - ERTS_SMP_LC_ASSERT(!(proc->static_flags - & ERTS_STC_FLG_SHADOW_PROC)); - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(proc) - & ERTS_PROC_LOCK_MAIN); - - ASSERT(ep->saved_current != &ep->exp.info.mfa); - proc->current = ep->saved_current; - ep->saved_current = NULL; - if (exception) { - Eterm* reg = erts_proc_sched_data(proc)->x_reg_array; - for (i = 0; i < ep->saved_argc; i++) - reg[i] = ep->rootset[i+1]; - } - ep->saved_argc = 0; -} #ifdef ERTS_DIRTY_SCHEDULERS @@ -2459,7 +2309,7 @@ restore_nif_mfa(Process* proc, NifExport* ep, int exception) * switch the process off a dirty scheduler thread and back onto a regular * scheduler thread, and then return the result from the dirty NIF. It also * restores the original NIF MFA when necessary based on the value of - * ep->fp set by execute_dirty_nif via init_nif_sched_data -- non-NULL + * ep->func set by execute_dirty_nif via init_nif_sched_data -- non-NULL * means restore, NULL means do not restore. */ static ERL_NIF_TERM @@ -2474,9 +2324,7 @@ dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc))); ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); ASSERT(ep); - ASSERT(!ep->exception_thrown); - if (ep->fp) - restore_nif_mfa(proc, ep, 0); + nif_export_restore(proc, ep); return argv[0]; } @@ -2486,148 +2334,100 @@ dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) static ERL_NIF_TERM dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + ERL_NIF_TERM ret; Process* proc; NifExport* ep; + Eterm exception; execution_state(env, &proc, NULL); + ASSERT(argc == 1); ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc))); ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); ASSERT(ep); - ASSERT(ep->exception_thrown); - if (ep->fp) - restore_nif_mfa(proc, ep, 1); - return enif_raise_exception(env, ep->rootset[0]); + exception = argv[0]; /* argv overwritten by restore below... */ + nif_export_cleanup_nif_mod(ep); + ret = enif_raise_exception(env, exception); + + /* Restore orig info for error and clear nif export in handle_error() */ + proc->freason |= EXF_RESTORE_NIF; + return ret; } /* - * Dirty NIF execution wrapper function. Invoke an application's dirty NIF, - * then check the result and schedule the appropriate finalizer function - * where needed. Also restore the original NIF MFA when appropriate. + * Dirty NIF scheduling wrapper function. Schedule a dirty NIF to execute. + * The dirty scheduler thread type (CPU or I/O) is indicated in flags + * parameter. */ -static ERL_NIF_TERM -execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +static ERTS_INLINE ERL_NIF_TERM +schedule_dirty_nif(ErlNifEnv* env, int flags, NativeFunPtr fp, + Eterm func_name, int argc, const ERL_NIF_TERM argv[]) { Process* proc; - NativeFunPtr fp; - NifExport* ep; - ERL_NIF_TERM result; - - execution_state(env, &proc, NULL); - ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); - fp = ep->fp; - - ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc))); - - /* - * Set ep->fp to NULL before the native call so we know later whether it scheduled another NIF for execution - */ - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - ASSERT(ep && fp); - - ep->fp = NULL; - erts_smp_atomic32_read_band_mb(&proc->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC - | ERTS_PSFLG_DIRTY_IO_PROC)); + ASSERT(is_atom(func_name)); + ASSERT(fp); - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); + ASSERT(flags==ERL_NIF_DIRTY_JOB_IO_BOUND || flags==ERL_NIF_DIRTY_JOB_CPU_BOUND); - result = (*fp)(env, argc, argv); + execution_state(env, &proc, NULL); - erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); + (void) erts_smp_atomic32_read_bset_nob(&proc->state, + (ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_IO_PROC), + (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND + ? ERTS_PSFLG_DIRTY_CPU_PROC + : ERTS_PSFLG_DIRTY_IO_PROC)); - if (erts_refc_dectest(&env->mod_nif->rt_dtor_cnt, 0) == 0 && env->mod_nif->mod == NULL) - close_lib(env->mod_nif); - /* - * If no more NIFs were scheduled by the native call via - * enif_schedule_nif(), then ep->fp will still be NULL as set above, in - * which case we need to restore the original NIF calling - * context. Reuse fp essentially as a boolean for this, passing it to - * init_nif_sched_data below. Both dirty_nif_exception and - * dirty_nif_finalizer then check ep->fp to decide whether or not to - * restore the original calling context. - */ - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - ASSERT(ep); - if (ep->fp) - fp = NULL; - if (is_non_value(result) || env->exception_thrown) { - if (proc->freason != TRAP) { - return init_nif_sched_data(env, dirty_nif_exception, fp, 0, argc, argv); - } else { - if (ep->fp == NULL) - restore_nif_mfa(proc, ep, 1); - return THE_NON_VALUE; - } - } - else - return init_nif_sched_data(env, dirty_nif_finalizer, fp, 0, 1, &result); + return schedule(env, fp, NULL, proc->current->module, func_name, argc, argv); } -/* - * Dirty NIF scheduling wrapper function. Schedule a dirty NIF to execute - * via the execute_dirty_nif() wrapper function. The dirty scheduler thread - * type (CPU or I/O) is indicated in flags parameter. - */ static ERTS_INLINE ERL_NIF_TERM -schedule_dirty_nif(ErlNifEnv* env, int flags, int argc, const ERL_NIF_TERM argv[]) +static_schedule_dirty_nif(ErlNifEnv* env, erts_aint32_t dirty_psflg, + int argc, const ERL_NIF_TERM argv[]) { - ERL_NIF_TERM result; - erts_aint32_t act, dirty_flag; - Process* proc; + Process *proc; + NifExport *ep; + Eterm mod, func; NativeFunPtr fp; - NifExport* ep; - int need_save, scheduler; - execution_state(env, &proc, &scheduler); - if (scheduler <= 0) { - ASSERT(scheduler < 0); - erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); - } + execution_state(env, &proc, NULL); + + /* + * Called in order to schedule statically determined + * dirty NIF calls... + * + * Note that 'current' does not point into a NifExport + * structure; only a structure with similar + * parts (located in code). + */ ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); - fp = ep->fp; + mod = proc->current->module; + func = proc->current->function; + fp = (NativeFunPtr) ep->func; + ASSERT(is_atom(mod) && is_atom(func)); ASSERT(fp); - ASSERT(flags==ERL_NIF_DIRTY_JOB_IO_BOUND || flags==ERL_NIF_DIRTY_JOB_CPU_BOUND); - - if (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND) - dirty_flag = ERTS_PSFLG_DIRTY_CPU_PROC; - else - dirty_flag = ERTS_PSFLG_DIRTY_IO_PROC; - - act = erts_smp_atomic32_read_bor_nob(&proc->state, dirty_flag); - if (!(act & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC))) - erts_refc_inc(&env->mod_nif->rt_dtor_cnt, 1); - else if ((act & (ERTS_PSFLG_DIRTY_CPU_PROC - | ERTS_PSFLG_DIRTY_IO_PROC)) & ~dirty_flag) { - /* clear other flag... */ - if (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND) - dirty_flag = ERTS_PSFLG_DIRTY_IO_PROC; - else - dirty_flag = ERTS_PSFLG_DIRTY_CPU_PROC; - erts_smp_atomic32_read_band_nob(&proc->state, ~dirty_flag); - } + (void) erts_smp_atomic32_read_bset_nob(&proc->state, + (ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_IO_PROC), + dirty_psflg); - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - need_save = (ep == NULL || !ep->saved_current); - result = init_nif_sched_data(env, execute_dirty_nif, fp, need_save, argc, argv); - if (scheduler <= 0) - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); - return result; + return schedule(env, fp, NULL, mod, func, argc, argv); } static ERL_NIF_TERM -schedule_dirty_io_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +static_schedule_dirty_io_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - return schedule_dirty_nif(env, ERL_NIF_DIRTY_JOB_IO_BOUND, argc, argv); + return static_schedule_dirty_nif(env, ERTS_PSFLG_DIRTY_IO_PROC, argc, argv); } static ERL_NIF_TERM -schedule_dirty_cpu_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +static_schedule_dirty_cpu_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - return schedule_dirty_nif(env, ERL_NIF_DIRTY_JOB_CPU_BOUND, argc, argv); + return static_schedule_dirty_nif(env, ERTS_PSFLG_DIRTY_CPU_PROC, argc, argv); } #endif /* ERTS_DIRTY_SCHEDULERS */ @@ -2646,23 +2446,42 @@ execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ERL_NIF_TERM result; execution_state(env, &proc, NULL); - ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); - fp = ep->fp; - ASSERT(!env->exception_thrown); - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); + fp = ep->func; ASSERT(ep); - ep->fp = NULL; + ASSERT(!env->exception_thrown); + + fp = (NativeFunPtr) ep->func; + +#ifdef DEBUG + ep->func = ERTS_DBG_NIF_NOT_SCHED_MARKER; +#endif + result = (*fp)(env, argc, argv); - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - ASSERT(ep); - /* - * If no NIFs were scheduled by the native call via - * enif_schedule_nif(), then ep->fp will still be NULL as set above, in - * which case we need to restore the original NIF MFA. - */ - if (ep->fp == NULL) - restore_nif_mfa(proc, ep, env->exception_thrown); + + ASSERT(ep == ERTS_PROC_GET_NIF_TRAP_EXPORT(proc)); + + if (is_value(result) || proc->freason != TRAP) { + /* Done (not rescheduled)... */ + ASSERT(ep->func == ERTS_DBG_NIF_NOT_SCHED_MARKER); + if (!env->exception_thrown) + nif_export_restore(proc, ep); + else { + nif_export_cleanup_nif_mod(ep); + /* + * Restore orig info for error and clear nif + * export in handle_error() + */ + proc->freason |= EXF_RESTORE_NIF; + } + } + +#ifdef DEBUG + if (ep->func == ERTS_DBG_NIF_NOT_SCHED_MARKER) + ep->func = NULL; +#endif + return result; } @@ -2672,9 +2491,8 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, int argc, const ERL_NIF_TERM argv[]) { Process* proc; - NifExport* ep; ERL_NIF_TERM fun_name_atom, result; - int need_save, scheduler; + int scheduler; if (argc > MAX_ARG) return enif_make_badarg(env); @@ -2689,35 +2507,16 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); } - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - need_save = (ep == NULL || !ep->saved_current); - - if (flags) { + if (flags == 0) + result = schedule(env, execute_nif, fp, proc->current->module, + fun_name_atom, argc, argv); #ifdef ERTS_DIRTY_SCHEDULERS - NativeFunPtr sched_fun; - int chkflgs = (flags & (ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND)); - if (chkflgs == ERL_NIF_DIRTY_JOB_IO_BOUND) - sched_fun = schedule_dirty_io_nif; - else if (chkflgs == ERL_NIF_DIRTY_JOB_CPU_BOUND) - sched_fun = schedule_dirty_cpu_nif; - else { - result = enif_make_badarg(env); - goto done; - } - result = init_nif_sched_data(env, sched_fun, fp, need_save, argc, argv); -#else - result = enif_make_badarg(env); + else if (!(flags & ~(ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND))) + result = schedule_dirty_nif(env, flags, fp, fun_name_atom, argc, argv); #endif - goto done; - } else - result = init_nif_sched_data(env, execute_nif, fp, need_save, argc, argv); - - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - ASSERT(ep); - ep->exp.info.mfa.function = (BeamInstr) fun_name_atom; + result = enif_make_badarg(env); -done: if (scheduler < 0) erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); @@ -3416,8 +3215,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) if (f->flags) { code_ptr[3] = (BeamInstr) f->fptr; code_ptr[1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ? - (BeamInstr) schedule_dirty_io_nif : - (BeamInstr) schedule_dirty_cpu_nif; + (BeamInstr) static_schedule_dirty_io_nif : + (BeamInstr) static_schedule_dirty_cpu_nif; } else #endif diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 6928b282ee..955b98b35b 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -48,6 +48,7 @@ #include "erl_bif_unique.h" #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" +#include "erl_nfunc_sched.h" #define ERTS_CHECK_TIME_REDS CONTEXT_REDS #define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0) @@ -193,12 +194,6 @@ static ErtsAuxWorkData *aux_thread_aux_work_data; #define ERTS_SCHDLR_SSPND_CHNG_ONLN (((erts_aint32_t) 1) << 2) #define ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN (((erts_aint32_t) 1) << 3) -typedef enum { - ERTS_SCHED_NORMAL, - ERTS_SCHED_DIRTY_CPU, - ERTS_SCHED_DIRTY_IO -} ErtsSchedType; - typedef struct { int ongoing; ErtsProcList *blckrs; @@ -10102,9 +10097,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) goto sunlock_sched_out_proc; } - ASSERT((state & ERTS_PSFLG_DIRTY_ACTIVE_SYS) - || *p->i == (BeamInstr) em_call_nif); - ASSERT(rq == ERTS_DIRTY_CPU_RUNQ ? (state & (ERTS_PSFLG_DIRTY_CPU_PROC | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) @@ -11976,7 +11968,6 @@ delete_process(Process* p) ErtsPSD *psd; struct saved_calls *scb; process_breakpoint_time_t *pbt; - void *nif_export; VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id)); VERBOSE(DEBUG_SHCOPY, ("[pid=%T] delete process: %p %p %p %p\n", p->common.id, @@ -11993,9 +11984,7 @@ delete_process(Process* p) if (pbt) erts_free(ERTS_ALC_T_BPD, (void *) pbt); - nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL); - if (nif_export) - erts_destroy_nif_export(nif_export); + erts_destroy_nif_export(p); /* Cleanup psd */ @@ -13564,6 +13553,9 @@ erts_dbg_check_halloc_lock(Process *p) ErtsSchedulerData *esdp; if (ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)) return 1; + if ((p->static_flags & ERTS_STC_FLG_SHADOW_PROC) + && ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())) + return 1; if (p->common.id == ERTS_INVALID_PID) return 1; esdp = erts_proc_sched_data(p); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 7193b7d1db..58a50f8cf4 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -838,8 +838,8 @@ typedef struct { #define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN -#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ((ErtsProcLocks) 0) -#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ((ErtsProcLocks) 0) +#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN typedef struct { ErtsProcLocks get_locks; @@ -1582,10 +1582,6 @@ Uint64 erts_get_proc_interval(void); Uint64 erts_ensure_later_proc_interval(Uint64); Uint64 erts_step_proc_interval(void); -int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */ -void erts_destroy_nif_export(void *); /* see erl_nif.c */ -int erts_check_nif_export_in_area(Process *p, char *start, Uint size); - ErtsProcList *erts_proclist_create(Process *); ErtsProcList *erts_proclist_copy(ErtsProcList *); void erts_proclist_destroy(ErtsProcList *); diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h index e431c3051b..99a8ff6bad 100644 --- a/erts/emulator/beam/error.h +++ b/erts/emulator/beam/error.h @@ -38,15 +38,12 @@ * Some of these have convenient aliases, like BADARG and BADARITH. */ -/* - * Bits 0-1 index the 'exception class tag' table. - */ -#define EXC_CLASSBITS 3 -#define GET_EXC_CLASS(x) ((x) & EXC_CLASSBITS) - /* * Exception class tags (indices into the 'exception_tag' array) */ +#define EXTAG_OFFSET 0 +#define EXTAG_BITS 2 + #define EXTAG_ERROR 0 #define EXTAG_EXIT 1 #define EXTAG_THROWN 2 @@ -54,20 +51,31 @@ #define NUMBER_EXC_TAGS 3 /* The number of exception class tags */ /* - * Exit code flags (bits 2-7) + * Index to the 'exception class tag' table. + */ +#define EXC_CLASSBITS ((1<> 8) +#define EXC_OFFSET (EXF_OFFSET+EXF_BITS) +#define EXC_BITS 5 + +#define EXC_INDEXBITS (((1<<(EXC_BITS+EXC_OFFSET))-1) \ + & ~((1<<(EXC_OFFSET))-1)) + +#define GET_EXC_INDEX(x) (((x) & EXC_INDEXBITS) >> EXC_OFFSET) /* * Exit codes used for raising a fresh exception. The primary exceptions @@ -107,46 +120,46 @@ /* Error with given arglist term * (exit reason in p->fvalue) */ -#define EXC_NORMAL ((1 << 8) | EXC_EXIT) +#define EXC_NORMAL ((1 << EXC_OFFSET) | EXC_EXIT) /* Normal exit (reason 'normal') */ -#define EXC_INTERNAL_ERROR ((2 << 8) | EXC_ERROR | EXF_PANIC) +#define EXC_INTERNAL_ERROR ((2 << EXC_OFFSET) | EXC_ERROR | EXF_PANIC) /* Things that shouldn't happen */ -#define EXC_BADARG ((3 << 8) | EXC_ERROR) +#define EXC_BADARG ((3 << EXC_OFFSET) | EXC_ERROR) /* Bad argument to a BIF */ -#define EXC_BADARITH ((4 << 8) | EXC_ERROR) +#define EXC_BADARITH ((4 << EXC_OFFSET) | EXC_ERROR) /* Bad arithmetic */ -#define EXC_BADMATCH ((5 << 8) | EXC_ERROR) +#define EXC_BADMATCH ((5 << EXC_OFFSET) | EXC_ERROR) /* Bad match in function body */ -#define EXC_FUNCTION_CLAUSE ((6 << 8) | EXC_ERROR) +#define EXC_FUNCTION_CLAUSE ((6 << EXC_OFFSET) | EXC_ERROR) /* No matching function head */ -#define EXC_CASE_CLAUSE ((7 << 8) | EXC_ERROR) +#define EXC_CASE_CLAUSE ((7 << EXC_OFFSET) | EXC_ERROR) /* No matching case clause */ -#define EXC_IF_CLAUSE ((8 << 8) | EXC_ERROR) +#define EXC_IF_CLAUSE ((8 << EXC_OFFSET) | EXC_ERROR) /* No matching if clause */ -#define EXC_UNDEF ((9 << 8) | EXC_ERROR) +#define EXC_UNDEF ((9 << EXC_OFFSET) | EXC_ERROR) /* No farity that matches */ -#define EXC_BADFUN ((10 << 8) | EXC_ERROR) +#define EXC_BADFUN ((10 << EXC_OFFSET) | EXC_ERROR) /* Not an existing fun */ -#define EXC_BADARITY ((11 << 8) | EXC_ERROR) +#define EXC_BADARITY ((11 << EXC_OFFSET) | EXC_ERROR) /* Attempt to call fun with * wrong number of arguments. */ -#define EXC_TIMEOUT_VALUE ((12 << 8) | EXC_ERROR) +#define EXC_TIMEOUT_VALUE ((12 << EXC_OFFSET) | EXC_ERROR) /* Bad time out value */ -#define EXC_NOPROC ((13 << 8) | EXC_ERROR) +#define EXC_NOPROC ((13 << EXC_OFFSET) | EXC_ERROR) /* No process or port */ -#define EXC_NOTALIVE ((14 << 8) | EXC_ERROR) +#define EXC_NOTALIVE ((14 << EXC_OFFSET) | EXC_ERROR) /* Not distributed */ -#define EXC_SYSTEM_LIMIT ((15 << 8) | EXC_ERROR) +#define EXC_SYSTEM_LIMIT ((15 << EXC_OFFSET) | EXC_ERROR) /* Ran out of something */ -#define EXC_TRY_CLAUSE ((16 << 8) | EXC_ERROR) +#define EXC_TRY_CLAUSE ((16 << EXC_OFFSET) | EXC_ERROR) /* No matching try clause */ -#define EXC_NOTSUP ((17 << 8) | EXC_ERROR) +#define EXC_NOTSUP ((17 << EXC_OFFSET) | EXC_ERROR) /* Not supported */ -#define EXC_BADMAP ((18 << 8) | EXC_ERROR) +#define EXC_BADMAP ((18 << EXC_OFFSET) | EXC_ERROR) /* Bad map */ -#define EXC_BADKEY ((19 << 8) | EXC_ERROR) +#define EXC_BADKEY ((19 << EXC_OFFSET) | EXC_ERROR) /* Bad key in map */ #define NUMBER_EXIT_CODES 20 /* The number of exit code indices */ @@ -154,7 +167,7 @@ /* * Internal pseudo-error codes. */ -#define TRAP (1 << 8) /* BIF Trap to erlang code */ +#define TRAP (1 << EXC_OFFSET) /* BIF Trap to erlang code */ /* * Aliases for some common exit codes. @@ -201,6 +214,7 @@ struct StackTrace { BeamInstr* pc; ErtsCodeMFA* current; int depth; /* number of saved pointers in trace[] */ + ErtsCodeMFA mfa; /* in case we need to make a copy of mfa */ BeamInstr *trace[1]; /* varying size - must be last in struct */ }; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 2b2f3c5cdc..d5ca3b04eb 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -61,12 +61,6 @@ struct enif_environment_t /* ErlNifEnv */ extern void erts_pre_nif(struct enif_environment_t*, Process*, struct erl_module_nif*, Process* tracee); extern void erts_post_nif(struct enif_environment_t* env); -#ifdef ERTS_DIRTY_SCHEDULERS -extern void erts_pre_dirty_nif(ErtsSchedulerData *, - struct enif_environment_t*, Process*, - struct erl_module_nif*); -extern void erts_post_dirty_nif(struct enif_environment_t* env); -#endif extern Eterm erts_nif_taints(Process* p); extern void erts_print_nif_taints(fmtfn_t to, void* to_arg); void erts_unload_nif(struct erl_module_nif* nif); @@ -78,6 +72,12 @@ extern Eterm erts_nif_call_function(Process *p, Process *tracee, struct enif_func_t *, int argc, Eterm *argv); +#ifdef ERTS_DIRTY_SCHEDULERS +int erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, + BeamInstr *I, Eterm *reg); +#endif /* ERTS_DIRTY_SCHEDULERS */ + + /* Driver handle (wrapper for old plain handle) */ #define ERL_DE_OK 0 #define ERL_DE_UNLOAD 1 @@ -993,7 +993,7 @@ void erts_queue_monitor_message(Process *, Eterm, Eterm); void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, - Eterm (*bif)(Process*,Eterm*)); + Eterm (*bif)(Process*, Eterm*, BeamInstr*)); void erts_init_bif(void); Eterm erl_send(Process *p, Eterm to, Eterm msg); diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index ec502d5a78..3fa48da1ec 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -56,6 +56,8 @@ #ifdef HIPE # include "hipe_mode_switch.h" #endif +#define ERTS_WANT_NFUNC_SCHED_INTERNALS__ +#include "erl_nfunc_sched.h" #undef M_TRIM_THRESHOLD #undef M_TOP_PAD -- cgit v1.2.3 From 95ec5d385cfba23c770d946871c0197bf374ff3c Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 28 Oct 2016 18:34:14 +0200 Subject: Optimize handling of BIF errors --- erts/emulator/beam/beam_emu.c | 139 ++++++++++++++++++----------------- erts/emulator/beam/bif.c | 25 +++---- erts/emulator/beam/erl_nfunc_sched.c | 4 +- erts/emulator/beam/erl_nfunc_sched.h | 10 +-- erts/emulator/beam/erl_nif.c | 4 - erts/emulator/beam/error.h | 1 - 6 files changed, 87 insertions(+), 96 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index befd2989f9..cfdf61eed0 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1057,9 +1057,10 @@ do { \ * The following functions are called directly by process_main(). * Don't inline them. */ -static BifFunction translate_gc_bif(void* gcf) NOINLINE; +static ErtsCodeMFA *ubif2mfa(void* uf) NOINLINE; +static ErtsCodeMFA *gcbif2mfa(void* gcf) NOINLINE; static BeamInstr* handle_error(Process* c_p, BeamInstr* pc, - Eterm* reg, BifFunction bf) NOINLINE; + Eterm* reg, ErtsCodeMFA* bif_mfa) NOINLINE; static BeamInstr* call_error_handler(Process* p, ErtsCodeMFA* mfa, Eterm* reg, Eterm func) NOINLINE; static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity, @@ -1088,7 +1089,7 @@ static BeamInstr* next_catch(Process* c_p, Eterm *reg); static void terminate_proc(Process* c_p, Eterm Value); static Eterm add_stacktrace(Process* c_p, Eterm Value, Eterm exc); static void save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, - BifFunction bf, Eterm args); + ErtsCodeMFA *bif_mfa, Eterm args); static struct StackTrace * get_trace_from_exc(Eterm exc); static Eterm make_arglist(Process* c_p, Eterm* reg, int a); @@ -2640,7 +2641,7 @@ do { \ } reg[0] = tmp_reg[0]; SWAPOUT; - I = handle_error(c_p, I, reg, bf); + I = handle_error(c_p, I, reg, ubif2mfa((void *) bf)); goto post_error_handling; } @@ -2676,7 +2677,7 @@ do { \ Goto(*I); } x(0) = x(live); - I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); + I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf)); goto post_error_handling; } @@ -2721,7 +2722,7 @@ do { \ live--; x(0) = x(live); x(1) = x(live+1); - I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); + I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf)); goto post_error_handling; } @@ -2767,7 +2768,7 @@ do { \ x(0) = x(live); x(1) = x(live+1); x(2) = x(live+2); - I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); + I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf)); goto post_error_handling; } @@ -2829,7 +2830,7 @@ do { \ reg[0] = tmp_reg[0]; reg[1] = tmp_reg[1]; SWAPOUT; - I = handle_error(c_p, I, reg, bf); + I = handle_error(c_p, I, reg, ubif2mfa((void *) bf)); goto post_error_handling; } @@ -2906,7 +2907,7 @@ do { \ * Error handling. SWAPOUT is not needed because it was done above. */ ASSERT(c_p->stop == E); - I = handle_error(c_p, I, reg, bf); + I = handle_error(c_p, I, reg, &export->info.mfa); goto post_error_handling; } @@ -3210,7 +3211,7 @@ do { \ SET_I(next); Dispatch(); } - I = handle_error(c_p, I, reg, apply_3); + I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); goto post_error_handling; } @@ -3225,7 +3226,7 @@ do { \ SET_I(next); Dispatch(); } - I = handle_error(c_p, I, reg, apply_3); + I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); goto post_error_handling; } @@ -3238,7 +3239,7 @@ do { \ SET_I(next); Dispatch(); } - I = handle_error(c_p, I, reg, apply_3); + I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); goto post_error_handling; } @@ -3253,7 +3254,7 @@ do { \ SET_I(next); Dispatch(); } - I = handle_error(c_p, I, reg, apply_3); + I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); goto post_error_handling; } @@ -3269,7 +3270,7 @@ do { \ SET_I(next); Dispatch(); } - I = handle_error(c_p, I, reg, apply_3); + I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); goto post_error_handling; } @@ -3690,7 +3691,7 @@ do { \ } Dispatch(); } - I = handle_error(c_p, c_p->cp, reg, vbf); + I = handle_error(c_p, c_p->cp, reg, c_p->current); goto post_error_handling; } } @@ -5045,7 +5046,7 @@ do { \ goto do_schedule; } else { HEAVY_SWAPIN; - I = handle_error(c_p, I, reg, hibernate_3); + I = handle_error(c_p, I, reg, &bif_export[BIF_hibernate_3]->info.mfa); goto post_error_handling; } } @@ -5437,17 +5438,28 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) #endif /* ERTS_DIRTY_SCHEDULERS */ } -static BifFunction -translate_gc_bif(void* gcf) +static ErtsCodeMFA * +gcbif2mfa(void* gcf) { - const ErtsGcBif* p; - - for (p = erts_gc_bifs; p->bif != 0; p++) { - if (p->gc_bif == gcf) { - return p->bif; - } + int i; + for (i = 0; erts_gc_bifs[i].bif; i++) { + if (erts_gc_bifs[i].gc_bif == gcf) + return &bif_export[erts_gc_bifs[i].exp_ix]->info.mfa; } erts_exit(ERTS_ERROR_EXIT, "bad gc bif"); + return NULL; +} + +static ErtsCodeMFA * +ubif2mfa(void* uf) +{ + int i; + for (i = 0; erts_u_bifs[i].bif; i++) { + if (erts_u_bifs[i].bif == uf) + return &bif_export[erts_u_bifs[i].exp_ix]->info.mfa; + } + erts_exit(ERTS_ERROR_EXIT, "bad u bif"); + return NULL; } /* @@ -5506,7 +5518,7 @@ Eterm error_atom[NUMBER_EXIT_CODES] = { */ static BeamInstr* -handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) +handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, ErtsCodeMFA *bif_mfa) { Eterm* hp; Eterm Value = c_p->fvalue; @@ -5514,9 +5526,16 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */ - if (c_p->freason & EXF_RESTORE_NIF) { - erts_nif_export_restore_error(c_p, &pc, reg, &bf); + if (c_p->freason & EXF_RESTORE_NIF) + erts_nif_export_restore_error(c_p, &pc, reg, &bif_mfa); + +#ifdef DEBUG + if (bif_mfa) { + /* Verify that bif_mfa does not point into our nif export */ + NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + ASSERT(!nep || !ErtsInArea(bif_mfa, (char *)nep, sizeof(NifExport))); } +#endif c_p->i = pc; /* In case we call erts_exit(). */ @@ -5542,7 +5561,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) * more modular. */ if (c_p->freason & EXF_SAVETRACE) { - save_stacktrace(c_p, pc, reg, bf, Args); + save_stacktrace(c_p, pc, reg, bif_mfa, Args); } /* @@ -5817,23 +5836,12 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) { */ static void -save_stacktrace_current(struct StackTrace *s, Process *c_p, ErtsCodeMFA *mfa) -{ - NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); - if (!nep || &nep->exp.info.mfa != mfa) - s->current = mfa; - else { - s->mfa = *mfa; - s->current = &s->mfa; - } -} - -static void -save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, - Eterm args) { +save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, + ErtsCodeMFA *bif_mfa, Eterm args) { struct StackTrace* s; int sz; int depth = erts_backtrace_depth; /* max depth (never negative) */ + if (depth > 0) { /* There will always be a current function */ depth --; @@ -5850,33 +5858,29 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, /* * If the failure was in a BIF other than 'error/1', 'error/2', - * 'exit/1' or 'throw/1', find the bif-table index and save the - * argument registers by consing up an arglist. + * 'exit/1' or 'throw/1', save BIF-MFA and save the argument + * registers by consing up an arglist. */ - if (bf != NULL && bf != error_1 && bf != error_2 && bf != exit_1 - && bf != throw_1 && bf != wrap_error_1 && bf != wrap_error_2 - && bf != wrap_exit_1 && bf != wrap_throw_1) { - int i; - int a; - for (i = 0; i < BIF_SIZE; i++) { - if (bf == bif_table[i].f || bf == bif_table[i].traced) { - save_stacktrace_current(s, c_p, &bif_export[i]->info.mfa); + if (bif_mfa) { + if (bif_mfa->module == am_erlang) { + switch (bif_mfa->function) { + case am_error: + if (bif_mfa->arity == 1 || bif_mfa->arity == 2) + goto non_bif_stacktrace; + break; + case am_exit: + if (bif_mfa->arity == 1) + goto non_bif_stacktrace; + break; + case am_throw: + if (bif_mfa->arity == 1) + goto non_bif_stacktrace; + break; + default: break; } } - if (i >= BIF_SIZE) { - /* - * The Bif does not really exist (no BIF entry). It is a - * TRAP and traps are called through apply_bif, which also - * sets c_p->current (luckily). - * OR it is a NIF called by call_nif where current is also set. - */ - ASSERT(c_p->current); - save_stacktrace_current(s, c_p, c_p->current); - } - - a = s->current->arity; - + s->current = bif_mfa; /* Save first stack entry */ ASSERT(pc); if (depth > 0) { @@ -5889,11 +5893,12 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, depth--; } s->pc = NULL; - args = make_arglist(c_p, reg, a); /* Overwrite CAR(c_p->ftrace) */ + args = make_arglist(c_p, reg, bif_mfa->arity); /* Overwrite CAR(c_p->ftrace) */ } else { - save_stacktrace_current(s, c_p, c_p->current); + non_bif_stacktrace: + s->current = c_p->current; /* * For a function_clause error, the arguments are in the beam * registers, c_p->cp is valid, and c_p->current is set. diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 7bcb7c196d..5f0564474f 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -5027,17 +5027,16 @@ void erts_init_bif(void) #define ERTS_SCHED_BIF_TRAP_MARKER ((void *) (UWord) 1) -static void +static ERTS_INLINE void schedule(Process *c_p, Process *dirty_shadow_proc, - ErtsCodeMFA *mfa, BifFunction *nif, BeamInstr *pc, + ErtsCodeMFA *mfa, BeamInstr *pc, ErtsBifFunc dfunc, void *ifunc, Eterm module, Eterm function, int argc, Eterm *argv) { ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); (void) erts_nif_export_schedule(c_p, dirty_shadow_proc, - mfa, nif, pc, - (BeamInstr) em_apply_bif, + mfa, pc, (BeamInstr) em_apply_bif, dfunc, ifunc, module, function, argc, argv); @@ -5120,11 +5119,10 @@ erts_schedule_bif(Process *proc, if (!ERTS_PROC_IS_EXITING(c_p)) { Export *exp; - BifFunction obif, dbif, ibif; + BifFunction dbif, ibif; BeamInstr *pc; /* - * obif - original bif * dbif - direct bif * ibif - indirect bif */ @@ -5160,14 +5158,12 @@ erts_schedule_bif(Process *proc, if (i == NULL) { ERTS_INTERNAL_ERROR("Missing instruction pointer"); - obif = NULL; } #ifdef HIPE else if (proc->flags & F_HIPE_MODE) { /* Pointer to bif export in i */ exp = (Export *) i; pc = c_p->cp; - obif = (BifFunction) exp->beam[1]; mfa = &exp->info.mfa; } #endif @@ -5175,19 +5171,16 @@ erts_schedule_bif(Process *proc, /* Pointer to bif export in i+1 */ exp = (Export *) i[1]; pc = i; - obif = (BifFunction) exp->beam[1]; mfa = &exp->info.mfa; } else if (em_apply_bif == (BeamInstr *) *i) { /* Pointer to bif in i+1, and mfa in i-3 */ - obif = (BifFunction) i[1]; pc = c_p->cp; mfa = erts_code_to_codemfa(i); } else { ERTS_INTERNAL_ERROR("erts_schedule_bif() called " "from unexpected instruction"); - obif = NULL; } ASSERT(bif); @@ -5197,8 +5190,8 @@ erts_schedule_bif(Process *proc, argc = (int) mfa->arity; } - schedule(c_p, dirty_shadow_proc, mfa, obif, - pc, dbif, ibif, mod, func, argc, argv); + schedule(c_p, dirty_shadow_proc, mfa, pc, dbif, ibif, + mod, func, argc, argv); } if (dirty_shadow_proc) @@ -5295,14 +5288,14 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * if (!exiting) { if (is_value(result)) - schedule(c_p, dirty_shadow_proc, NULL, NULL, NULL, dirty_bif_result, + schedule(c_p, dirty_shadow_proc, NULL, NULL, dirty_bif_result, NULL, am_erts_internal, am_dirty_bif_result, 1, &result); else if (dirty_shadow_proc->freason != TRAP) { Eterm argv[2]; ASSERT(dirty_shadow_proc->freason <= MAX_SMALL); argv[0] = make_small(dirty_shadow_proc->freason); argv[1] = dirty_shadow_proc->fvalue; - schedule(c_p, dirty_shadow_proc, NULL, NULL, NULL, + schedule(c_p, dirty_shadow_proc, NULL, NULL, dirty_bif_exception, NULL, am_erts_internal, am_dirty_bif_exception, 2, argv); } @@ -5310,7 +5303,7 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * /* Dirty BIF did an ordinary trap... */ ASSERT(!(erts_smp_atomic32_read_nob(&c_p->state) & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC))); - schedule(c_p, dirty_shadow_proc, NULL, NULL, NULL, + schedule(c_p, dirty_shadow_proc, NULL, NULL, dirty_bif_trap, (void *) dirty_shadow_proc->i, am_erts_internal, am_dirty_bif_trap, dirty_shadow_proc->arity, reg); diff --git a/erts/emulator/beam/erl_nfunc_sched.c b/erts/emulator/beam/erl_nfunc_sched.c index 0333545255..bc3fd83d7a 100644 --- a/erts/emulator/beam/erl_nfunc_sched.c +++ b/erts/emulator/beam/erl_nfunc_sched.c @@ -63,7 +63,7 @@ erts_destroy_nif_export(Process *p) NifExport * erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, - ErtsCodeMFA *mfa, void *nif, BeamInstr *pc, + ErtsCodeMFA *mfa, BeamInstr *pc, BeamInstr instr, void *dfunc, void *ifunc, Eterm mod, Eterm func, @@ -111,8 +111,6 @@ erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, nep->argv[i] = reg[i]; nep->pc = pc; nep->cp = c_p->cp; - ASSERT(nif); - nep->nif = nif; nep->mfa = mfa; nep->current = c_p->current; ASSERT(argc >= 0); diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h index 72de7f0eb5..d7eccb28ba 100644 --- a/erts/emulator/beam/erl_nfunc_sched.h +++ b/erts/emulator/beam/erl_nfunc_sched.h @@ -52,7 +52,7 @@ typedef struct { NifExport *erts_new_proc_nif_export(Process *c_p, int argc); void erts_destroy_nif_export(Process *p); NifExport *erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, - ErtsCodeMFA *mfa, void *nif, BeamInstr *pc, + ErtsCodeMFA *mfa, BeamInstr *pc, BeamInstr instr, void *dfunc, void *ifunc, Eterm mod, Eterm func, @@ -65,7 +65,7 @@ ERTS_GLB_INLINE int erts_check_nif_export_in_area(Process *p, char *start, Uint size); ERTS_GLB_INLINE void erts_nif_export_restore(Process *c_p, NifExport *ep); ERTS_GLB_INLINE void erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, - Eterm *reg, void **nif); + Eterm *reg, ErtsCodeMFA **nif_mfa); ERTS_GLB_INLINE Process *erts_proc_shadow2real(Process *c_p); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -132,8 +132,8 @@ erts_nif_export_restore(Process *c_p, NifExport *ep) } ERTS_GLB_INLINE void -erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, Eterm *reg, - void **nif) +erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, + Eterm *reg, ErtsCodeMFA **nif_mfa) { NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); int ix; @@ -141,7 +141,7 @@ erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, Eterm *reg, ASSERT(nep); *pc = nep->pc; c_p->cp = nep->cp; - *nif = nep->nif; + *nif_mfa = nep->mfa; for (ix = 0; ix < nep->argc; ix++) reg[ix] = nep->argv[ix]; erts_nif_export_restore(c_p, nep); diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index ee4b22f7ca..fd756692f9 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -266,7 +266,6 @@ static ERTS_INLINE ERL_NIF_TERM schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp, Eterm mod, Eterm func_name, int argc, const ERL_NIF_TERM argv[]) { - Export *exp; NifExport *ep; Process *c_p, *dirty_shadow_proc; @@ -278,11 +277,8 @@ schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp, ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); - exp = ErtsContainerStruct(c_p->current, Export, info.mfa); - ep = erts_nif_export_schedule(c_p, dirty_shadow_proc, c_p->current, - (BifFunction) exp->beam[1], c_p->cp, (BeamInstr) em_call_nif, direct_fp, indirect_fp, diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h index 99a8ff6bad..64c08b1570 100644 --- a/erts/emulator/beam/error.h +++ b/erts/emulator/beam/error.h @@ -214,7 +214,6 @@ struct StackTrace { BeamInstr* pc; ErtsCodeMFA* current; int depth; /* number of saved pointers in trace[] */ - ErtsCodeMFA mfa; /* in case we need to make a copy of mfa */ BeamInstr *trace[1]; /* varying size - must be last in struct */ }; -- cgit v1.2.3 From 04e119e22a68d686b9e8df17c0a4836c4a5b91ea Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 8 Nov 2016 09:51:03 +0100 Subject: Return and exception trace for nif-export scheduled BIFs The support is somewhat primitive, since it is determined at call time if trace on return or exception should be sent. --- erts/emulator/beam/beam_bp.c | 25 ++++++++++++++++++ erts/emulator/beam/bif.c | 8 +++--- erts/emulator/beam/erl_alloc.c | 7 +++++ erts/emulator/beam/erl_alloc.types | 3 ++- erts/emulator/beam/erl_nfunc_sched.c | 40 ++++++++++++++++++++++++++++- erts/emulator/beam/erl_nfunc_sched.h | 50 +++++++++++++++++++++++++++++++++--- erts/emulator/beam/erl_nif.c | 8 +++--- erts/emulator/beam/erl_trace.h | 5 ++++ 8 files changed, 133 insertions(+), 13 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 455362f5ae..27329a339e 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -32,6 +32,7 @@ #include "erl_binary.h" #include "beam_bp.h" #include "erl_term.h" +#include "erl_nfunc_sched.h" /* ************************************************************************* ** Macros @@ -805,6 +806,30 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) result = func(p, args, I); + if (erts_nif_export_check_save_trace(p, result, + applying, ep, + cp, flags, + flags_meta, I, + meta_tracer)) { + /* + * erts_bif_trace_epilogue() will be called + * later when appropriate via the NIF export + * scheduling functionality... + */ + return result; + } + + return erts_bif_trace_epilogue(p, result, applying, ep, cp, + flags, flags_meta, I, + meta_tracer); +} + +Eterm +erts_bif_trace_epilogue(Process *p, Eterm result, int applying, + Export* ep, BeamInstr *cp, Uint32 flags, + Uint32 flags_meta, BeamInstr* I, + ErtsTracer meta_tracer) +{ if (applying && (flags & MATCH_SET_RETURN_TO_TRACE)) { BeamInstr i_return_trace = beam_return_trace[0]; BeamInstr i_return_to_trace = beam_return_to_trace[0]; diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 5f0564474f..cae346267c 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -5047,7 +5047,7 @@ schedule(Process *c_p, Process *dirty_shadow_proc, static BIF_RETTYPE dirty_bif_result(BIF_ALIST_1) { NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(BIF_P); - erts_nif_export_restore(BIF_P, nep); + erts_nif_export_restore(BIF_P, nep, BIF_ARG_1); BIF_RET(BIF_ARG_1); } @@ -5062,7 +5062,7 @@ static BIF_RETTYPE dirty_bif_trap(BIF_ALIST) ASSERT(BIF_P->arity == nep->exp.info.mfa.arity); - erts_nif_export_restore(BIF_P, nep); + erts_nif_export_restore(BIF_P, nep, THE_NON_VALUE); BIF_P->i = (BeamInstr *) nep->func; BIF_P->freason = TRAP; @@ -5216,12 +5216,12 @@ call_bif(Process *c_p, Eterm *reg, BeamInstr *I) ret = (*bif)(c_p, reg, I); if (is_value(ret)) - erts_nif_export_restore(c_p, nep); + erts_nif_export_restore(c_p, nep, ret); else if (c_p->freason != TRAP) c_p->freason |= EXF_RESTORE_NIF; /* restore in handle_error() */ else if (nep->func == ERTS_SCHED_BIF_TRAP_MARKER) { /* BIF did an ordinary trap... */ - erts_nif_export_restore(c_p, nep); + erts_nif_export_restore(c_p, nep, ret); } /* else: * BIF rescheduled itself using erts_schedule_bif(). diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 40a45c961f..4d990a9c56 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -44,6 +44,7 @@ #include "erl_hl_timer.h" #include "erl_cpu_topology.h" #include "erl_thr_queue.h" +#include "erl_nfunc_sched.h" #if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE) #include "erl_check_io.h" #endif @@ -679,6 +680,8 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_ABIF_TIMER)] = erts_timer_type_size(ERTS_ALC_T_ABIF_TIMER); #endif + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_EXP_TRACE)] + = sizeof(NifExportTrace); #ifdef HARD_DEBUG hdbg_init(); @@ -2437,6 +2440,10 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) fi, ERTS_ALC_T_ABIF_TIMER); #endif + add_fix_values(&size.processes, + &size.processes_used, + fi, + ERTS_ALC_T_NIF_EXP_TRACE); } if (want.atom || want.atom_used) { diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 6e8710eb8a..70eca5b49c 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -376,7 +376,8 @@ type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data type LL_TEMP_TERM LONG_LIVED SYSTEM ll_temp_term -type NIF_TRAP_EXPORT STANDARD CODE nif_trap_export_entry +type NIF_TRAP_EXPORT STANDARD PROCESSES nif_trap_export_entry +type NIF_EXP_TRACE FIXED_SIZE PROCESSES nif_export_trace type EXPORT LONG_LIVED CODE export_entry type MONITOR_SH FIXED_SIZE PROCESSES monitor_sh type NLINK_SH FIXED_SIZE PROCESSES nlink_sh diff --git a/erts/emulator/beam/erl_nfunc_sched.c b/erts/emulator/beam/erl_nfunc_sched.c index bc3fd83d7a..1bebc1eda4 100644 --- a/erts/emulator/beam/erl_nfunc_sched.c +++ b/erts/emulator/beam/erl_nfunc_sched.c @@ -28,6 +28,7 @@ #include "erl_process.h" #include "bif.h" #include "erl_nfunc_sched.h" +#include "erl_trace.h" NifExport * erts_new_proc_nif_export(Process *c_p, int argc) @@ -44,9 +45,12 @@ erts_new_proc_nif_export(Process *c_p, int argc) nep->argc = -1; /* unused marker */ nep->argv_size = argc; + nep->trace = NULL; old_nep = ERTS_PROC_SET_NIF_TRAP_EXPORT(c_p, nep); - if (old_nep) + if (old_nep) { + ASSERT(!nep->trace); erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, old_nep); + } return nep; } @@ -61,6 +65,40 @@ erts_destroy_nif_export(Process *p) } } +void +erts_nif_export_save_trace(Process *c_p, NifExport *nep, int applying, + Export* ep, BeamInstr *cp, Uint32 flags, + Uint32 flags_meta, BeamInstr* I, + ErtsTracer meta_tracer) +{ + NifExportTrace *netp; + ASSERT(nep && nep->argc >= 0); + ASSERT(!nep->trace); + netp = erts_alloc(ERTS_ALC_T_NIF_EXP_TRACE, + sizeof(NifExportTrace)); + netp->applying = applying; + netp->ep = ep; + netp->cp = cp; + netp->flags = flags; + netp->flags_meta = flags_meta; + netp->I = I; + netp->meta_tracer = NIL; + erts_tracer_update(&netp->meta_tracer, meta_tracer); + nep->trace = netp; +} + +void +erts_nif_export_restore_trace(Process *c_p, Eterm result, NifExport *nep) +{ + NifExportTrace *netp = nep->trace; + nep->trace = NULL; + erts_bif_trace_epilogue(c_p, result, netp->applying, netp->ep, + netp->cp, netp->flags, netp->flags_meta, + netp->I, netp->meta_tracer); + erts_tracer_update(&netp->meta_tracer, NIL); + erts_free(ERTS_ALC_T_NIF_EXP_TRACE, netp); +} + NifExport * erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, ErtsCodeMFA *mfa, BeamInstr *pc, diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h index d7eccb28ba..5c98957cb7 100644 --- a/erts/emulator/beam/erl_nfunc_sched.h +++ b/erts/emulator/beam/erl_nfunc_sched.h @@ -23,6 +23,17 @@ #include "erl_process.h" #include "bif.h" +#include "error.h" + +typedef struct { + int applying; + Export* ep; + BeamInstr *cp; + Uint32 flags; + Uint32 flags_meta; + BeamInstr* I; + ErtsTracer meta_tracer; +} NifExportTrace; /* * NIF exports need a few more items than the Export struct provides, @@ -39,6 +50,7 @@ typedef struct { struct erl_module_nif* m; /* NIF module, or NULL if BIF */ void *func; /* Indirect NIF or BIF to execute (may be unused) */ ErtsCodeMFA *current;/* Current as set when originally called */ + NifExportTrace *trace; /* --- The following is only used on error --- */ BeamInstr *pc; /* Program counter */ BeamInstr *cp; /* Continuation pointer */ @@ -50,6 +62,11 @@ typedef struct { } NifExport; NifExport *erts_new_proc_nif_export(Process *c_p, int argc); +void erts_nif_export_save_trace(Process *c_p, NifExport *nep, int applying, + Export* ep, BeamInstr *cp, Uint32 flags, + Uint32 flags_meta, BeamInstr* I, + ErtsTracer meta_tracer); +void erts_nif_export_restore_trace(Process *c_p, Eterm result, NifExport *nep); void erts_destroy_nif_export(Process *p); NifExport *erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, ErtsCodeMFA *mfa, BeamInstr *pc, @@ -63,9 +80,15 @@ ERTS_GLB_INLINE int erts_setup_nif_export_rootset(Process* proc, Eterm** objv, Uint* nobj); ERTS_GLB_INLINE int erts_check_nif_export_in_area(Process *p, char *start, Uint size); -ERTS_GLB_INLINE void erts_nif_export_restore(Process *c_p, NifExport *ep); +ERTS_GLB_INLINE void erts_nif_export_restore(Process *c_p, NifExport *ep, + Eterm result); ERTS_GLB_INLINE void erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, Eterm *reg, ErtsCodeMFA **nif_mfa); +ERTS_GLB_INLINE int erts_nif_export_check_save_trace(Process *c_p, Eterm result, + int applying, Export* ep, + BeamInstr *cp, Uint32 flags, + Uint32 flags_meta, BeamInstr* I, + ErtsTracer meta_tracer); ERTS_GLB_INLINE Process *erts_proc_shadow2real(Process *c_p); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -119,7 +142,7 @@ erts_check_nif_export_in_area(Process *p, char *start, Uint size) } ERTS_GLB_INLINE void -erts_nif_export_restore(Process *c_p, NifExport *ep) +erts_nif_export_restore(Process *c_p, NifExport *ep, Eterm result) { ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); ERTS_SMP_LC_ASSERT(!(c_p->static_flags @@ -129,6 +152,8 @@ erts_nif_export_restore(Process *c_p, NifExport *ep) c_p->current = ep->current; ep->argc = -1; /* Unused nif-export marker... */ + if (ep->trace) + erts_nif_export_restore_trace(c_p, result, ep); } ERTS_GLB_INLINE void @@ -144,7 +169,26 @@ erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, *nif_mfa = nep->mfa; for (ix = 0; ix < nep->argc; ix++) reg[ix] = nep->argv[ix]; - erts_nif_export_restore(c_p, nep); + erts_nif_export_restore(c_p, nep, THE_NON_VALUE); +} + +ERTS_GLB_INLINE int +erts_nif_export_check_save_trace(Process *c_p, Eterm result, + int applying, Export* ep, + BeamInstr *cp, Uint32 flags, + Uint32 flags_meta, BeamInstr* I, + ErtsTracer meta_tracer) +{ + if (is_non_value(result) && c_p->freason == TRAP) { + NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + if (nep && nep->argc >= 0) { + erts_nif_export_save_trace(c_p, nep, applying, ep, + cp, flags, flags_meta, + I, meta_tracer); + return 1; + } + } + return 0; } ERTS_GLB_INLINE Process * diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index fd756692f9..d946844f15 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2290,9 +2290,9 @@ erts_nif_export_cleanup_nif_mod(NifExport *ep) } static ERTS_INLINE void -nif_export_restore(Process *c_p, NifExport *ep) +nif_export_restore(Process *c_p, NifExport *ep, Eterm res) { - erts_nif_export_restore(c_p, ep); + erts_nif_export_restore(c_p, ep, res); ASSERT(ep->m); nif_export_cleanup_nif_mod(ep); } @@ -2320,7 +2320,7 @@ dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc))); ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); ASSERT(ep); - nif_export_restore(proc, ep); + nif_export_restore(proc, ep, argv[0]); return argv[0]; } @@ -2462,7 +2462,7 @@ execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) /* Done (not rescheduled)... */ ASSERT(ep->func == ERTS_DBG_NIF_NOT_SCHED_MARKER); if (!env->exception_thrown) - nif_export_restore(proc, ep); + nif_export_restore(proc, ep, result); else { nif_export_cleanup_nif_mod(ep); /* diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h index 378b6de49c..01fe1e5e23 100644 --- a/erts/emulator/beam/erl_trace.h +++ b/erts/emulator/beam/erl_trace.h @@ -143,6 +143,11 @@ Uint erts_trace_flag2bit(Eterm flag); int erts_trace_flags(Eterm List, Uint *pMask, ErtsTracer *pTracer, int *pCpuTimestamp); Eterm erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr *I); +Eterm +erts_bif_trace_epilogue(Process *p, Eterm result, int applying, + Export* ep, BeamInstr *cp, Uint32 flags, + Uint32 flags_meta, BeamInstr* I, + ErtsTracer meta_tracer); #ifdef ERTS_SMP void erts_send_pending_trace_msgs(ErtsSchedulerData *esdp); -- cgit v1.2.3 From d0e88c0c69f94625daf9cafa192bac97115e9072 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 16 Dec 2016 16:35:57 +0100 Subject: Perform potentially long GC on dirty schedulers if available --- erts/emulator/beam/beam_bif_load.c | 26 ++- erts/emulator/beam/beam_emu.c | 119 ++++++----- erts/emulator/beam/bif.c | 9 +- erts/emulator/beam/bif.tab | 1 - erts/emulator/beam/erl_gc.c | 169 +++++++++++++--- erts/emulator/beam/erl_gc.h | 9 +- erts/emulator/beam/erl_nfunc_sched.h | 1 - erts/emulator/beam/erl_process.c | 375 +++++++++++++++++++++++++++-------- erts/emulator/beam/erl_process.h | 45 ++++- erts/emulator/beam/ops.tab | 7 +- 10 files changed, 562 insertions(+), 199 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 36bbe81ed5..22576503ef 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -913,7 +913,7 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed la = ERTS_COPY_LITERAL_AREA(); if (!la) - return am_ok; + goto return_ok; oh = la->off_heap; literals = (char *) &la->start[0]; @@ -977,6 +977,11 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed * this is not completely certain). We go for * the GC directly instead of scanning everything * one more time... + * + * Also note that calling functions expect a + * major GC to be performed if gc_allowed is set + * to true. If you change this, you need to fix + * callers... */ goto literal_gc; } @@ -1051,6 +1056,13 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed } } +return_ok: + +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p))) + c_p->flags &= ~F_DIRTY_CLA; +#endif + return am_ok; literal_gc: @@ -1061,13 +1073,13 @@ literal_gc: if (c_p->flags & F_DISABLE_GC) return THE_NON_VALUE; - FLAGS(c_p) |= F_NEED_FULLSWEEP; - - *redsp += erts_garbage_collect_nobump(c_p, 0, c_p->arg_reg, c_p->arity, fcalls); + *redsp += erts_garbage_collect_literals(c_p, (Eterm *) literals, lit_bsize, + oh, fcalls); - erts_garbage_collect_literals(c_p, (Eterm *) literals, lit_bsize, oh); - - *redsp += lit_bsize / 64; /* Need, better value... */ +#ifdef ERTS_DIRTY_SCHEDULERS + if (c_p->flags & F_DIRTY_CLA) + return THE_NON_VALUE; +#endif return am_ok; } diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index cfdf61eed0..85d92321b8 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -5327,10 +5327,25 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) ASSERT(!(c_p->flags & F_HIPE_MODE)); ERTS_MSACC_UPDATE_CACHE_X(); - reg = esdp->x_reg_array; - { + /* + * Set fcalls even though we ignore it, so we don't + * confuse code accessing it... + */ + if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) + c_p->fcalls = 0; + else + c_p->fcalls = CONTEXT_REDS; + + if (erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_DIRTY_RUNNING_SYS) { + erts_execute_dirty_system_task(c_p); + goto do_dirty_schedule; + } + else { + ErtsCodeMFA *codemfa; Eterm* argp; - int i; + int i, exiting; + + reg = esdp->x_reg_array; argp = c_p->arg_reg; for (i = c_p->arity - 1; i >= 0; i--) { @@ -5346,15 +5361,6 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) I = c_p->i; - /* - * Set fcalls even though we ignore it, so we don't - * confuse code accessing it... - */ - if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) - c_p->fcalls = 0; - else - c_p->fcalls = CONTEXT_REDS; - SWAPIN; #ifdef USE_VM_PROBES @@ -5378,62 +5384,55 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) DTRACE2(process_scheduled, process_buf, fun_buf); } #endif - } - { - int exiting; - - { - /* - * call_nif is always first instruction in function: - * - * I[-3]: Module - * I[-2]: Function - * I[-1]: Arity - * I[0]: &&call_nif - * I[1]: Function pointer to NIF function - * I[2]: Pointer to erl_module_nif - * I[3]: Function pointer to dirty NIF - * - * This layout is determined by the NifExport struct - */ - ErtsCodeMFA *codemfa; + /* + * call_nif is always first instruction in function: + * + * I[-3]: Module + * I[-2]: Function + * I[-1]: Arity + * I[0]: &&call_nif + * I[1]: Function pointer to NIF function + * I[2]: Pointer to erl_module_nif + * I[3]: Function pointer to dirty NIF + * + * This layout is determined by the NifExport struct + */ - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); - codemfa = erts_code_to_codemfa(I); + codemfa = erts_code_to_codemfa(I); - DTRACE_NIF_ENTRY(c_p, codemfa); - c_p->current = codemfa; - SWAPOUT; - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + DTRACE_NIF_ENTRY(c_p, codemfa); + c_p->current = codemfa; + SWAPOUT; + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - if (em_apply_bif == (BeamInstr *) *I) { - exiting = erts_call_dirty_bif(esdp, c_p, I, reg); - } - else { - ASSERT(em_call_nif == (BeamInstr *) *I); - exiting = erts_call_dirty_nif(esdp, c_p, I, reg); - } + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + if (em_apply_bif == (BeamInstr *) *I) { + exiting = erts_call_dirty_bif(esdp, c_p, I, reg); + } + else { + ASSERT(em_call_nif == (BeamInstr *) *I); + exiting = erts_call_dirty_nif(esdp, c_p, I, reg); + } - ASSERT(!(c_p->flags & F_HIBERNATE_SCHED)); + ASSERT(!(c_p->flags & F_HIBERNATE_SCHED)); - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); - if (exiting) - goto do_dirty_schedule; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); + if (exiting) + goto do_dirty_schedule; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - DTRACE_NIF_RETURN(c_p, codemfa); - ERTS_HOLE_CHECK(c_p); - SWAPIN; - I = c_p->i; - goto context_switch; - } + DTRACE_NIF_RETURN(c_p, codemfa); + ERTS_HOLE_CHECK(c_p); + SWAPIN; + I = c_p->i; + goto context_switch; } #endif /* ERTS_DIRTY_SCHEDULERS */ } diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index cae346267c..a5b66db35c 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -3865,13 +3865,6 @@ BIF_RETTYPE now_0(BIF_ALIST_0) /**********************************************************************/ -BIF_RETTYPE garbage_collect_0(BIF_ALIST_0) -{ - FLAGS(BIF_P) |= F_NEED_FULLSWEEP; - erts_garbage_collect(BIF_P, 0, NULL, 0); - return am_true; -} - /* * Pass atom 'minor' for relaxed generational GC run. This is only * recommendation, major run may still be chosen by VM. @@ -4324,7 +4317,7 @@ BIF_RETTYPE group_leader_2(BIF_ALIST_2) erts_smp_proc_unlock(new_member, ERTS_PROC_LOCK_STATUS); if (new_member == BIF_P || !(erts_smp_atomic32_read_nob(&new_member->state) - & (ERTS_PSFLG_DIRTY_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS))) { + & ERTS_PSFLG_DIRTY_RUNNING)) { new_member->group_leader = STORE_NC_IN_PROC(new_member, BIF_ARG_1); } diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index f461da913e..47fdcfa7a4 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -68,7 +68,6 @@ gcbif erlang:float/1 bif erlang:float_to_list/1 bif erlang:float_to_list/2 bif erlang:fun_info/2 -bif erlang:garbage_collect/0 bif erts_internal:garbage_collect/1 bif erlang:get/0 bif erlang:get/1 diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 6093b0cf39..5b299d9acf 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -118,11 +118,14 @@ static Eterm *full_sweep_heaps(Process *p, char *oh, Uint oh_size, Eterm *objv, int nobj); static int garbage_collect(Process* p, ErlHeapFragment *live_hf_end, - int need, Eterm* objv, int nobj, int fcalls); + int need, Eterm* objv, int nobj, int fcalls, + Uint max_young_gen_usage); static int major_collection(Process* p, ErlHeapFragment *live_hf_end, - int need, Eterm* objv, int nobj, Uint *recl); + int need, Eterm* objv, int nobj, + Uint ygen_usage, Uint *recl); static int minor_collection(Process* p, ErlHeapFragment *live_hf_end, - int need, Eterm* objv, int nobj, Uint *recl); + int need, Eterm* objv, int nobj, + Uint ygen_usage, Uint *recl); static void do_minor(Process *p, ErlHeapFragment *live_hf_end, char *mature, Uint mature_size, Uint new_sz, Eterm* objv, int nobj); @@ -413,15 +416,15 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end, regs = erts_proc_sched_data(p)->x_reg_array; } #endif - cost = garbage_collect(p, live_hf_end, 0, regs, p->arity, p->fcalls); + cost = garbage_collect(p, live_hf_end, 0, regs, p->arity, p->fcalls, 0); } else { - cost = garbage_collect(p, live_hf_end, 0, regs, arity, p->fcalls); + cost = garbage_collect(p, live_hf_end, 0, regs, arity, p->fcalls, 0); } } else { Eterm val[1]; val[0] = result; - cost = garbage_collect(p, live_hf_end, 0, val, 1, p->fcalls); + cost = garbage_collect(p, live_hf_end, 0, val, 1, p->fcalls, 0); result = val[0]; } BUMP_REDS(p, cost); @@ -599,6 +602,32 @@ young_gen_usage(Process *p) } \ } while (0) +#ifdef ERTS_DIRTY_SCHEDULERS + +static ERTS_INLINE void +check_for_possibly_long_gc(Process *p, Uint ygen_usage) +{ + int major; + Uint sz; + + major = (p->flags & F_NEED_FULLSWEEP) || GEN_GCS(p) >= MAX_GEN_GCS(p); + + sz = ygen_usage; + sz += p->hend - p->stop; + if (p->flags & F_ON_HEAP_MSGQ) + sz += p->msg.len; + if (major) + sz += p->old_htop - p->old_heap; + + if (sz >= ERTS_POTENTIALLY_LONG_GC_HSIZE) { + ASSERT(!(p->flags & (F_DISABLE_GC|F_DELAY_GC))); + p->flags |= major ? F_DIRTY_MAJOR_GC : F_DIRTY_MINOR_GC; + erts_schedule_dirty_sys_execution(p); + } +} + +#endif + /* * Garbage collect a process. * @@ -609,13 +638,15 @@ young_gen_usage(Process *p) */ static int garbage_collect(Process* p, ErlHeapFragment *live_hf_end, - int need, Eterm* objv, int nobj, int fcalls) + int need, Eterm* objv, int nobj, int fcalls, + Uint max_young_gen_usage) { Uint reclaimed_now = 0; + Uint ygen_usage; Eterm gc_trace_end_tag; int reds; ErtsMonotonicTime start_time = 0; /* Shut up faulty warning... */ - ErtsSchedulerData *esdp; + ErtsSchedulerData *esdp = erts_proc_sched_data(p); erts_aint32_t state; ERTS_MSACC_PUSH_STATE_M(); #ifdef USE_VM_PROBES @@ -624,13 +655,26 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, ERTS_CHK_MBUF_SZ(p); - ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls) - >= erts_proc_sched_data(p)->virtual_reds); + ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls) >= esdp->virtual_reds); state = erts_smp_atomic32_read_nob(&p->state); - if (p->flags & (F_DISABLE_GC|F_DELAY_GC) || state & ERTS_PSFLG_EXITING) + if ((p->flags & (F_DISABLE_GC|F_DELAY_GC)) || state & ERTS_PSFLG_EXITING) { +#ifdef ERTS_DIRTY_SCHEDULERS + delay_gc_before_start: +#endif return delay_garbage_collection(p, live_hf_end, need, fcalls); + } + + ygen_usage = max_young_gen_usage ? max_young_gen_usage : young_gen_usage(p); + +#ifdef ERTS_DIRTY_SCHEDULERS + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + check_for_possibly_long_gc(p, ygen_usage); + if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) + goto delay_gc_before_start; + } +#endif if (p->abandoned_heap) live_hf_end = ERTS_INVALID_HFRAG_PTR; @@ -639,8 +683,6 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_GC); - esdp = erts_get_scheduler_data(); - erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); if (erts_system_monitor_long_gc != 0) start_time = erts_get_monotonic_time(esdp); @@ -667,14 +709,25 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, trace_gc(p, am_gc_minor_start, need, THE_NON_VALUE); } DTRACE2(gc_minor_start, pidbuf, need); - reds = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); + reds = minor_collection(p, live_hf_end, need, objv, nobj, + ygen_usage, &reclaimed_now); DTRACE2(gc_minor_end, pidbuf, reclaimed_now); if (reds == -1) { if (IS_TRACED_FL(p, F_TRACE_GC)) { trace_gc(p, am_gc_minor_end, reclaimed_now, THE_NON_VALUE); } +#ifdef ERTS_DIRTY_SCHEDULERS + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + p->flags |= F_NEED_FULLSWEEP; + check_for_possibly_long_gc(p, ygen_usage); + if (p->flags & F_DIRTY_MAJOR_GC) + goto delay_gc_after_start; + } +#endif goto do_major_collection; } + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) + p->flags &= ~F_DIRTY_MINOR_GC; gc_trace_end_tag = am_gc_minor_end; } else { do_major_collection: @@ -683,7 +736,10 @@ do_major_collection: trace_gc(p, am_gc_major_start, need, THE_NON_VALUE); } DTRACE2(gc_major_start, pidbuf, need); - reds = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); + reds = major_collection(p, live_hf_end, need, objv, nobj, + ygen_usage, &reclaimed_now); + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) + p->flags &= ~(F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC); DTRACE2(gc_major_end, pidbuf, reclaimed_now); gc_trace_end_tag = am_gc_major_end; ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC); @@ -713,6 +769,9 @@ do_major_collection: am_kill, NIL, NULL, 0); erts_smp_proc_unlock(p, locks & ERTS_PROC_LOCKS_ALL_MINOR); +#ifdef ERTS_DIRTY_SCHEDULERS + delay_gc_after_start: +#endif /* erts_send_exit_signal looks for ERTS_PSFLG_GC, so we have to remove it after the signal is sent */ erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); @@ -795,7 +854,7 @@ do_major_collection: int erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj, int fcalls) { - int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, fcalls); + int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, fcalls, 0); int reds_left = ERTS_REDS_LEFT(p, fcalls); if (reds > reds_left) reds = reds_left; @@ -806,7 +865,7 @@ erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj, int fca void erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) { - int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, p->fcalls); + int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, p->fcalls, 0); BUMP_REDS(p, reds); ASSERT(CONTEXT_REDS - ERTS_BIF_REDS_LEFT(p) >= erts_proc_sched_data(p)->virtual_reds); @@ -832,6 +891,20 @@ erts_garbage_collect_hibernate(Process* p) if (p->flags & F_DISABLE_GC) ERTS_INTERNAL_ERROR("GC disabled"); +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p))) + p->flags &= ~(F_DIRTY_GC_HIBERNATE|F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC); + else { + Uint flags = p->flags; + p->flags |= F_NEED_FULLSWEEP; + check_for_possibly_long_gc(p, (p->htop - p->heap) + p->mbuf_sz); + if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) { + p->flags = flags|F_DIRTY_GC_HIBERNATE; + return; + } + p->flags = flags; + } +#endif /* * Preliminaries. */ @@ -843,7 +916,6 @@ erts_garbage_collect_hibernate(Process* p) * Do it. */ - heap_size = p->heap_sz + (p->old_htop - p->old_heap) + p->mbuf_sz; heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_TMP_HEAP, @@ -984,10 +1056,11 @@ static ERTS_INLINE void offset_nstack(Process* p, Sint offs, #endif /* HIPE */ -void +int erts_garbage_collect_literals(Process* p, Eterm* literals, Uint byte_lit_size, - struct erl_off_heap_header* oh) + struct erl_off_heap_header* oh, + int fcalls) { Uint lit_size = byte_lit_size / sizeof(Eterm); Uint old_heap_size; @@ -999,20 +1072,49 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint area_size; Eterm* old_htop; Uint n; + Uint ygen_usage = 0; struct erl_off_heap_header** prev = NULL; + Sint64 reds; + + if (p->flags & (F_DISABLE_GC|F_DELAY_GC)) + ERTS_INTERNAL_ERROR("GC disabled"); + + /* + * First an ordinary major collection... + */ + + p->flags |= F_NEED_FULLSWEEP; + +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p))) + p->flags &= ~F_DIRTY_CLA; + else { + ygen_usage = young_gen_usage(p); + check_for_possibly_long_gc(p, + (byte_lit_size/sizeof(Uint) + + 2*ygen_usage)); + if (p->flags & F_DIRTY_MAJOR_GC) { + p->flags |= F_DIRTY_CLA; + return 10; + } + } +#endif + + reds = (Sint64) garbage_collect(p, ERTS_INVALID_HFRAG_PTR, 0, + p->arg_reg, p->arity, fcalls, + ygen_usage); + + ASSERT(!(p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC))); - if (p->flags & F_DISABLE_GC) - return; /* * Set GC state. */ erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); /* - * We assume that the caller has already done a major collection - * (which has discarded the old heap), so that we don't have to cope - * with pointer to literals on the old heap. We will now allocate - * an old heap to contain the literals. + * Just did a major collection (which has discarded the old heap), + * so that we don't have to cope with pointer to literals on the + * old heap. We will now allocate an old heap to contain the literals. */ ASSERT(p->old_heap == 0); /* Must NOT have an old heap yet. */ @@ -1155,15 +1257,21 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, * Restore status. */ erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); + + reds += (Sint64) gc_cost((p->htop - p->heap) + byte_lit_size/sizeof(Uint), 0); + if (reds > INT_MAX) + return INT_MAX; + return (int) reds; } static int minor_collection(Process* p, ErlHeapFragment *live_hf_end, - int need, Eterm* objv, int nobj, Uint *recl) + int need, Eterm* objv, int nobj, + Uint ygen_usage, Uint *recl) { Eterm *mature = p->abandoned_heap ? p->abandoned_heap : p->heap; Uint mature_size = p->high_water - mature; - Uint size_before = young_gen_usage(p); + Uint size_before = ygen_usage; /* * Check if we have gone past the max heap size limit @@ -1534,7 +1642,8 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, static int major_collection(Process* p, ErlHeapFragment *live_hf_end, - int need, Eterm* objv, int nobj, Uint *recl) + int need, Eterm* objv, int nobj, + Uint ygen_usage, Uint *recl) { Uint size_before, size_after, stack_size; Eterm* n_heap; @@ -1552,7 +1661,7 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, * to receive all live data. */ - size_before = young_gen_usage(p); + size_before = ygen_usage; size_before += p->old_htop - p->old_heap; stack_size = p->hend - p->stop; diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 54ea9ca3c0..0dfdf52028 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -25,6 +25,8 @@ /* GC declarations shared by beam/erl_gc.c and hipe/hipe_gc.c */ +#define ERTS_POTENTIALLY_LONG_GC_HSIZE (128*1024) /* Words */ + #include "erl_map.h" #if defined(DEBUG) && !ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -145,9 +147,10 @@ void erts_garbage_collect_hibernate(struct process* p); Eterm erts_gc_after_bif_call_lhf(struct process* p, ErlHeapFragment *live_hf_end, Eterm result, Eterm* regs, Uint arity); Eterm erts_gc_after_bif_call(struct process* p, Eterm result, Eterm* regs, Uint arity); -void erts_garbage_collect_literals(struct process* p, Eterm* literals, - Uint lit_size, - struct erl_off_heap_header* oh); +int erts_garbage_collect_literals(struct process* p, Eterm* literals, + Uint lit_size, + struct erl_off_heap_header* oh, + int fcalls); Uint erts_next_heap_size(Uint, Uint); Eterm erts_heap_sizes(struct process* p); diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h index 5c98957cb7..55a3a6dbf6 100644 --- a/erts/emulator/beam/erl_nfunc_sched.h +++ b/erts/emulator/beam/erl_nfunc_sched.h @@ -54,7 +54,6 @@ typedef struct { /* --- The following is only used on error --- */ BeamInstr *pc; /* Program counter */ BeamInstr *cp; /* Continuation pointer */ - void *nif; /* Original NIF/BIF call */ ErtsCodeMFA *mfa; /* MFA of original call */ int argc; /* Number of arguments in original call */ int argv_size; /* Allocated size of argv */ diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 955b98b35b..da4b468d66 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -6439,10 +6439,30 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, int enqueue; /* < 0 -> use proxy */ ErtsRunQueue* runq; - if (is_normal_sched) - running_flgs = ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS; - else + if (!is_normal_sched) running_flgs = ERTS_PSFLG_DIRTY_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS; + else { + running_flgs = ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS; +#ifdef ERTS_DIRTY_SCHEDULERS + if (state & ERTS_PSFLG_DIRTY_ACTIVE_SYS + && (p->flags & (F_DELAY_GC|F_DISABLE_GC))) { + /* + * Delay dirty GC; will be enabled automatically + * again by next GC... + */ + + /* + * No normal execution until dirty CLA or hibernat has + * been handled... + */ + ASSERT(!(p->flags & (F_DIRTY_CLA | F_DIRTY_GC_HIBERNATE))); + + state = erts_smp_atomic32_read_band_nob(&p->state, + ~ERTS_PSFLG_DIRTY_ACTIVE_SYS); + state &= ~ERTS_PSFLG_DIRTY_ACTIVE_SYS; + } +#endif + } a = state; @@ -8642,8 +8662,7 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, erts_aint32_t state; state = erts_smp_atomic32_read_nob(&rp->state); ASSERT((state & ERTS_PSFLG_PENDING_EXIT) - || !(state & (ERTS_PSFLG_RUNNING - | ERTS_PSFLG_DIRTY_RUNNING_SYS))); + || !(state & ERTS_PSFLG_RUNNING)); } #endif @@ -9981,6 +10000,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) | ERTS_PSFLG_FREE | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING_SYS | ERTS_PSFLG_EXITING)) == ERTS_PSFLG_EXITING) & (!!is_normal_sched)) @@ -9992,7 +10012,14 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) | ERTS_PSFLG_PENDING_EXIT | ERTS_PSFLG_ACTIVE_SYS | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) - != ERTS_PSFLG_SUSPENDED)); + != ERTS_PSFLG_SUSPENDED) +#ifdef ERTS_DIRTY_SCHEDULERS + & (!(state & (ERTS_PSFLG_EXITING + | ERTS_PSFLG_PENDING_EXIT)) + | (!!is_normal_sched)) +#endif + ); + if (run_process) { if (state & (ERTS_PSFLG_ACTIVE_SYS | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) @@ -10133,65 +10160,70 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) } } - if (state & (ERTS_PSFLG_RUNNING_SYS - | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { - /* - * GC is normally never delayed when a process - * is scheduled out, but might be when executing - * hand written beam assembly in - * prim_eval:'receive'. If GC is delayed we are - * not allowed to execute system tasks. - */ - if (!(p->flags & F_DELAY_GC)) { - int cost = execute_sys_tasks(p, &state, reds); - calls += cost; - reds -= cost; - if (reds <= 0 + if (is_normal_sched) { + + if (state & ERTS_PSFLG_RUNNING_SYS) { + /* + * GC is normally never delayed when a process + * is scheduled out, but might be when executing + * hand written beam assembly in + * prim_eval:'receive'. If GC is delayed we are + * not allowed to execute system tasks. + */ + if (!(p->flags & F_DELAY_GC)) { + int cost = execute_sys_tasks(p, &state, reds); + calls += cost; + reds -= cost; + if (reds <= 0) + goto sched_out_proc; #ifdef ERTS_DIRTY_SCHEDULERS - || !is_normal_sched - || (state & ERTS_PSFLGS_DIRTY_WORK) + if (state & ERTS_PSFLGS_DIRTY_WORK) + goto sched_out_proc; #endif - ) { - goto sched_out_proc; - } - } + } - ASSERT(state & psflg_running_sys); - ASSERT(!(state & psflg_running)); + ASSERT(state & psflg_running_sys); + ASSERT(!(state & psflg_running)); - while (1) { - erts_aint32_t n, e; + while (1) { + erts_aint32_t n, e; - if (((state & (ERTS_PSFLG_SUSPENDED - | ERTS_PSFLG_ACTIVE)) != ERTS_PSFLG_ACTIVE) - && !(state & ERTS_PSFLG_EXITING)) { - goto sched_out_proc; - } + if (((state & (ERTS_PSFLG_SUSPENDED + | ERTS_PSFLG_ACTIVE)) != ERTS_PSFLG_ACTIVE) + && !(state & ERTS_PSFLG_EXITING)) { + goto sched_out_proc; + } - n = e = state; - n &= ~psflg_running_sys; - n |= psflg_running; + n = e = state; + n &= ~psflg_running_sys; + n |= psflg_running; - state = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); - if (state == e) { - state = n; - break; - } + state = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); + if (state == e) { + state = n; + break; + } - ASSERT(state & psflg_running_sys); - ASSERT(!(state & psflg_running)); - } - } + ASSERT(state & psflg_running_sys); + ASSERT(!(state & psflg_running)); + } + } - if (ERTS_IS_GC_DESIRED(p) && !ERTS_SCHEDULER_IS_DIRTY_IO(esdp)) { - if (!(state & ERTS_PSFLG_EXITING) && !(p->flags & (F_DELAY_GC|F_DISABLE_GC))) { - int cost = scheduler_gc_proc(p, reds); - calls += cost; - reds -= cost; - if (reds <= 0) - goto sched_out_proc; - } - } + if (ERTS_IS_GC_DESIRED(p)) { + if (!(state & ERTS_PSFLG_EXITING) + && !(p->flags & (F_DELAY_GC|F_DISABLE_GC))) { + int cost = scheduler_gc_proc(p, reds); + calls += cost; + reds -= cost; + if (reds <= 0) + goto sched_out_proc; +#ifdef ERTS_DIRTY_SCHEDULERS + if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) + goto sched_out_proc; +#endif + } + } + } if (proxy_p) { free_proxy_proc(proxy_p); @@ -10203,7 +10235,13 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); /* Never run a suspended process */ - ASSERT(!(ERTS_PSFLG_SUSPENDED & erts_smp_atomic32_read_nob(&p->state))); +#ifdef DEBUG + { + erts_aint32_t dstate = erts_smp_atomic32_read_nob(&p->state); + ASSERT(!(ERTS_PSFLG_SUSPENDED & dstate) + || (ERTS_PSFLG_DIRTY_RUNNING_SYS & dstate)); + } +#endif ASSERT(erts_proc_read_refc(p) > 0); @@ -10224,9 +10262,18 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) } static int -notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result) +notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, + Eterm st_result, int normal_sched) { - Process *rp = erts_proc_lookup(st->requester); + Process *rp; +#ifdef ERTS_DIRTY_SCHEDULERS + if (!normal_sched) + rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, + st->requester, 0, + ERTS_P2P_FLG_INC_REFC); + else +#endif + rp = erts_proc_lookup(st->requester); if (rp) { ErtsProcLocks rp_locks; ErlOffHeap *ohp; @@ -10274,6 +10321,11 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result) if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); + +#ifdef ERTS_DIRTY_SCHEDULERS + if (!normal_sched) + erts_proc_dec_refc(rp); +#endif } erts_cleanup_offheap(&st->off_heap); @@ -10429,18 +10481,23 @@ done: } static void save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio); +#ifdef ERTS_DIRTY_SCHEDULERS +static void save_dirty_task(Process *c_p, ErtsProcSysTask *st); +#endif static int execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) { - int garbage_collected = 0; + int minor_gc = 0, major_gc = 0; erts_aint32_t state = *statep; int reds = in_reds; int qmask = 0; + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p))); ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN); do { + ErtsProcSysTaskType type; ErtsProcSysTask *st; int st_prio; Eterm st_res; @@ -10458,7 +10515,9 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) if (!st) break; - switch (st->type) { + type = st->type; + + switch (type) { case ERTS_PSTT_GC_MAJOR: case ERTS_PSTT_GC_MINOR: if (c_p->flags & F_DISABLE_GC) { @@ -10467,12 +10526,23 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) reds--; } else { - if (!garbage_collected) { - if (st->type == ERTS_PSTT_GC_MAJOR) { + if (!minor_gc + || (!major_gc && type == ERTS_PSTT_GC_MAJOR)) { + if (type == ERTS_PSTT_GC_MAJOR) { FLAGS(c_p) |= F_NEED_FULLSWEEP; } reds -= scheduler_gc_proc(c_p, reds); - garbage_collected = 1; +#ifdef ERTS_DIRTY_SCHEDULERS + if (c_p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) { + save_dirty_task(c_p, st); + st = NULL; + break; + } +#endif + if (type == ERTS_PSTT_GC_MAJOR) + minor_gc = major_gc = 1; + else + minor_gc = 1; } st_res = am_true; } @@ -10499,20 +10569,31 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) case ERTS_PSTT_CLA: { int fcalls; int cla_reds = 0; + int do_gc; + if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) fcalls = reds; else fcalls = reds - CONTEXT_REDS; - st_res = erts_proc_copy_literal_area(c_p, - &cla_reds, - fcalls, - st->arg[0] == am_true); + do_gc = st->arg[0] == am_true; + st_res = erts_proc_copy_literal_area(c_p, &cla_reds, + fcalls, do_gc); reds -= cla_reds; if (is_non_value(st_res)) { +#ifdef ERTS_DIRTY_SCHEDULERS + if (c_p->flags & F_DIRTY_CLA) { + save_dirty_task(c_p, st); + st = NULL; + break; + } +#endif /* Needed gc, but gc was disabled */ save_gc_task(c_p, st, st_prio); st = NULL; + break; } + if (do_gc) /* We did a major gc */ + minor_gc = major_gc = 1; break; } case ERTS_PSTT_COHMQ: @@ -10531,7 +10612,7 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) } if (st) - reds += notify_sys_task_executed(c_p, st, st_res); + reds += notify_sys_task_executed(c_p, st, st_res, 1); state = erts_smp_atomic32_read_acqb(&c_p->state); } while (qmask && reds > 0); @@ -10559,9 +10640,18 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) Eterm st_res; int st_prio; - st = fetch_sys_task(c_p, state, &qmask, &st_prio); - if (!st) - break; +#ifdef ERTS_DIRTY_SCHEDULERS + if (c_p->dirty_sys_tasks) { + st = c_p->dirty_sys_tasks; + c_p->dirty_sys_tasks = st->next; + } + else +#endif + { + st = fetch_sys_task(c_p, state, &qmask, &st_prio); + if (!st) + break; + } switch (st->type) { case ERTS_PSTT_GC_MAJOR: @@ -10585,7 +10675,7 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) break; } - reds += notify_sys_task_executed(c_p, st, st_res); + reds += notify_sys_task_executed(c_p, st, st_res, 1); state = erts_smp_atomic32_read_acqb(&c_p->state); } while (qmask && reds < max_reds); @@ -10595,6 +10685,92 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) #ifdef ERTS_DIRTY_SCHEDULERS +void +erts_execute_dirty_system_task(Process *c_p) +{ + Eterm cla_res = THE_NON_VALUE; + ErtsProcSysTask *stasks; + + /* + * If multiple operations, perform them in the following + * order (in order to avoid unnecessary GC): + * 1. Copy Literal Area (implies major GC). + * 2. GC Hibernate (implies major GC if not woken). + * 3. Major GC (implies minor GC). + * 4. Minor GC. + * + * System task requests are handled after the actual + * operations have been performed... + */ + + ASSERT(!(c_p->flags & (F_DELAY_GC|F_DISABLE_GC))); + + if (c_p->flags & F_DIRTY_CLA) { + int cla_reds = 0; + cla_res = erts_proc_copy_literal_area(c_p, &cla_reds, c_p->fcalls, 1); + ASSERT(is_value(cla_res)); + } + + if (c_p->flags & F_DIRTY_GC_HIBERNATE) { + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); + if (c_p->msg.len) + c_p->flags &= ~F_DIRTY_GC_HIBERNATE; /* operation aborted... */ + else { + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); + c_p->fvalue = NIL; + erts_garbage_collect_hibernate(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + } + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); + } + + if (c_p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) { + if (c_p->flags & F_DIRTY_MAJOR_GC) + c_p->flags |= F_NEED_FULLSWEEP; + (void) erts_garbage_collect_nobump(c_p, 0, c_p->arg_reg, + c_p->arity, c_p->fcalls); + } + + ASSERT(!(c_p->flags & (F_DIRTY_CLA + | F_DIRTY_GC_HIBERNATE + | F_DIRTY_MAJOR_GC + | F_DIRTY_MINOR_GC))); + + stasks = c_p->dirty_sys_tasks; + c_p->dirty_sys_tasks = NULL; + + while (stasks) { + Eterm st_res; + ErtsProcSysTask *st = stasks; + stasks = st->next; + + switch (st->type) { + case ERTS_PSTT_CLA: + ASSERT(is_value(st_res)); + st_res = cla_res; + break; + case ERTS_PSTT_GC_MAJOR: + st_res = am_true; + break; + case ERTS_PSTT_GC_MINOR: + st_res = am_true; + break; + + default: + ERTS_INTERNAL_ERROR("Not supported dirty system task"); + break; + } + + (void) notify_sys_task_executed(c_p, st, st_res, 0); + + } + + erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_DIRTY_ACTIVE_SYS); +} + static BIF_RETTYPE dispatch_system_task(Process *c_p, erts_aint_t fail_state, ErtsProcSysTask *st, Eterm target, @@ -10797,7 +10973,7 @@ request_system_task(Process *c_p, Eterm requester, Eterm target, ERTS_INTERNAL_ERROR("Unknown failure schedule_process_sys_task()"); failure = am_internal_error; } - notify_sys_task_executed(c_p, st, failure); + notify_sys_task_executed(c_p, st, failure, 1); } ERTS_BIF_PREP_RET(ret, am_ok); @@ -11012,6 +11188,15 @@ save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio) } } +#ifdef ERTS_DIRTY_SCHEDULERS +static void +save_dirty_task(Process *c_p, ErtsProcSysTask *st) +{ + st->next = c_p->dirty_sys_tasks; + c_p->dirty_sys_tasks = st; +} +#endif + int erts_set_gc_state(Process *c_p, int enable) { @@ -11348,6 +11533,7 @@ static void early_init_process_struct(void *varg, Eterm data) proc->common.id = make_internal_pid(data); #ifdef ERTS_DIRTY_SCHEDULERS erts_smp_atomic32_init_nob(&proc->dirty_state, 0); + proc->dirty_sys_tasks = NULL; #endif erts_smp_atomic32_init_relb(&proc->state, arg->state); @@ -11856,6 +12042,7 @@ void erts_init_empty_process(Process *p) #ifdef ERTS_DIRTY_SCHEDULERS erts_smp_atomic32_init_nob(&p->dirty_state, 0); + p->dirty_sys_tasks = NULL; #endif erts_smp_atomic32_init_nob(&p->state, (erts_aint32_t) PRIORITY_NORMAL); @@ -12443,7 +12630,9 @@ send_exit_signal(Process *c_p, /* current process if and only } set_proc_exiting(c_p, state, rsn, NULL); } - else if (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))) { + else if (!(state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING_SYS))) { /* Process not running ... */ ErtsProcLocks need_locks = ~(*rp_locks) & ERTS_PROC_LOCKS_ALL; ErlHeapFragment *bp = NULL; @@ -12470,8 +12659,7 @@ send_exit_signal(Process *c_p, /* current process if and only ErlOffHeap *ohp; Uint rsn_sz = size_object(rsn); #ifdef ERTS_DIRTY_SCHEDULERS - if (state & (ERTS_PSFLG_DIRTY_RUNNING - | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + if (state & ERTS_PSFLG_DIRTY_RUNNING) { bp = new_message_buffer(rsn_sz); ohp = &bp->off_heap; hp = &bp->mem[0]; @@ -12488,7 +12676,7 @@ send_exit_signal(Process *c_p, /* current process if and only set_proc_exiting(rp, state, rsn_cpy, bp); } else { /* Process running... */ - + /* * The pending exit will be discovered when the process * is scheduled out if not discovered earlier. @@ -12510,8 +12698,35 @@ send_exit_signal(Process *c_p, /* current process if and only &bp->off_heap); rp->pending_exit.bp = bp; } - erts_smp_atomic32_read_bor_relb(&rp->state, - ERTS_PSFLG_PENDING_EXIT); + + /* + * If no dirty work has been scheduled, pending exit will + * be discovered when the process is scheduled. If dirty work + * has been scheduled, we may need to add it to a normal run + * queue... + */ +#ifndef ERTS_DIRTY_SCHEDULERS + (void) erts_smp_atomic32_read_bor_relb(&rp->state, + ERTS_PSFLG_PENDING_EXIT); +#else + { + erts_aint32_t a = erts_smp_atomic32_read_nob(&rp->state); + while (1) { + erts_aint32_t n, e; + int dwork; + n = e = a; + n |= ERTS_PSFLG_PENDING_EXIT; + dwork = !!(n & ERTS_PSFLGS_DIRTY_WORK); + n &= ~ERTS_PSFLGS_DIRTY_WORK; + a = erts_smp_atomic32_cmpxchg_mb(&rp->state, n, e); + if (a == e) { + if (dwork) + erts_schedule_process(rp, n, *rp_locks); + break; + } + } + } +#endif } } /* else: diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 58a50f8cf4..3938c4c913 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1060,6 +1060,9 @@ struct process { Uint64 bin_old_vheap; /* Virtual old heap size for binaries */ ErtsProcSysTaskQs *sys_task_qs; +#ifdef ERTS_DIRTY_SCHEDULERS + ErtsProcSysTask *dirty_sys_tasks; +#endif erts_smp_atomic32_t state; /* Process state flags (see ERTS_PSFLG_*) */ #ifdef ERTS_DIRTY_SCHEDULERS @@ -1384,14 +1387,18 @@ extern int erts_system_profile_ts_type; #define F_FORCE_GC (1 << 10) /* Force gc at process in-scheduling */ #define F_DISABLE_GC (1 << 11) /* Disable GC (see below) */ #define F_OFF_HEAP_MSGQ (1 << 12) /* Off heap msg queue */ -#define F_ON_HEAP_MSGQ (1 << 13) /* Off heap msg queue */ +#define F_ON_HEAP_MSGQ (1 << 13) /* On heap msg queue */ #define F_OFF_HEAP_MSGQ_CHNG (1 << 14) /* Off heap msg queue changing */ #define F_ABANDONED_HEAP_USE (1 << 15) /* Have usage of abandoned heap */ #define F_DELAY_GC (1 << 16) /* Similar to disable GC (see below) */ #define F_SCHDLR_ONLN_WAITQ (1 << 17) /* Process enqueued waiting to change schedulers online */ #define F_HAVE_BLCKD_NMSCHED (1 << 18) /* Process has blocked normal multi-scheduling */ -#define F_HIPE_MODE (1 << 19) +#define F_HIPE_MODE (1 << 19) /* Process is executing in HiPE mode */ #define F_DELAYED_DEL_PROC (1 << 20) /* Delay delete process (dirty proc exit case) */ +#define F_DIRTY_CLA (1 << 21) /* Dirty copy literal area scheduled */ +#define F_DIRTY_GC_HIBERNATE (1 << 22) /* Dirty GC hibernate scheduled */ +#define F_DIRTY_MAJOR_GC (1 << 23) /* Dirty major GC scheduled */ +#define F_DIRTY_MINOR_GC (1 << 24) /* Dirty minor GC scheduled */ /* * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent @@ -1573,7 +1580,9 @@ void erts_init_scheduling(int, int , int, int, int #endif ); - +#ifdef ERTS_DIRTY_SCHEDULERS +void erts_execute_dirty_system_task(Process *c_p); +#endif int erts_set_gc_state(Process *c_p, int enable); Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable); Eterm erts_system_check_request(Process *c_p); @@ -1918,6 +1927,8 @@ ErtsSchedulerData *erts_get_scheduler_data(void) void erts_schedule_process(Process *, erts_aint32_t, ErtsProcLocks); ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks); +ERTS_GLB_INLINE void erts_schedule_dirty_sys_execution(Process *c_p); + #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks) @@ -1927,6 +1938,34 @@ erts_proc_notify_new_message(Process *p, ErtsProcLocks locks) if (!(state & ERTS_PSFLG_ACTIVE)) erts_schedule_process(p, state, locks); } + +ERTS_GLB_INLINE void +erts_schedule_dirty_sys_execution(Process *c_p) +{ + erts_aint32_t a, n, e; + + a = erts_smp_atomic32_read_nob(&c_p->state); + + /* + * Only a currently executing process schedules + * itself for dirty-sys execution... + */ + + ASSERT(a & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)); + + /* Don't set dirty-active-sys if we are about to exit... */ + + while (!(a & (ERTS_PSFLG_DIRTY_ACTIVE_SYS + | ERTS_PSFLG_EXITING + | ERTS_PSFLG_PENDING_EXIT))) { + e = a; + n = a | ERTS_PSFLG_DIRTY_ACTIVE_SYS; + a = erts_smp_atomic32_cmpxchg_mb(&c_p->state, n, e); + if (a == e) + break; /* dirty-active-sys set */ + } +} + #endif #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 4ef04d020a..ec36b23059 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -802,15 +802,10 @@ call_ext_last u==1 Bif=u$bif:erts_internal:check_process_code/1 D => i_call_ext_ call_ext_only u==1 Bif=u$bif:erts_internal:check_process_code/1 => i_call_ext_only Bif # -# The BIFs erlang:garbage_collect/0 must be called like a function, +# The BIFs erts_internal:garbage_collect/1 must be called like a function, # to allow them to invoke the garbage collector. (The stack pointer must # be saved and p->arity must be zeroed, which is not done on ordinary BIF calls.) # - -call_ext u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext Bif -call_ext_last u==0 Bif=u$bif:erlang:garbage_collect/0 D => i_call_ext_last Bif D -call_ext_only u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext_only Bif - call_ext u==1 Bif=u$bif:erts_internal:garbage_collect/1 => i_call_ext Bif call_ext_last u==1 Bif=u$bif:erts_internal:garbage_collect/1 D => i_call_ext_last Bif D call_ext_only u==1 Bif=u$bif:erts_internal:garbage_collect/1 => i_call_ext_only Bif -- cgit v1.2.3 From 9e9e7bf64fa97663dd4fb645c014880205fb46db Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 9 Jan 2017 15:13:58 +0100 Subject: erts: Add assertions for correct ErlNifEnv when constructing container terms. --- erts/emulator/beam/erl_nif.c | 107 +++++++++++++++++++++++++++++++++++++++++-- erts/emulator/beam/global.h | 7 +++ 2 files changed, 110 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 5499512dd1..10514f6b3f 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -89,6 +89,14 @@ static void add_readonly_check(ErlNifEnv*, unsigned char* ptr, unsigned sz); # define ADD_READONLY_CHECK(ENV,PTR,SIZE) ((void)0) #endif +#ifdef ERTS_NIF_ASSERT_IN_ENV +# define ASSERT_IN_ENV(ENV, TERM, NR, TYPE) dbg_assert_in_env(ENV, TERM, NR, TYPE, __func__) +static void dbg_assert_in_env(ErlNifEnv*, Eterm term, int nr, const char* type, const char* func); +# include "erl_gc.h" +#else +# define ASSERT_IN_ENV(ENV, TERM, NR, TYPE) +#endif + #ifdef DEBUG static int is_offheap(const ErlOffHeap* off_heap); #endif @@ -196,6 +204,9 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif, ASSERT(p->common.id != ERTS_INVALID_PID); +#ifdef ERTS_NIF_ASSERT_IN_ENV + env->dbg_disable_assert_in_env = 0; +#endif #if defined(DEBUG) && defined(ERTS_DIRTY_SCHEDULERS) { ErtsSchedulerData *esdp = erts_get_scheduler_data(); @@ -474,11 +485,15 @@ setup_nif_env(struct enif_msg_environment_t* msg_env, HEAP_END(&msg_env->phony_proc) = phony_heap; MBUF(&msg_env->phony_proc) = NULL; msg_env->phony_proc.common.id = ERTS_INVALID_PID; + msg_env->env.tracee = tracee; + #ifdef FORCE_HEAP_FRAGS msg_env->phony_proc.space_verified = 0; msg_env->phony_proc.space_verified_from = NULL; #endif - msg_env->env.tracee = tracee; +#ifdef ERTS_NIF_ASSERT_IN_ENV + msg_env->env.dbg_disable_assert_in_env = 0; +#endif } ErlNifEnv* enif_alloc_env(void) @@ -1586,6 +1601,9 @@ int enif_make_existing_atom_len(ErlNifEnv* env, const char* name, size_t len, ERL_NIF_TERM enif_make_tuple(ErlNifEnv* env, unsigned cnt, ...) { +#ifdef ERTS_NIF_ASSERT_IN_ENV + int nr = 0; +#endif Eterm* hp = alloc_heap(env,cnt+1); Eterm ret = make_tuple(hp); va_list ap; @@ -1593,7 +1611,9 @@ ERL_NIF_TERM enif_make_tuple(ErlNifEnv* env, unsigned cnt, ...) *hp++ = make_arityval(cnt); va_start(ap,cnt); while (cnt--) { - *hp++ = va_arg(ap,Eterm); + Eterm elem = va_arg(ap,Eterm); + ASSERT_IN_ENV(env, elem, ++nr, "tuple"); + *hp++ = elem; } va_end(ap); return ret; @@ -1601,12 +1621,16 @@ ERL_NIF_TERM enif_make_tuple(ErlNifEnv* env, unsigned cnt, ...) ERL_NIF_TERM enif_make_tuple_from_array(ErlNifEnv* env, const ERL_NIF_TERM arr[], unsigned cnt) { +#ifdef ERTS_NIF_ASSERT_IN_ENV + int nr = 0; +#endif Eterm* hp = alloc_heap(env,cnt+1); Eterm ret = make_tuple(hp); const Eterm* src = arr; *hp++ = make_arityval(cnt); while (cnt--) { + ASSERT_IN_ENV(env, *src, ++nr, "tuple"); *hp++ = *src++; } return ret; @@ -1617,6 +1641,8 @@ ERL_NIF_TERM enif_make_list_cell(ErlNifEnv* env, Eterm car, Eterm cdr) Eterm* hp = alloc_heap(env,2); Eterm ret = make_list(hp); + ASSERT_IN_ENV(env, car, 0, "head of list cell"); + ASSERT_IN_ENV(env, cdr, 0, "tail of list cell"); CAR(hp) = car; CDR(hp) = cdr; return ret; @@ -1628,6 +1654,9 @@ ERL_NIF_TERM enif_make_list(ErlNifEnv* env, unsigned cnt, ...) return NIL; } else { +#ifdef ERTS_NIF_ASSERT_IN_ENV + int nr = 0; +#endif Eterm* hp = alloc_heap(env,cnt*2); Eterm ret = make_list(hp); Eterm* last = &ret; @@ -1635,8 +1664,10 @@ ERL_NIF_TERM enif_make_list(ErlNifEnv* env, unsigned cnt, ...) va_start(ap,cnt); while (cnt--) { + Eterm term = va_arg(ap,Eterm); *last = make_list(hp); - *hp = va_arg(ap,Eterm); + ASSERT_IN_ENV(env, term, ++nr, "list"); + *hp = term; last = ++hp; ++hp; } @@ -1648,14 +1679,19 @@ ERL_NIF_TERM enif_make_list(ErlNifEnv* env, unsigned cnt, ...) ERL_NIF_TERM enif_make_list_from_array(ErlNifEnv* env, const ERL_NIF_TERM arr[], unsigned cnt) { +#ifdef ERTS_NIF_ASSERT_IN_ENV + int nr = 0; +#endif Eterm* hp = alloc_heap(env,cnt*2); Eterm ret = make_list(hp); Eterm* last = &ret; const Eterm* src = arr; while (cnt--) { + Eterm term = *src++; *last = make_list(hp); - *hp = *src++; + ASSERT_IN_ENV(env, term, ++nr, "list"); + *hp = term; last = ++hp; ++hp; } @@ -2789,6 +2825,10 @@ int enif_make_map_put(ErlNifEnv* env, if (!is_map(map_in)) { return 0; } + ASSERT_IN_ENV(env, map_in, 0, "old map"); + ASSERT_IN_ENV(env, key, 0, "key"); + ASSERT_IN_ENV(env, value, 0, "value"); + flush_env(env); *map_out = erts_maps_put(env->proc, key, value, map_in); cache_env(env); @@ -2823,6 +2863,10 @@ int enif_make_map_update(ErlNifEnv* env, return 0; } + ASSERT_IN_ENV(env, map_in, 0, "old map"); + ASSERT_IN_ENV(env, key, 0, "key"); + ASSERT_IN_ENV(env, value, 0, "value"); + flush_env(env); res = erts_maps_update(env->proc, key, value, map_in, map_out); cache_env(env); @@ -3543,6 +3587,9 @@ Eterm erts_nif_call_function(Process *p, Process *tracee, clear_offheap(&MSO(p)); erts_pre_nif(&env, p, mod, tracee); +#ifdef ERTS_NIF_ASSERT_IN_ENV + env.dbg_disable_assert_in_env = 1; +#endif nif_result = (*fun->fptr)(&env, argc, argv); if (env.exception_thrown) nif_result = THE_NON_VALUE; @@ -3565,6 +3612,9 @@ Eterm erts_nif_call_function(Process *p, Process *tracee, so we create a phony one. */ struct enif_msg_environment_t msg_env; pre_nif_noproc(&msg_env, mod, tracee); +#ifdef ERTS_NIF_ASSERT_IN_ENV + msg_env.env.dbg_disable_assert_in_env = 1; +#endif nif_result = (*fun->fptr)(&msg_env.env, argc, argv); if (msg_env.env.exception_thrown) nif_result = THE_NON_VALUE; @@ -3629,6 +3679,55 @@ static unsigned calc_checksum(unsigned char* ptr, unsigned size) #endif /* READONLY_CHECK */ +#ifdef ERTS_NIF_ASSERT_IN_ENV +static void dbg_assert_in_env(ErlNifEnv* env, Eterm term, + int nr, const char* type, const char* func) +{ + Uint saved_used_size; + Eterm* real_htop; + + if (is_immed(term) + || (is_non_value(term) && env->exception_thrown) + || erts_is_literal(term, ptr_val(term))) + return; + + if (env->dbg_disable_assert_in_env) { + /* + * Trace nifs may cheat as built terms are discarded after return. + * ToDo: Check if 'term' is part of argv[]. + */ + return; + } + + if (env->heap_frag) { + ASSERT(env->heap_frag == MBUF(env->proc)); + ASSERT(env->hp >= env->heap_frag->mem); + ASSERT(env->hp <= env->heap_frag->mem + env->heap_frag->alloc_size); + saved_used_size = env->heap_frag->used_size; + env->heap_frag->used_size = env->hp - env->heap_frag->mem; + real_htop = NULL; + } + else { + real_htop = env->hp; + } + if (!erts_dbg_within_proc(ptr_val(term), env->proc, real_htop)) { + fprintf(stderr, "\r\nFAILED ASSERTION in %s:\r\n", func); + if (nr) { + fprintf(stderr, "Term #%d of the %s is not from same ErlNifEnv.", + nr, type); + } + else { + fprintf(stderr, "The %s is not from the same ErlNifEnv.", type); + } + fprintf(stderr, "\r\nABORTING\r\n"); + abort(); + } + if (env->heap_frag) { + env->heap_frag->used_size = saved_used_size; + } +} +#endif + #ifdef HAVE_USE_DTRACE #define MESSAGE_BUFSIZ 1024 diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 2b2f3c5cdc..5d07490152 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -45,6 +45,9 @@ struct enif_func_t; +#ifdef DEBUG +# define ERTS_NIF_ASSERT_IN_ENV +#endif struct enif_environment_t /* ErlNifEnv */ { struct erl_module_nif* mod_nif; @@ -57,6 +60,10 @@ struct enif_environment_t /* ErlNifEnv */ int exception_thrown; /* boolean */ Process *tracee; int exiting; /* boolean (dirty nifs might return in exiting state) */ + +#ifdef ERTS_NIF_ASSERT_IN_ENV + int dbg_disable_assert_in_env; +#endif }; extern void erts_pre_nif(struct enif_environment_t*, Process*, struct erl_module_nif*, Process* tracee); -- cgit v1.2.3 From 10ace079fb6d0d626a16a5c6726c6bb8e6bb3878 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 21 Dec 2016 17:43:28 +0100 Subject: erts: Cleanup enif_make_reverse_list --- erts/emulator/beam/erl_nif.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 10514f6b3f..cd44eb880e 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1724,13 +1724,9 @@ void enif_system_info(ErlNifSysInfo *sip, size_t si_size) driver_system_info(sip, si_size); } -int enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list) { - Eterm *listptr, ret = NIL, *hp; - - if (is_nil(term)) { - *list = term; - return 1; - } +int enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list) +{ + Eterm *listptr, ret, *hp; ret = NIL; -- cgit v1.2.3 From 9e862df2bf26b9b1d79ba3b447945b7c0612e3d0 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 12 Jan 2017 18:12:38 +0100 Subject: erts: Add macro ERTS_PROC_LOCKS_HIGHER_THAN to safeguard against bugs due to future proc lock changes. The two places now using ERTS_PROC_LOCKS_HIGHER_THAN were kind of bugs as BTM and TRACE locks were missing. But there was probably no way to get there with BTM or TRACE locked. --- erts/emulator/beam/erl_message.c | 10 ++++++---- erts/emulator/beam/erl_process_lock.h | 4 ++++ 2 files changed, 10 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 118adc0c1b..04f9640a17 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -285,9 +285,11 @@ erts_queue_dist_message(Process *rcvr, if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) { if (erts_smp_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) { ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ; - if (rcvr_locks & ERTS_PROC_LOCK_STATUS) { - erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_STATUS); - need_locks |= ERTS_PROC_LOCK_STATUS; + ErtsProcLocks unlocks = + rcvr_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ); + if (unlocks) { + erts_smp_proc_unlock(rcvr, unlocks); + need_locks |= unlocks; } erts_smp_proc_lock(rcvr, need_locks); } @@ -406,7 +408,7 @@ queue_messages(Process* receiver, if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) goto exiting; - need_locks = receiver_locks & (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + need_locks = receiver_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ); if (need_locks) { erts_smp_proc_unlock(receiver, need_locks); } diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index 2cccf0697a..e7d68f0a09 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -219,6 +219,10 @@ typedef struct erts_proc_lock_t_ { #define ERTS_PROC_LOCKS_ALL_MINOR (ERTS_PROC_LOCKS_ALL \ & ~ERTS_PROC_LOCK_MAIN) +/* All locks we first must unlock to lock L */ +#define ERTS_PROC_LOCKS_HIGHER_THAN(L) \ + (ERTS_PROC_LOCKS_ALL & (~(L) & ~((L)-1))) + #define ERTS_PIX_LOCKS_BITS 10 #define ERTS_NO_OF_PIX_LOCKS (1 << ERTS_PIX_LOCKS_BITS) -- cgit v1.2.3 From 5de2d73abd5d369676e43acd17a9d9db9795ccd5 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 13 Jan 2017 14:22:31 +0100 Subject: Fix memory leak of temporary heap This bug was introduced in previous commit, and has never been released in an official OTP version. --- erts/emulator/beam/erl_gc.c | 11 +++++------ erts/emulator/beam/erl_gc.h | 1 + erts/emulator/beam/erl_process.c | 18 +----------------- 3 files changed, 7 insertions(+), 23 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 561e7fbdee..ace1524326 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -110,7 +110,6 @@ typedef struct { static Uint setup_rootset(Process*, Eterm*, int, Rootset*); static void cleanup_rootset(Rootset *rootset); -static void deallocate_previous_young_generation(Process *c_p); static Eterm *full_sweep_heaps(Process *p, int hibernate, Eterm *n_heap, Eterm* n_htop, @@ -839,7 +838,7 @@ erts_garbage_collect_hibernate(Process* p) disallow_heap_frag_ref_in_heap(p, heap, htop); #endif - deallocate_previous_young_generation(p); + erts_deallocate_young_generation(p); p->heap = heap; p->high_water = htop; @@ -1496,7 +1495,7 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, disallow_heap_frag_ref_in_heap(p, n_heap, n_htop); #endif - deallocate_previous_young_generation(p); + erts_deallocate_young_generation(p); HEAP_START(p) = n_heap; HEAP_TOP(p) = n_htop; @@ -1594,7 +1593,7 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, disallow_heap_frag_ref_in_heap(p, n_heap, n_htop); #endif - deallocate_previous_young_generation(p); + erts_deallocate_young_generation(p); HEAP_START(p) = n_heap; HEAP_TOP(p) = n_htop; @@ -1756,8 +1755,8 @@ adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj) return adjusted; } -static void -deallocate_previous_young_generation(Process *c_p) +void +erts_deallocate_young_generation(Process *c_p) { Eterm *orig_heap; diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 54ea9ca3c0..9a177b7c59 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -157,5 +157,6 @@ void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*); void erts_free_heap_frags(struct process* p); Eterm erts_max_heap_size_map(Sint, Uint, Eterm **, Uint *); int erts_max_heap_size(Eterm, Uint *, Uint *); +void erts_deallocate_young_generation(Process *c_p); #endif /* __ERL_GC_H__ */ diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index b345c35a7e..8d13723188 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -11912,7 +11912,6 @@ erts_cleanup_empty_process(Process* p) static void delete_process(Process* p) { - Eterm *heap; ErtsPSD *psd; struct saved_calls *scb; process_breakpoint_time_t *pbt; @@ -11967,13 +11966,8 @@ delete_process(Process* p) hipe_delete_process(&p->hipe); #endif - heap = p->abandoned_heap ? p->abandoned_heap : p->heap; + erts_deallocate_young_generation(p); -#ifdef DEBUG - sys_memset(heap, DEBUG_BAD_BYTE, p->heap_sz*sizeof(Eterm)); -#endif - - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, (void*) heap, p->heap_sz*sizeof(Eterm)); if (p->old_heap != NULL) { #ifdef DEBUG @@ -11985,16 +11979,6 @@ delete_process(Process* p) (p->old_hend-p->old_heap)*sizeof(Eterm)); } - /* - * Free all pending message buffers. - */ - if (p->mbuf != NULL) { - free_message_buffer(p->mbuf); - } - - if (p->msg_frag) - erts_cleanup_messages(p->msg_frag); - erts_erase_dicts(p); /* free all pending messages */ -- cgit v1.2.3 From 7350e237d66f5098b53de7fa58e725753c04d530 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 13 Jan 2017 17:46:58 +0100 Subject: Do not automatically define HARDDEBUG when DEBUG is defined --- erts/emulator/beam/erl_gc.c | 4 ++++ erts/emulator/beam/erl_gc.h | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index af799d09da..46be91c5d5 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -59,6 +59,10 @@ # define ERTS_GC_ASSERT(B) ((void) 1) #endif +#if defined(DEBUG) && 0 +# define HARDDEBUG 1 +#endif + /* * Returns number of elements in an array. */ diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 54ea9ca3c0..49cc43a48f 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -27,10 +27,6 @@ #include "erl_map.h" -#if defined(DEBUG) && !ERTS_GLB_INLINE_INCL_FUNC_DEF -# define HARDDEBUG 1 -#endif - #define IS_MOVED_BOXED(x) (!is_header((x))) #define IS_MOVED_CONS(x) (is_non_value((x))) -- cgit v1.2.3 From d74d7a4cc0808e1f6e710e5d17ec55bfe093cdd9 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 13 Jan 2017 18:19:30 +0100 Subject: erts: Fix enif_send from noproc and no msg_env Only try direct allocation on receiver heap when called in a real process context to avoid trylock on our own main lock. --- erts/emulator/beam/erl_nif.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 6b265a8b80..dd4b88e4a3 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -690,7 +690,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, Uint sz = size_object(msg); ErlOffHeap *ohp; Eterm *hp; - if (env && !env->tracee) { + if (c_p && !env->tracee) { full_flush_env(env); mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); full_cache_env(env); -- cgit v1.2.3 From 43eebdd25a0d4a17f076f017cb3ea7da5cfd1808 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 22 Dec 2016 17:48:51 +0100 Subject: Switch between scheduler types when multi-scheduling is blocked --- erts/emulator/beam/bif.h | 6 - erts/emulator/beam/erl_port_task.c | 2 +- erts/emulator/beam/erl_process.c | 996 ++++++++++++++++++++++++------------- erts/emulator/beam/erl_process.h | 22 +- 4 files changed, 674 insertions(+), 352 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index ad55b19d0a..01cca90a7a 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -498,12 +498,6 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, Eterm args[], int nargs); -typedef enum { - ERTS_SCHED_NORMAL, - ERTS_SCHED_DIRTY_CPU, - ERTS_SCHED_DIRTY_IO -} ErtsSchedType; - #ifdef ERTS_DIRTY_SCHEDULERS int erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *reg); diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 3102e44c11..4836b9e2d3 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -936,7 +936,7 @@ enqueue_port(ErtsRunQueue *runq, Port *pp) erts_smp_inc_runq_len(runq, &runq->ports.info, ERTS_PORT_PRIO_LEVEL); #ifdef ERTS_SMP - if (runq->halt_in_progress) + if (ERTS_RUNQ_FLGS_GET_NOB(runq) & ERTS_RUNQ_FLG_HALTING) erts_non_empty_runq(runq); #endif } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index da4b468d66..084ecb14ec 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -107,9 +107,31 @@ #define LOW_BIT (1 << PRIORITY_LOW) #define PORT_BIT (1 << ERTS_PORT_PRIO_LEVEL) -#define ERTS_EMPTY_RUNQ(RQ) \ - ((ERTS_RUNQ_FLGS_GET_NOB((RQ)) & ERTS_RUNQ_FLGS_QMASK) == 0 \ - && (RQ)->misc.start == NULL) +#define ERTS_IS_RUNQ_EMPTY_FLGS(FLGS) \ + (!((FLGS) & (ERTS_RUNQ_FLGS_QMASK|ERTS_RUNQ_FLG_MISC_OP))) + +#define ERTS_IS_RUNQ_EMPTY_PORTS_FLGS(FLGS) \ + (!((FLGS) & (PORT_BIT|ERTS_RUNQ_FLG_MISC_OP))) + +#define ERTS_EMPTY_RUNQ(RQ) \ + ERTS_IS_RUNQ_EMPTY_FLGS(ERTS_RUNQ_FLGS_GET_NOB((RQ))) + +#define ERTS_EMPTY_RUNQ_PORTS(RQ) \ + ERTS_IS_RUNQ_EMPTY_FLGS(ERTS_RUNQ_FLGS_GET_NOB((RQ))) + +static ERTS_INLINE int +runq_got_work_to_execute_flags(Uint32 flags) +{ + if (flags & ERTS_RUNQ_FLG_HALTING) + return !ERTS_IS_RUNQ_EMPTY_PORTS_FLGS(flags); + return !ERTS_IS_RUNQ_EMPTY_FLGS(flags); +} + +static ERTS_INLINE int +runq_got_work_to_execute(ErtsRunQueue *rq) +{ + return runq_got_work_to_execute_flags(ERTS_RUNQ_FLGS_GET_NOB(rq)); +} #undef RUNQ_READ_RQ #undef RUNQ_SET_RQ @@ -141,9 +163,6 @@ do { \ # define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) #endif -#define ERTS_EMPTY_RUNQ_PORTS(RQ) \ - (RUNQ_READ_LEN(&(RQ)->ports.info.len) == 0 && (RQ)->misc.start == NULL) - const Process erts_invalid_process = {{ERTS_INVALID_PID}}; extern BeamInstr beam_apply[]; @@ -200,158 +219,139 @@ typedef struct { ErtsProcList *chngq; } ErtsMultiSchedulingBlock; +typedef struct { + Uint32 normal; +#ifdef ERTS_DIRTY_SCHEDULERS + Uint32 dirty_cpu; + Uint32 dirty_io; +#endif +} ErtsSchedTypeCounters; + static struct { erts_smp_mtx_t mtx; - Uint32 online; - Uint32 curr_online; - Uint32 active; + ErtsSchedTypeCounters online; + ErtsSchedTypeCounters curr_online; + ErtsSchedTypeCounters active; erts_smp_atomic32_t changing; ErtsProcList *chngq; Eterm changer; ErtsMultiSchedulingBlock nmsb; /* Normal multi Scheduling Block */ ErtsMultiSchedulingBlock msb; /* Multi Scheduling Block */ +#ifdef ERTS_DIRTY_SCHEDULERS + ErtsSchedType last_msb_dirty_type; +#endif } schdlr_sspnd; -#define ERTS_SCHDLR_SSPND_S_BITS 10 -#define ERTS_SCHDLR_SSPND_DCS_BITS 11 -#define ERTS_SCHDLR_SSPND_DIS_BITS 11 - -#define ERTS_SCHDLR_SSPND_S_MASK ((1 << ERTS_SCHDLR_SSPND_S_BITS)-1) -#define ERTS_SCHDLR_SSPND_DCS_MASK ((1 << ERTS_SCHDLR_SSPND_DCS_BITS)-1) -#define ERTS_SCHDLR_SSPND_DIS_MASK ((1 << ERTS_SCHDLR_SSPND_DIS_BITS)-1) - -#define ERTS_SCHDLR_SSPND_S_SHIFT 0 -#define ERTS_SCHDLR_SSPND_DCS_SHIFT (ERTS_SCHDLR_SSPND_S_SHIFT \ - + ERTS_SCHDLR_SSPND_S_BITS) -#define ERTS_SCHDLR_SSPND_DIS_SHIFT (ERTS_SCHDLR_SSPND_DCS_SHIFT \ - + ERTS_SCHDLR_SSPND_DCS_BITS) - -#if (ERTS_SCHDLR_SSPND_S_BITS \ - + ERTS_SCHDLR_SSPND_DCS_BITS \ - + ERTS_SCHDLR_SSPND_DIS_BITS) > 32 -# error Wont fit in Uint32 -#endif +static void init_scheduler_suspend(void); -#if (ERTS_MAX_NO_OF_SCHEDULERS-1) > ERTS_SCHDLR_SSPND_S_MASK -# error Max no schedulers wont fit in its bit-field -#endif -#if ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS > ERTS_SCHDLR_SSPND_DCS_MASK -# error Max no dirty cpu schedulers wont fit in its bit-field -#endif -#if ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS > ERTS_SCHDLR_SSPND_DIS_MASK -# error Max no dirty io schedulers wont fit in its bit-field +static ERTS_INLINE Uint32 +schdlr_sspnd_eq_nscheds(ErtsSchedTypeCounters *val1p, ErtsSchedTypeCounters *val2p) +{ + int res = val1p->normal == val2p->normal; +#ifdef ERTS_DIRTY_SCHEDULERS + res &= val1p->dirty_cpu == val2p->dirty_cpu; + res &= val1p->dirty_io == val2p->dirty_io; #endif - -#define ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(S, DCS, DIS) \ - ((((Uint32) (((S) & ERTS_SCHDLR_SSPND_S_MASK))-1) \ - << ERTS_SCHDLR_SSPND_S_SHIFT) \ - | ((((Uint32) ((DCS) & ERTS_SCHDLR_SSPND_DCS_MASK)) \ - << ERTS_SCHDLR_SSPND_DCS_SHIFT)) \ - | ((((Uint32) ((DIS) & ERTS_SCHDLR_SSPND_DIS_MASK)) \ - << ERTS_SCHDLR_SSPND_DIS_SHIFT))) - -static void init_scheduler_suspend(void); + return res; +} static ERTS_INLINE Uint32 -schdlr_sspnd_get_nscheds(Uint32 *valp, ErtsSchedType type) +schdlr_sspnd_get_nscheds(ErtsSchedTypeCounters *valp, + ErtsSchedType type) { - Uint32 res = (Uint32) (*valp); switch (type) { case ERTS_SCHED_NORMAL: - res >>= ERTS_SCHDLR_SSPND_S_SHIFT; - res &= (Uint32) ERTS_SCHDLR_SSPND_S_MASK; - res++; - break; + return valp->normal; +#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: - res >>= ERTS_SCHDLR_SSPND_DCS_SHIFT; - res &= (Uint32) ERTS_SCHDLR_SSPND_DCS_MASK; - break; + return valp->dirty_cpu; case ERTS_SCHED_DIRTY_IO: - res >>= ERTS_SCHDLR_SSPND_DIS_SHIFT; - res &= (Uint32) ERTS_SCHDLR_SSPND_DIS_MASK; - break; + return valp->dirty_io; +#else + case ERTS_SCHED_DIRTY_CPU: + case ERTS_SCHED_DIRTY_IO: + return 0; +#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); return 0; } +} +static ERTS_INLINE Uint32 +schdlr_sspnd_get_nscheds_tot(ErtsSchedTypeCounters *valp) +{ + Uint32 res = valp->normal; +#ifdef ERTS_DIRTY_SCHEDULERS + res += valp->dirty_cpu; + res += valp->dirty_io; +#endif return res; } static ERTS_INLINE void -schdlr_sspnd_dec_nscheds(Uint32 *valp, ErtsSchedType type) +schdlr_sspnd_dec_nscheds(ErtsSchedTypeCounters *valp, + ErtsSchedType type) { ASSERT(schdlr_sspnd_get_nscheds(valp, type) > 0); switch (type) { case ERTS_SCHED_NORMAL: - *valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_S_SHIFT; + valp->normal--; break; +#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: - *valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_DCS_SHIFT; + valp->dirty_cpu--; break; case ERTS_SCHED_DIRTY_IO: - *valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_DIS_SHIFT; + valp->dirty_io--; break; +#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); } } static ERTS_INLINE void -schdlr_sspnd_inc_nscheds(Uint32 *valp, ErtsSchedType type) +schdlr_sspnd_inc_nscheds(ErtsSchedTypeCounters *valp, + ErtsSchedType type) { switch (type) { case ERTS_SCHED_NORMAL: - ASSERT(schdlr_sspnd_get_nscheds(valp, type) - < ERTS_MAX_NO_OF_SCHEDULERS-1); - *valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_S_SHIFT; + valp->normal++; break; +#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: - ASSERT(schdlr_sspnd_get_nscheds(valp, type) - < ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS); - *valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_DCS_SHIFT; + valp->dirty_cpu++; break; case ERTS_SCHED_DIRTY_IO: - ASSERT(schdlr_sspnd_get_nscheds(valp, type) - < ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS); - *valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_DIS_SHIFT; + valp->dirty_io++; break; +#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); } } static ERTS_INLINE void -schdlr_sspnd_set_nscheds(Uint32 *valp, ErtsSchedType type, Uint32 no) +schdlr_sspnd_set_nscheds(ErtsSchedTypeCounters *valp, + ErtsSchedType type, Uint32 no) { - Uint32 val = *valp; - switch (type) { case ERTS_SCHED_NORMAL: - ASSERT(no > 0); - val &= ~(((Uint32) ERTS_SCHDLR_SSPND_S_MASK) - << ERTS_SCHDLR_SSPND_S_SHIFT); - val |= (((no-1) & ((Uint32) ERTS_SCHDLR_SSPND_S_MASK)) - << ERTS_SCHDLR_SSPND_S_SHIFT); + valp->normal = no; break; +#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: - val &= ~(((Uint32) ERTS_SCHDLR_SSPND_DCS_MASK) - << ERTS_SCHDLR_SSPND_DCS_SHIFT); - val |= ((no & ((Uint32) ERTS_SCHDLR_SSPND_DCS_MASK)) - << ERTS_SCHDLR_SSPND_DCS_SHIFT); + valp->dirty_cpu = no; break; case ERTS_SCHED_DIRTY_IO: - val &= ~(((Uint32) ERTS_SCHDLR_SSPND_DIS_MASK) - << ERTS_SCHDLR_SSPND_DIS_SHIFT); - val |= ((no & ((Uint32) ERTS_SCHDLR_SSPND_DIS_MASK)) - << ERTS_SCHDLR_SSPND_DIS_SHIFT); + valp->dirty_io = no; break; +#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); } - - *valp = val; } static struct { @@ -2293,7 +2293,8 @@ static ERTS_INLINE erts_aint32_t handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) { unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_REAP_PORTS); - awdp->esdp->run_queue->halt_in_progress = 1; + ERTS_RUNQ_FLGS_SET(awdp->esdp->run_queue, ERTS_RUNQ_FLG_HALTING); + if (erts_smp_atomic32_dec_read_acqb(&erts_halt_progress) == 0) { int i, max = erts_ptab_max(&erts_port); erts_smp_atomic32_set_nob(&erts_halt_progress, 1); @@ -2833,11 +2834,12 @@ static erts_aint32_t sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi) { erts_aint32_t oflgs; - erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_WAITING); + erts_aint32_t nflgs; erts_aint32_t xflgs = 0; do { + nflgs = (xflgs & ERTS_SSI_FLG_MSB_EXEC); + nflgs |= ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING; oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; @@ -2859,7 +2861,7 @@ sched_prep_cont_spin_wait(ErtsSchedulerSleepInfo *ssi) if (oflgs == xflgs) return nflgs; xflgs = oflgs; - nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED; + nflgs |= oflgs & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC); } while (oflgs & ERTS_SSI_FLG_WAITING); return oflgs; } @@ -2909,7 +2911,7 @@ sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type) return oflgs; } xflgs = oflgs; - nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED; + nflgs |= oflgs & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC); } } @@ -3044,6 +3046,8 @@ aux_thread(void *unused) return NULL; } +static void suspend_scheduler(ErtsSchedulerData *esdp); + #endif /* ERTS_SMP */ static void @@ -3209,8 +3213,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) } - if (flgs & ~ERTS_SSI_FLG_SUSPENDED) - erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) + erts_smp_atomic32_read_band_nob(&ssi->flags, + (ERTS_SSI_FLG_SUSPENDED + | ERTS_SSI_FLG_MSB_EXEC)); if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); @@ -3409,8 +3415,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_smp_runq_lock(rq); } clear_sys_scheduling(); - if (flgs & ~ERTS_SSI_FLG_SUSPENDED) - erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) + erts_smp_atomic32_read_band_nob(&ssi->flags, + (ERTS_SSI_FLG_SUSPENDED + | ERTS_SSI_FLG_MSB_EXEC)); #endif if (!working) sched_wall_time_change(esdp, working = 1); @@ -3433,17 +3441,54 @@ ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi) oflgs = erts_smp_atomic32_cmpxchg_relb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return oflgs; - nflgs = oflgs & ERTS_SSI_FLG_SUSPENDED; + nflgs = oflgs & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC); xflgs = oflgs; } } +static ERTS_INLINE void +ssi_wake(ErtsSchedulerSleepInfo *ssi) +{ + erts_sched_finish_poke(ssi, ssi_flags_set_wake(ssi)); +} + +#ifdef ERTS_DIRTY_SCHEDULERS + static void -wake_scheduler(ErtsRunQueue *rq) +dcpu_sched_ix_suspend_wake(Uint ix) { - ErtsSchedulerSleepInfo *ssi; - erts_aint32_t flgs; + ErtsSchedulerSleepInfo* ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + ssi_wake(ssi); +} + +static void +dio_sched_ix_suspend_wake(Uint ix) +{ + ErtsSchedulerSleepInfo* ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + ssi_wake(ssi); +} + +static void +dcpu_sched_ix_wake(Uint ix) +{ + ssi_wake(ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix)); +} + +#if 0 +static void +dio_sched_ix_wake(Uint ix) +{ + ssi_wake(ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix)); +} +#endif +#endif + +static void +wake_scheduler(ErtsRunQueue *rq) +{ /* * The unlocked run queue is not strictly necessary * from a thread safety or deadlock prevention @@ -3455,10 +3500,7 @@ wake_scheduler(ErtsRunQueue *rq) ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq) || ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); - ssi = rq->scheduler->ssi; - - flgs = ssi_flags_set_wake(ssi); - erts_sched_finish_poke(ssi, flgs); + ssi_wake(rq->scheduler->ssi); } #ifdef ERTS_DIRTY_SCHEDULERS @@ -3505,6 +3547,13 @@ wake_dirty_schedulers(ErtsRunQueue *rq, int one) } while (ssi); } } + +static void +wake_dirty_scheduler(ErtsRunQueue *rq) +{ + wake_dirty_schedulers(rq, 1); +} + #endif #define ERTS_NO_USED_RUNQS_SHIFT 16 @@ -3645,7 +3694,7 @@ smp_notify_inc_runq(ErtsRunQueue *runq) if (runq) { #ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) - wake_dirty_schedulers(runq, 1); + wake_dirty_scheduler(runq); else #endif wake_scheduler(runq); @@ -3949,8 +3998,7 @@ suspend_run_queue(ErtsRunQueue *rq) wake_scheduler(rq); } -static void scheduler_ix_resume_wake(Uint ix); -static void scheduler_ssi_resume_wake(ErtsSchedulerSleepInfo *ssi); +static void nrml_sched_ix_resume_wake(Uint ix); static ERTS_INLINE void resume_run_queue(ErtsRunQueue *rq) @@ -3958,16 +4006,19 @@ resume_run_queue(ErtsRunQueue *rq) int pix; Uint32 oflgs; + ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); + erts_smp_runq_lock(rq); oflgs = ERTS_RUNQ_FLGS_READ_BSET(rq, (ERTS_RUNQ_FLG_OUT_OF_WORK | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK - | ERTS_RUNQ_FLG_SUSPENDED), + | ERTS_RUNQ_FLG_SUSPENDED + | ERTS_RUNQ_FLG_MSB_EXEC), (ERTS_RUNQ_FLG_OUT_OF_WORK | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); - if (oflgs & ERTS_RUNQ_FLG_SUSPENDED) { + if (oflgs & (ERTS_RUNQ_FLG_SUSPENDED|ERTS_RUNQ_FLG_MSB_EXEC)) { erts_aint32_t len; rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; @@ -3986,10 +4037,7 @@ resume_run_queue(ErtsRunQueue *rq) erts_smp_runq_unlock(rq); -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) -#endif - scheduler_ix_resume_wake(rq->ix); + nrml_sched_ix_resume_wake(rq->ix); } typedef struct { @@ -4047,28 +4095,22 @@ evacuate_run_queue(ErtsRunQueue *rq, int prio_q; ErtsRunQueue *to_rq; ErtsMigrationPaths *mps; - ErtsMigrationPath *mp = NULL; + ErtsMigrationPath *mp; ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) -#endif - { - mps = erts_get_migration_paths_managed(); - mp = &mps->mpath[rq->ix]; - } + ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); + + mps = erts_get_migration_paths_managed(); + mp = &mps->mpath[rq->ix]; /* Evacuate scheduled misc ops */ if (rq->misc.start) { ErtsMiscOpList *start, *end; -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); -#endif to_rq = mp->misc_evac_runq; if (!to_rq) return; @@ -4077,6 +4119,7 @@ evacuate_run_queue(ErtsRunQueue *rq, end = rq->misc.end; rq->misc.start = NULL; rq->misc.end = NULL; + ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP); erts_smp_runq_unlock(rq); erts_smp_runq_lock(to_rq); @@ -4097,9 +4140,6 @@ evacuate_run_queue(ErtsRunQueue *rq, if (rq->ports.start) { Port *prt; -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); -#endif to_rq = mp->prio[ERTS_PORT_PRIO_LEVEL].runq; if (!to_rq) return; @@ -4137,15 +4177,10 @@ evacuate_run_queue(ErtsRunQueue *rq, int notify = 0; to_rq = NULL; -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) -#endif - { - if (!mp->prio[prio_q].runq) - return; - if (prio_q == PRIORITY_NORMAL && !mp->prio[PRIORITY_LOW].runq) - return; - } + if (!mp->prio[prio_q].runq) + return; + if (prio_q == PRIORITY_NORMAL && !mp->prio[PRIORITY_LOW].runq) + return; proc = dequeue_process(rq, prio_q, &state); while (proc) { @@ -4201,11 +4236,6 @@ evacuate_run_queue(ErtsRunQueue *rq, goto handle_next_proc; } -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) - clear_proc_dirty_queue_bit(real_proc, rq, qbit); -#endif - if (ERTS_PSFLG_BOUND & real_state) { /* Bound processes get stuck here... */ proc->next = NULL; @@ -4219,16 +4249,7 @@ evacuate_run_queue(ErtsRunQueue *rq, int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state); erts_smp_runq_unlock(rq); -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) - /* - * dirty run queues evacuate only to run - * queue 0 during multi-scheduling blocking - */ - to_rq = ERTS_RUNQ_IX(0); - else -#endif - to_rq = mp->prio[prio].runq; + to_rq = mp->prio[prio].runq; RUNQ_SET_RQ(&proc->run_queue, to_rq); erts_smp_runq_lock(to_rq); @@ -4263,7 +4284,7 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq, erts_smp_runq_lock(vrq); - if (rq->halt_in_progress) + if (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_HALTING) goto no_procs; /* @@ -4362,8 +4383,7 @@ check_possible_steal_victim(ErtsRunQueue *rq, int *rq_lockedp, int vix) { ErtsRunQueue *vrq = ERTS_RUNQ_IX(vix); Uint32 flags = ERTS_RUNQ_FLGS_GET(vrq); - if ((flags & (ERTS_RUNQ_FLG_NONEMPTY - | ERTS_RUNQ_FLG_PROTECTED)) == ERTS_RUNQ_FLG_NONEMPTY) + if (runq_got_work_to_execute_flags(flags) & (!(flags & ERTS_RUNQ_FLG_PROTECTED))) return try_steal_task_from_victim(rq, rq_lockedp, vrq, flags); else return 0; @@ -4431,11 +4451,9 @@ try_steal_task(ErtsRunQueue *rq) if (!rq_locked) erts_smp_runq_lock(rq); - if (!res) - res = rq->halt_in_progress ? - !ERTS_EMPTY_RUNQ_PORTS(rq) : !ERTS_EMPTY_RUNQ(rq); - - return res; + if (res) + return res; + return runq_got_work_to_execute(rq); } /* Run queue balancing */ @@ -5347,7 +5365,7 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags) #ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) { if (rq->waiting) { - wake_dirty_schedulers(rq, 1); + wake_dirty_scheduler(rq); } } else #endif @@ -5710,13 +5728,29 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER, MAX_REG * sizeof(FloatDef)); #ifdef ERTS_DIRTY_SCHEDULERS + esdp->run_queue = runq; if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) { esdp->no = 0; + if (runq == ERTS_DIRTY_CPU_RUNQ) + esdp->type = ERTS_SCHED_DIRTY_CPU; + else { + ASSERT(runq == ERTS_DIRTY_IO_RUNQ); + esdp->type = ERTS_SCHED_DIRTY_IO; + } ERTS_DIRTY_SCHEDULER_NO(esdp) = (Uint) num; + if (num == 1) { + /* + * Multi-scheduling block functionality depends + * on finding dirty scheduler number 1 here... + */ + runq->scheduler = esdp; + } } else { + esdp->type = ERTS_SCHED_NORMAL; esdp->no = (Uint) num; ERTS_DIRTY_SCHEDULER_NO(esdp) = 0; + runq->scheduler = esdp; } esdp->dirty_shadow_process = shadow_proc; if (shadow_proc) { @@ -5728,6 +5762,8 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, shadow_proc->static_flags = ERTS_STC_FLG_SHADOW_PROC; } #else + runq->scheduler = esdp; + esdp->run_queue = runq; esdp->no = (Uint) num; #endif @@ -5740,9 +5776,6 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, erts_init_atom_cache_map(&esdp->atom_cache_map); - esdp->run_queue = runq; - esdp->run_queue->scheduler = esdp; - esdp->last_monotonic_time = 0; esdp->check_time_reds = 0; @@ -5858,7 +5891,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online erts_smp_atomic32_set_nob(&rq->len, 0); rq->wakeup_other = 0; rq->wakeup_other_reds = 0; - rq->halt_in_progress = 0; rq->procs.pending_exiters = NULL; rq->procs.context_switches = 0; @@ -6974,15 +7006,8 @@ resume_process(Process *p, ErtsProcLocks locks) #ifdef ERTS_SMP -static void -scheduler_ix_resume_wake(Uint ix) -{ - ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); - scheduler_ssi_resume_wake(ssi); -} - -static void -scheduler_ssi_resume_wake(ErtsSchedulerSleepInfo *ssi) +static ERTS_INLINE void +sched_resume_wake__(ErtsSchedulerSleepInfo *ssi) { erts_aint32_t xflgs = (ERTS_SSI_FLG_SLEEPING | ERTS_SSI_FLG_TSE_SLEEPING @@ -6996,9 +7021,31 @@ scheduler_ssi_resume_wake(ErtsSchedulerSleepInfo *ssi) break; } xflgs = oflgs; - } while (oflgs & ERTS_SSI_FLG_SUSPENDED); + } while (oflgs & (ERTS_SSI_FLG_MSB_EXEC|ERTS_SSI_FLG_SUSPENDED)); +} + +static void +nrml_sched_ix_resume_wake(Uint ix) +{ + sched_resume_wake__(ERTS_SCHED_SLEEP_INFO_IX(ix)); } +#ifdef ERTS_DIRTY_SCHEDULERS + +static void +dcpu_sched_ix_resume_wake(Uint ix) +{ + sched_resume_wake__(ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix)); +} + +static void +dio_sched_ix_resume_wake(Uint ix) +{ + sched_resume_wake__(ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix)); +} + +#endif + static erts_aint32_t sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, erts_aint32_t xpct) { @@ -7078,9 +7125,18 @@ static void init_scheduler_suspend(void) { erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd"); - schdlr_sspnd.online = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0); - schdlr_sspnd.curr_online = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0); - schdlr_sspnd.active = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0); + schdlr_sspnd.online.normal = 1; + schdlr_sspnd.curr_online.normal = 1; + schdlr_sspnd.active.normal = 1; +#ifdef ERTS_DIRTY_SCHEDULERS + schdlr_sspnd.online.dirty_cpu = 0; + schdlr_sspnd.curr_online.dirty_cpu = 0; + schdlr_sspnd.active.dirty_cpu = 0; + schdlr_sspnd.online.dirty_io = 0; + schdlr_sspnd.curr_online.dirty_io = 0; + schdlr_sspnd.active.dirty_io = 0; + schdlr_sspnd.last_msb_dirty_type = ERTS_SCHED_DIRTY_IO; +#endif erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0); schdlr_sspnd.chngq = NULL; schdlr_sspnd.changer = am_false; @@ -7141,6 +7197,252 @@ schdlr_sspnd_resume_procs(ErtsSchedType sched_type, } } +#ifdef ERTS_DIRTY_SCHEDULERS + +static ERTS_INLINE int +have_dirty_work(void) +{ + return !(ERTS_EMPTY_RUNQ(ERTS_DIRTY_CPU_RUNQ) + | ERTS_EMPTY_RUNQ(ERTS_DIRTY_IO_RUNQ)); +} + +#define ERTS_MSB_NONE_PRIO_BIT PORT_BIT + +static ERTS_INLINE Uint32 +msb_runq_prio_bit(Uint32 flgs) +{ + int pbit; + + pbit = (int) (flgs & ERTS_RUNQ_FLGS_PROCS_QMASK); + if (flgs & PORT_BIT) { + /* rate ports as proc prio high */ + pbit |= HIGH_BIT; + } + if (flgs & ERTS_RUNQ_FLG_MISC_OP) { + /* rate misc ops as proc prio normal */ + pbit |= NORMAL_BIT; + } + if (flgs & LOW_BIT) { + /* rate low prio as normal (avoid starvation) */ + pbit |= NORMAL_BIT; + } + if (!pbit) + pbit = (int) ERTS_MSB_NONE_PRIO_BIT; + else + pbit &= -pbit; /* least significant bit set... */ + ASSERT(pbit); + + /* High prio low value; low prio high value... */ + return (Uint32) pbit; +} + +static ERTS_INLINE void +msb_runq_prio_bits(Uint32 *nrmlp, Uint32 *dcpup, Uint32 *diop) +{ + Uint32 flgs = ERTS_RUNQ_FLGS_GET(ERTS_RUNQ_IX(0)); + if (flgs & ERTS_RUNQ_FLG_HALTING) { + /* + * Emulator is halting; only execute port jobs + * on normal scheduler. Ensure that we switch + * to the normal scheduler. + */ + *nrmlp = HIGH_BIT; + *dcpup = ERTS_MSB_NONE_PRIO_BIT; + *diop = ERTS_MSB_NONE_PRIO_BIT; + } + else { + *nrmlp = msb_runq_prio_bit(flgs); + + flgs = ERTS_RUNQ_FLGS_GET(ERTS_DIRTY_CPU_RUNQ); + *dcpup = msb_runq_prio_bit(flgs); + + flgs = ERTS_RUNQ_FLGS_GET(ERTS_DIRTY_IO_RUNQ); + *diop = msb_runq_prio_bit(flgs); + } +} + +static int +msb_scheduler_type_switch(ErtsSchedType sched_type, + ErtsSchedulerData *esdp, + long no) +{ + Uint32 nrml_prio, dcpu_prio, dio_prio; + ErtsSchedType exec_type; + ErtsRunQueue *exec_rq; +#ifdef DEBUG + erts_aint32_t dbg_val; +#endif + + ASSERT(schdlr_sspnd.msb.ongoing); + + /* + * This function determines how to switch + * between scheduler types when multi-scheduling + * is blocked. + * + * If no dirty work exist, we always select + * execution of normal scheduler. If nothing + * executes, normal scheduler 1 should be waiting + * in sys_schedule(), otherwise we cannot react + * on I/O events. + * + * We unconditionally switch back to normal + * scheduler after executing dirty in order to + * make sure we check for I/O... + */ + + msb_runq_prio_bits(&nrml_prio, &dcpu_prio, &dio_prio); + + exec_type = ERTS_SCHED_NORMAL; + if (sched_type == ERTS_SCHED_NORMAL) { + + /* + * Check priorities of work in the + * different run-queues and determine + * run-queue with highest prio job... + */ + + if ((dcpu_prio == ERTS_MSB_NONE_PRIO_BIT) + & (dio_prio == ERTS_MSB_NONE_PRIO_BIT)) { + /* + * No dirty work exist; continue on normal + * scheduler... + */ + return 0; + } + + if (dcpu_prio < nrml_prio) { + exec_type = ERTS_SCHED_DIRTY_CPU; + if (dio_prio < dcpu_prio) + exec_type = ERTS_SCHED_DIRTY_IO; + } + else { + if (dio_prio < nrml_prio) + exec_type = ERTS_SCHED_DIRTY_IO; + } + + /* + * Make sure to alternate between dirty types + * inbetween normal execution if highest + * priorities are equal. + */ + + if (exec_type == ERTS_SCHED_NORMAL) { + if (dcpu_prio == nrml_prio) + exec_type = ERTS_SCHED_DIRTY_CPU; + else if (dio_prio == nrml_prio) + exec_type = ERTS_SCHED_DIRTY_IO; + else { + /* + * Normal work has higher prio than + * dirty work; continue on normal + * scheduler... + */ + return 0; + } + } + + ASSERT(exec_type != ERTS_SCHED_NORMAL); + if (dio_prio == dcpu_prio) { + /* Alter between dirty types... */ + if (schdlr_sspnd.last_msb_dirty_type == ERTS_SCHED_DIRTY_IO) + exec_type = ERTS_SCHED_DIRTY_CPU; + else + exec_type = ERTS_SCHED_DIRTY_IO; + } + } + + ASSERT(sched_type != exec_type); + + if (exec_type != ERTS_SCHED_NORMAL) + schdlr_sspnd.last_msb_dirty_type = exec_type; + else { + erts_aint32_t calls; + /* + * Going back to normal scheduler after + * dirty execution; make sure it will check + * for I/O... + */ + if (ERTS_USE_MODIFIED_TIMING()) + calls = ERTS_MODIFIED_TIMING_INPUT_REDS + 1; + else + calls = INPUT_REDUCTIONS + 1; + erts_smp_atomic32_set_nob(&function_calls, calls); + + if ((nrml_prio == ERTS_MSB_NONE_PRIO_BIT) + & ((dcpu_prio != ERTS_MSB_NONE_PRIO_BIT) + | (dio_prio != ERTS_MSB_NONE_PRIO_BIT))) { + /* + * We have dirty work, but an empty + * normal run-queue. + * + * Since the normal run-queue is + * empty, the normal scheduler will + * go to sleep when selected for + * execution. We have dirty work to + * do, so we only want it to check + * I/O, and then come back here and + * switch to dirty execution. + * + * To prevent the scheduler from going + * to sleep we trick it into believing + * it has work to do... + */ + ERTS_RUNQ_FLGS_SET_NOB(ERTS_RUNQ_IX(0), + ERTS_RUNQ_FLG_MISC_OP); + } + } + + /* + * Suspend this scheduler and wake up scheduler + * number one of another type... + */ +#ifdef DEBUG + dbg_val = +#else + (void) +#endif + erts_smp_atomic32_read_bset_mb(&esdp->ssi->flags, + (ERTS_SSI_FLG_SUSPENDED + | ERTS_SSI_FLG_MSB_EXEC), + ERTS_SSI_FLG_SUSPENDED); + ASSERT(dbg_val & ERTS_SSI_FLG_MSB_EXEC); + + switch (exec_type) { + case ERTS_SCHED_NORMAL: + exec_rq = ERTS_RUNQ_IX(0); + break; + case ERTS_SCHED_DIRTY_CPU: + exec_rq = ERTS_DIRTY_CPU_RUNQ; + break; + case ERTS_SCHED_DIRTY_IO: + exec_rq = ERTS_DIRTY_IO_RUNQ; + break; + default: + ERTS_INTERNAL_ERROR("Invalid scheduler type"); + exec_rq = NULL; + break; + } + +#ifdef DEBUG + dbg_val = +#else + (void) +#endif + erts_smp_atomic32_read_bset_mb(&exec_rq->scheduler->ssi->flags, + (ERTS_SSI_FLG_SUSPENDED + | ERTS_SSI_FLG_MSB_EXEC), + ERTS_SSI_FLG_MSB_EXEC); + ASSERT(dbg_val & ERTS_SSI_FLG_SUSPENDED); + + wake_scheduler(exec_rq); + + return 1; /* suspend this scheduler... */ + +} + +#endif + static void suspend_scheduler(ErtsSchedulerData *esdp) { @@ -7166,32 +7468,49 @@ suspend_scheduler(ErtsSchedulerData *esdp) * Regardless of why a scheduler is suspended, it ends up here. */ -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { + +#if !defined(ERTS_DIRTY_SCHEDULERS) + + sched_type = ERTS_SCHED_NORMAL; + online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN; + no = esdp->no; + ASSERT(no != 1); + +#else + + sched_type = esdp->type; + switch (sched_type) { + case ERTS_SCHED_NORMAL: + online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN; + no = esdp->no; + break; + case ERTS_SCHED_DIRTY_CPU: + online_flag = ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN; + no = ERTS_DIRTY_SCHEDULER_NO(esdp); + break; + case ERTS_SCHED_DIRTY_IO: + online_flag = 0; no = ERTS_DIRTY_SCHEDULER_NO(esdp); - if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) { - online_flag = ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN; - sched_type = ERTS_SCHED_DIRTY_CPU; - } - else { - online_flag = 0; - sched_type = ERTS_SCHED_DIRTY_IO; - } + break; + default: + ERTS_INTERNAL_ERROR("Invalid scheduler type"); + return; } - else -#endif - { - online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN; - no = esdp->no; - sched_type = ERTS_SCHED_NORMAL; + + if (erts_smp_atomic32_read_nob(&ssi->flags) & ERTS_SSI_FLG_MSB_EXEC) { + ASSERT(no == 1); + if (!msb_scheduler_type_switch(sched_type, esdp, no)) + return; + /* Suspend and let scheduler 1 of another type execute... */ } - ASSERT(sched_type != ERTS_SCHED_NORMAL || no != 1); +#endif if (sched_type != ERTS_SCHED_NORMAL) erts_smp_runq_unlock(esdp->run_queue); else { - evacuate_run_queue(esdp->run_queue, &sbp); + if (no != 1) + evacuate_run_queue(esdp->run_queue, &sbp); erts_smp_runq_unlock(esdp->run_queue); @@ -7200,8 +7519,6 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_inactive); - sched_wall_time_change(esdp, 0); - } erts_smp_mtx_lock(&schdlr_sspnd.mtx); @@ -7210,9 +7527,6 @@ suspend_scheduler(ErtsSchedulerData *esdp) schdlr_sspnd_dec_nscheds(&schdlr_sspnd.active, sched_type); - ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, - ERTS_SCHED_NORMAL) >= 1); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); while (1) { @@ -7234,9 +7548,13 @@ suspend_scheduler(ErtsSchedulerData *esdp) ERTS_SCHED_NORMAL) == 1) { clr_flg = ERTS_SCHDLR_SSPND_CHNG_NMSB; } - else if (schdlr_sspnd.active - == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)) { - clr_flg = ERTS_SCHDLR_SSPND_CHNG_MSB; + else if (schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL) == 1 + && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_DIRTY_CPU) == 0 + && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_DIRTY_IO) == 0) { + clr_flg = ERTS_SCHDLR_SSPND_CHNG_MSB; } if (clr_flg) { @@ -7308,10 +7626,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) } } - if (curr_online - && (sched_type == ERTS_SCHED_NORMAL - ? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) - : !schdlr_sspnd.msb.ongoing)) { + if (curr_online) { flgs = erts_smp_atomic32_read_acqb(&ssi->flags); if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) break; @@ -7327,13 +7642,11 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (sched_type != ERTS_SCHED_NORMAL) aux_work = 0; else { - erts_aint32_t qmask; - qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue) - & ERTS_RUNQ_FLGS_QMASK); + int evacuate = no == 1 ? 0 : !ERTS_EMPTY_RUNQ(esdp->run_queue); aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work|qmask) { + if (aux_work|evacuate) { if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); sched_wall_time_change(esdp, 1); @@ -7345,7 +7658,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); - if (qmask) { + if (evacuate) { erts_smp_runq_lock(esdp->run_queue); evacuate_run_queue(esdp->run_queue, &sbp); erts_smp_runq_unlock(esdp->run_queue); @@ -7460,7 +7773,8 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (changing) { if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) && !schdlr_sspnd.msb.ongoing - && schdlr_sspnd.online == schdlr_sspnd.active) { + && schdlr_sspnd_eq_nscheds(&schdlr_sspnd.online, + &schdlr_sspnd.active)) { erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, ~ERTS_SCHDLR_SSPND_CHNG_MSB); } @@ -7475,19 +7789,16 @@ suspend_scheduler(ErtsSchedulerData *esdp) } } ASSERT(no <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, sched_type)); - ASSERT((sched_type == ERTS_SCHED_NORMAL - ? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) - : !schdlr_sspnd.msb.ongoing)); } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - ASSERT(!resume.msb.chngrs); schdlr_sspnd_resume_procs(sched_type, &resume); ASSERT(curr_online); - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (sched_type == ERTS_SCHED_NORMAL) { + (void) erts_get_monotonic_time(esdp); if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_active); @@ -7497,8 +7808,6 @@ suspend_scheduler(ErtsSchedulerData *esdp) } } - if (sched_type == ERTS_SCHED_NORMAL) - (void) erts_get_monotonic_time(esdp); erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); @@ -7768,10 +8077,8 @@ erts_set_schedulers_online(Process *p, erts_sched_poke(ssi); } } else { - for (ix = dirty_online; ix < dirty_no; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - scheduler_ssi_resume_wake(ssi); - } + for (ix = dirty_online; ix < dirty_no; ix++) + dcpu_sched_ix_resume_wake(ix); } } if (!dirty_only) @@ -7799,19 +8106,19 @@ erts_set_schedulers_online(Process *p, else /* if decrease */ { #ifdef ERTS_DIRTY_SCHEDULERS if (change_dirty) { - ErtsSchedulerSleepInfo* ssi; if (schdlr_sspnd.msb.ongoing) { - for (ix = dirty_no; ix < dirty_online; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_sched_poke(ssi); - } - } else { - for (ix = dirty_no; ix < dirty_online; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); + for (ix = dirty_no; ix < dirty_online; ix++) + erts_sched_poke(ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix)); + } + else { + for (ix = dirty_no; ix < dirty_online; ix++) + dcpu_sched_ix_suspend_wake(ix); + /* + * Newly suspended scheduler may have just been + * about to handle a task. Make sure someone takes + * care of such a task... + */ + dcpu_sched_ix_wake(0); } } if (!dirty_only) @@ -7874,9 +8181,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal { int resume_proc, ix, res, have_unlocked_plocks = 0; ErtsProcList *plp; -#ifdef ERTS_DIRTY_SCHEDULERS - ErtsSchedulerSleepInfo* ssi; -#endif ErtsMultiSchedulingBlock *msbp; erts_aint32_t chng_flg; int have_blckd_flg; @@ -7923,8 +8227,9 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal erts_proclist_store_last(&msbp->blckrs, plp); p->flags |= have_blckd_flg; ASSERT(normal - ? 1 == schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, ERTS_SCHED_NORMAL) - : schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)); + ? 1 == schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL) + : schdlr_sspnd_get_nscheds_tot(&schdlr_sspnd.active) == 1); ASSERT(erts_proc_sched_data(p)->no == 1); if (schdlr_sspnd.msb.ongoing) res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; @@ -7942,59 +8247,41 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal } ASSERT(!msbp->ongoing); msbp->ongoing = 1; - if (schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0) - || (normal && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, - ERTS_SCHED_NORMAL) == 1)) { - ASSERT(erts_proc_sched_data(p)->no == 1); - plp = proclist_create(p); - erts_proclist_store_last(&msbp->blckrs, plp); - if (schdlr_sspnd.msb.ongoing) - res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; - else - res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED; - } - else { - erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, - chng_flg); - change_no_used_runqs(1); - for (ix = 1; ix < erts_no_run_queues; ix++) - suspend_run_queue(ERTS_RUNQ_IX(ix)); - for (ix = 1; ix < online; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - wake_scheduler(rq); - } + erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, + chng_flg); + change_no_used_runqs(1); + for (ix = 1; ix < erts_no_run_queues; ix++) + suspend_run_queue(ERTS_RUNQ_IX(ix)); -#ifdef ERTS_DIRTY_SCHEDULERS - if (!normal) { - for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); + for (ix = 1; ix < online; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + wake_scheduler(rq); + } - for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { - ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0); - } +#ifdef ERTS_DIRTY_SCHEDULERS + if (!normal) { + ERTS_RUNQ_FLGS_SET_NOB(ERTS_RUNQ_IX(0), ERTS_RUNQ_FLG_MSB_EXEC); + erts_smp_atomic32_read_bor_nob(&ERTS_RUNQ_IX(0)->scheduler->ssi->flags, + ERTS_SSI_FLG_MSB_EXEC); + for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) + dcpu_sched_ix_suspend_wake(ix); + for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) + dio_sched_ix_suspend_wake(ix); + } #endif - wait_until_msb: + wait_until_msb: - ASSERT(chng_flg & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)); + ASSERT(chng_flg & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)); - plp = proclist_create(p); - erts_proclist_store_last(&msbp->chngq, plp); - resume_proc = 0; - if (schdlr_sspnd.msb.ongoing) - res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED; - else - res = ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED; - } + plp = proclist_create(p); + erts_proclist_store_last(&msbp->chngq, plp); + resume_proc = 0; + if (schdlr_sspnd.msb.ongoing) + res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED; + else + res = ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED; ASSERT(erts_proc_sched_data(p)); } } @@ -8026,27 +8313,19 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal } } if (!msbp->blckrs && !msbp->chngq) { - int online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, - ERTS_SCHED_NORMAL); + int online; erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, chng_flg); p->flags &= ~have_blckd_flg; msbp->ongoing = 0; - if (online == 1) { - /* No normal schedulers to resume */ - ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, - ERTS_SCHED_NORMAL) == 1); -#ifndef ERTS_DIRTY_SCHEDULERS - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~chng_flg); -#endif - } - else if (!(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)) { - if (plocks) { + if (!(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)) { + if (plocks) { have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } + online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL); change_no_used_runqs(online); /* Resume all online run queues */ @@ -8057,19 +8336,15 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal suspend_run_queue(ERTS_RUNQ_IX(ix)); } #ifdef ERTS_DIRTY_SCHEDULERS - if (!normal) { - ASSERT(!schdlr_sspnd.msb.ongoing); + if (!schdlr_sspnd.msb.ongoing) { + /* Get rid of msb-exec flag in run-queue of scheduler 1 */ + resume_run_queue(ERTS_RUNQ_IX(0)); online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, ERTS_SCHED_DIRTY_CPU); - for (ix = 0; ix < online; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - scheduler_ssi_resume_wake(ssi); - } - - for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { - ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); - scheduler_ssi_resume_wake(ssi); - } + for (ix = 0; ix < online; ix++) + dcpu_sched_ix_resume_wake(ix); + for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) + dio_sched_ix_resume_wake(ix); } #endif } @@ -9734,7 +10009,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) if (!is_normal_sched) { if (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) - & ERTS_SSI_FLG_SUSPENDED) { + & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) { suspend_scheduler(esdp); } } @@ -9744,8 +10019,10 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) ASSERT(is_normal_sched); - if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND|ERTS_RUNQ_FLG_SUSPENDED)) { - if (flags & ERTS_RUNQ_FLG_SUSPENDED) { + if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND + | ERTS_RUNQ_FLG_SUSPENDED + | ERTS_RUNQ_FLG_MSB_EXEC)) { + if (flags & (ERTS_RUNQ_FLG_SUSPENDED|ERTS_RUNQ_FLG_MSB_EXEC)) { (void) ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_EXEC); suspend_scheduler(esdp); flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC); @@ -9784,13 +10061,12 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) flags = ERTS_RUNQ_FLGS_GET_NOB(rq); - if (!is_normal_sched && rq->halt_in_progress) { + if (!is_normal_sched & !!(flags & ERTS_RUNQ_FLG_HALTING)) { /* Wait for emulator to terminate... */ while (1) erts_milli_sleep(1000*1000); } - else if ((!(flags & ERTS_RUNQ_FLGS_QMASK) && !rq->misc.start) - || (rq->halt_in_progress && ERTS_EMPTY_RUNQ_PORTS(rq))) { + else if (!runq_got_work_to_execute_flags(flags)) { /* Prepare for scheduler wait */ #ifdef ERTS_SMP ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); @@ -9804,21 +10080,44 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) if (flags & ERTS_RUNQ_FLG_INACTIVE) empty_runq(rq); else { - if (is_normal_sched && try_steal_task(rq)) - goto continue_check_activities_to_run; - - empty_runq(rq); - - /* - * Check for ERTS_RUNQ_FLG_SUSPENDED has to be done - * after trying to steal a task. - */ - flags = ERTS_RUNQ_FLGS_GET_NOB(rq); - if (flags & ERTS_RUNQ_FLG_SUSPENDED) { - non_empty_runq(rq); - flags |= ERTS_RUNQ_FLG_NONEMPTY; - goto continue_check_activities_to_run_known_flags; + ASSERT(!runq_got_work_to_execute(rq)); + if (!is_normal_sched) { + /* Dirty scheduler */ + if (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) + & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) { + /* Go suspend... */ + goto continue_check_activities_to_run_known_flags; + } + } + else { + /* Normal scheduler */ + if (try_steal_task(rq)) + goto continue_check_activities_to_run; + /* + * Check for suspend has to be done after trying + * to steal a task... + */ + flags = ERTS_RUNQ_FLGS_GET_NOB(rq); + if ((flags & ERTS_RUNQ_FLG_SUSPENDED) +#ifdef ERTS_DIRTY_SCHEDULERS + /* If multi scheduling block and we have + * dirty work, suspend and let dirty + * scheduler handle work... */ + || ((((flags & (ERTS_RUNQ_FLG_HALTING + | ERTS_RUNQ_FLG_MSB_EXEC)) + == ERTS_RUNQ_FLG_MSB_EXEC)) + && have_dirty_work()) +#endif + ) { + non_empty_runq(rq); + flags |= ERTS_RUNQ_FLG_NONEMPTY; + /* + * Go suspend... + */ + goto continue_check_activities_to_run_known_flags; + } } + empty_runq(rq); } #endif @@ -9870,7 +10169,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) #endif } - if (rq->misc.start) + if (flags & ERTS_RUNQ_FLG_MISC_OP) exec_misc_ops(rq); #ifdef ERTS_SMP @@ -9881,13 +10180,15 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) * Find a new port to run. */ - if (RUNQ_READ_LEN(&rq->ports.info.len)) { + flags = ERTS_RUNQ_FLGS_GET_NOB(rq); + + if (flags & PORT_BIT) { int have_outstanding_io; have_outstanding_io = erts_port_task_execute(rq, &esdp->current_port); if ((!erts_eager_check_io && have_outstanding_io && fcalls > 2*input_reductions) - || rq->halt_in_progress) { + || (flags & ERTS_RUNQ_FLG_HALTING)) { /* * If we have performed more than 2*INPUT_REDUCTIONS since * last call to erl_sys_schedule() and we still haven't @@ -11419,6 +11720,8 @@ erts_schedule_misc_op(void (*func)(void *), void *arg) non_empty_runq(rq); #endif + ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP); + erts_smp_runq_unlock(rq); smp_notify_inc_runq(rq); @@ -11449,6 +11752,9 @@ exec_misc_ops(ErtsRunQueue *rq) rq->misc.end = NULL; } + if (!rq->misc.start) + ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP); + erts_smp_runq_unlock(rq); while (molp) { @@ -13576,6 +13882,8 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { erts_print(to, to_arg, "WAITING"); break; case ERTS_SSI_FLG_SUSPENDED: erts_print(to, to_arg, "SUSPENDED"); break; + case ERTS_SSI_FLG_MSB_EXEC: + erts_print(to, to_arg, "MSB_EXEC"); break; default: erts_print(to, to_arg, "UNKNOWN(%d)", flg); break; } @@ -13685,6 +13993,12 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { erts_print(to, to_arg, "NONEMPTY"); break; case ERTS_RUNQ_FLG_PROTECTED: erts_print(to, to_arg, "PROTECTED"); break; + case ERTS_RUNQ_FLG_EXEC: + erts_print(to, to_arg, "EXEC"); break; + case ERTS_RUNQ_FLG_MSB_EXEC: + erts_print(to, to_arg, "MSB_EXEC"); break; + case ERTS_RUNQ_FLG_MISC_OP: + erts_print(to, to_arg, "MISC_OP"); break; default: erts_print(to, to_arg, "UNKNOWN(%d)", flg); break; } @@ -13734,11 +14048,11 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { * A nice system halt closing all open port goes as follows: * 1) This function schedules the aux work ERTS_SSI_AUX_WORK_REAP_PORTS * on all schedulers, then schedules itself out. - * 2) All shedulers detect this and set the flag halt_in_progress + * 2) All shedulers detect this and set the flag ERTS_RUNQ_FLG_HALTING * on their run queue. The last scheduler sets all non-closed ports * ERTS_PORT_SFLG_HALT. Global atomic erts_halt_progress is used * as refcount to determine which is last. - * 3) While the run ques has flag halt_in_progress no processes + * 3) While the run queues has flag ERTS_RUNQ_FLG_HALTING no processes * will be scheduled, only ports. * 4) When the last port closes that scheduler calls erlang:halt/1. * The same global atomic is used as refcount. @@ -13753,8 +14067,8 @@ void erts_halt(int code) erts_no_schedulers, -1)) { #ifdef ERTS_DIRTY_SCHEDULERS - ERTS_DIRTY_CPU_RUNQ->halt_in_progress = 1; - ERTS_DIRTY_IO_RUNQ->halt_in_progress = 1; + ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_CPU_RUNQ, ERTS_RUNQ_FLG_HALTING); + ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_IO_RUNQ, ERTS_RUNQ_FLG_HALTING); #endif erts_halt_code = code; notify_reap_ports_relb(); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 3938c4c913..fc2055332b 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -170,8 +170,14 @@ extern int erts_sched_thread_suggested_stack_size; (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6)) #define ERTS_RUNQ_FLG_EXEC \ (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 7)) +#define ERTS_RUNQ_FLG_MSB_EXEC \ + (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 8)) +#define ERTS_RUNQ_FLG_MISC_OP \ + (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 9)) +#define ERTS_RUNQ_FLG_HALTING \ + (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 10)) -#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 8) +#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 11) #define ERTS_RUNQ_FLGS_MIGRATION_QMASKS \ (ERTS_RUNQ_FLGS_EMIGRATE_QMASK \ @@ -262,8 +268,9 @@ typedef enum { #define ERTS_SSI_FLG_TSE_SLEEPING (((erts_aint32_t) 1) << 2) #define ERTS_SSI_FLG_WAITING (((erts_aint32_t) 1) << 3) #define ERTS_SSI_FLG_SUSPENDED (((erts_aint32_t) 1) << 4) +#define ERTS_SSI_FLG_MSB_EXEC (((erts_aint32_t) 1) << 5) -#define ERTS_SSI_FLGS_MAX 5 +#define ERTS_SSI_FLGS_MAX 6 #define ERTS_SSI_FLGS_SLEEP_TYPE \ (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING) @@ -274,7 +281,8 @@ typedef enum { #define ERTS_SSI_FLGS_ALL \ (ERTS_SSI_FLGS_SLEEP \ | ERTS_SSI_FLG_WAITING \ - | ERTS_SSI_FLG_SUSPENDED) + | ERTS_SSI_FLG_SUSPENDED \ + | ERTS_SSI_FLG_MSB_EXEC) /* * Keep ERTS_SSI_AUX_WORK flags ordered in expected frequency @@ -393,6 +401,12 @@ typedef struct { Process* last; } ErtsRunPrioQueue; +typedef enum { + ERTS_SCHED_NORMAL, + ERTS_SCHED_DIRTY_CPU, + ERTS_SCHED_DIRTY_IO +} ErtsSchedType; + typedef struct ErtsSchedulerData_ ErtsSchedulerData; typedef struct ErtsRunQueue_ ErtsRunQueue; @@ -478,7 +492,6 @@ struct ErtsRunQueue_ { erts_smp_atomic32_t len; int wakeup_other; int wakeup_other_reds; - int halt_in_progress; struct { ErtsProcList *pending_exiters; @@ -642,6 +655,7 @@ struct ErtsSchedulerData_ { #endif ErtsSchedulerSleepInfo *ssi; Process *current_process; + ErtsSchedType type; Uint no; /* Scheduler number for normal schedulers */ #ifdef ERTS_DIRTY_SCHEDULERS ErtsDirtySchedId dirty_no; /* Scheduler number for dirty schedulers */ -- cgit v1.2.3 From fd1fb9070333fe5b91f4528841a4b0fe71bbf096 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 12 Jan 2017 14:25:30 +0100 Subject: Always return info from system_info(dirty_[cpu|io]_schedulers*) --- erts/emulator/beam/erl_bif_info.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 4659cb5a7b..5f2122b969 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -2680,20 +2680,30 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) erts_schedulers_state(NULL, NULL, &active, NULL, NULL, NULL, NULL, NULL); BIF_RET(make_small(active)); #endif -#if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS) } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers", BIF_ARG_1)) { Uint dirty_cpu; +#ifdef ERTS_DIRTY_SCHEDULERS erts_schedulers_state(NULL, NULL, NULL, &dirty_cpu, NULL, NULL, NULL, NULL); +#else + dirty_cpu = 0; +#endif BIF_RET(make_small(dirty_cpu)); } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers_online", BIF_ARG_1)) { Uint dirty_cpu_onln; +#ifdef ERTS_DIRTY_SCHEDULERS erts_schedulers_state(NULL, NULL, NULL, NULL, &dirty_cpu_onln, NULL, NULL, NULL); +#else + dirty_cpu_onln = 0; +#endif BIF_RET(make_small(dirty_cpu_onln)); } else if (ERTS_IS_ATOM_STR("dirty_io_schedulers", BIF_ARG_1)) { Uint dirty_io; +#ifdef ERTS_DIRTY_SCHEDULERS erts_schedulers_state(NULL, NULL, NULL, NULL, NULL, NULL, &dirty_io, NULL); - BIF_RET(make_small(dirty_io)); +#else + dirty_io = 0; #endif + BIF_RET(make_small(dirty_io)); } else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) { res = make_small(erts_no_run_queues); BIF_RET(res); -- cgit v1.2.3 From f9459092940943876dff040ee997515b96fd5d50 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 4 Jan 2017 18:10:26 +0100 Subject: Scheduler wall time support for dirty schedulers --- erts/emulator/beam/atom.names | 3 +- erts/emulator/beam/beam_debug.c | 13 +- erts/emulator/beam/bif.c | 2 +- erts/emulator/beam/erl_bif_info.c | 7 +- erts/emulator/beam/erl_process.c | 366 +++++++++++++++++++++++++++++++------- erts/emulator/beam/erl_process.h | 9 +- 6 files changed, 327 insertions(+), 73 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 7df350116a..dd0a42b5ba 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -574,6 +574,8 @@ atom safe atom save_calls atom scheduler atom scheduler_id +atom scheduler_wall_time +atom scheduler_wall_time_all atom schedulers_online atom scheme atom scientific @@ -678,7 +680,6 @@ atom waiting atom wall_clock atom warning atom warning_msg -atom scheduler_wall_time atom wordsize atom write_concurrency atom xor diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 3ffbb0364c..8326d348af 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -994,13 +994,20 @@ dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I) Eterm *hp, *hp2; Uint sz; int i; + ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); + int dirty_io = esdp->type == ERTS_SCHED_DIRTY_IO; + if (ERTS_PROC_IS_EXITING(real_c_p)) goto badarg; dirty_send_message(c_p, arg2, am_alive); - /* Wait until dead */ - while (!ERTS_PROC_IS_EXITING(real_c_p)) - erts_thr_yield(); + /* Wait until dead */ + while (!ERTS_PROC_IS_EXITING(real_c_p)) { + if (dirty_io) + ms_wait(c_p, make_small(100), 0); + else + erts_thr_yield(); + } ms_wait(c_p, make_small(1000), 0); diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index a5b66db35c..65c370c55b 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -4601,7 +4601,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) erts_aint32_t new = BIF_ARG_2 == am_true ? 1 : 0; erts_aint32_t old = erts_smp_atomic32_xchg_nob(&sched_wall_time, new); - Eterm ref = erts_sched_wall_time_request(BIF_P, 1, new); + Eterm ref = erts_sched_wall_time_request(BIF_P, 1, new, 0, 0); ASSERT(is_value(ref)); BIF_TRAP2(await_sched_wall_time_mod_trap, BIF_P, diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 5f2122b969..5ee19aead8 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -3385,7 +3385,12 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) Eterm* hp; if (BIF_ARG_1 == am_scheduler_wall_time) { - res = erts_sched_wall_time_request(BIF_P, 0, 0); + res = erts_sched_wall_time_request(BIF_P, 0, 0, 1, 0); + if (is_non_value(res)) + BIF_RET(am_undefined); + BIF_TRAP1(gather_sched_wall_time_res_trap, BIF_P, res); + } else if (BIF_ARG_1 == am_scheduler_wall_time_all) { + res = erts_sched_wall_time_request(BIF_P, 0, 0, 1, 1); if (is_non_value(res)) BIF_RET(am_undefined); BIF_TRAP1(gather_sched_wall_time_res_trap, BIF_P, res); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 084ecb14ec..8c9aa45a81 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -804,15 +804,28 @@ erts_late_init_process(void) } +#define ERTS_SCHED_WTIME_IDLE ~((Uint64) 0) + static void -init_sched_wall_time(ErtsSchedWallTime *swtp) +init_sched_wall_time(ErtsSchedulerData *esdp, Uint64 time_stamp) { - swtp->need = erts_sched_balance_util; - swtp->enabled = 0; - swtp->start = 0; - swtp->working.total = 0; - swtp->working.start = 0; - swtp->working.currently = 0; +#ifdef ERTS_DIRTY_SCHEDULERS + if (esdp->type != ERTS_SCHED_NORMAL) { + erts_atomic32_init_nob(&esdp->sched_wall_time.u.mod, 0); + esdp->sched_wall_time.enabled = 1; + esdp->sched_wall_time.start = time_stamp; + esdp->sched_wall_time.working.total = 0; + esdp->sched_wall_time.working.start = ERTS_SCHED_WTIME_IDLE; + } + else +#endif + { + esdp->sched_wall_time.u.need = erts_sched_balance_util; + esdp->sched_wall_time.enabled = 0; + esdp->sched_wall_time.start = 0; + esdp->sched_wall_time.working.total = 0; + esdp->sched_wall_time.working.start = 0; + } } static ERTS_INLINE Uint64 @@ -1003,31 +1016,145 @@ init_runq_sched_util(ErtsRunQueueSchedUtil *rqsu, int enabled) #endif /* ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT */ -static ERTS_INLINE void +#ifdef ERTS_DIRTY_SCHEDULERS + +typedef struct { + Uint64 working; + Uint64 total; +} ErtsDirtySchedWallTime; + +static void +read_dirty_sched_wall_time(ErtsSchedulerData *esdp, ErtsDirtySchedWallTime *info) +{ + erts_aint32_t mod1; + Uint64 working, start, ts; + + mod1 = erts_atomic32_read_nob(&esdp->sched_wall_time.u.mod); + + while (1) { + erts_aint32_t mod2; + + /* Spin until values are not written... */ + while (1) { + if ((mod1 & 1) == 0) + break; + ERTS_SPIN_BODY; + mod1 = erts_atomic32_read_nob(&esdp->sched_wall_time.u.mod); + } + + ERTS_THR_READ_MEMORY_BARRIER; + + working = esdp->sched_wall_time.working.total; + start = esdp->sched_wall_time.working.start; + + ERTS_THR_READ_MEMORY_BARRIER; + + mod2 = erts_atomic32_read_nob(&esdp->sched_wall_time.u.mod); + if (mod1 == mod2) + break; + mod1 = mod2; + } + + ts = sched_wall_time_ts(); + ts -= esdp->sched_wall_time.start; + + info->total = ts; + + if (start == ERTS_SCHED_WTIME_IDLE || ts < start) + info->working = working; + else + info->working = working + (ts - start); + + if (info->working > info->total) + info->working = info->total; +} + +#endif + +#ifdef ERTS_SMP + +static void +dirty_sched_wall_time_change(ErtsSchedulerData *esdp, int working) +{ + erts_aint32_t mod; + Uint64 ts = sched_wall_time_ts(); + + ts -= esdp->sched_wall_time.start; + + /* + * This thread is the only thread writing in + * this sched_wall_time struct. We set 'mod' to + * an odd value while writing... + */ + mod = erts_atomic32_read_dirty(&esdp->sched_wall_time.u.mod); + ASSERT((mod & 1) == 0); + mod++; + + erts_atomic32_set_nob(&esdp->sched_wall_time.u.mod, mod); + ERTS_THR_WRITE_MEMORY_BARRIER; + + if (working) { + ASSERT(esdp->sched_wall_time.working.start + == ERTS_SCHED_WTIME_IDLE); + + esdp->sched_wall_time.working.start = ts; + + } + else { + Uint64 total; + + ASSERT(esdp->sched_wall_time.working.start + != ERTS_SCHED_WTIME_IDLE); + + total = esdp->sched_wall_time.working.total; + total += ts - esdp->sched_wall_time.working.start; + + esdp->sched_wall_time.working.total = total; + esdp->sched_wall_time.working.start = ERTS_SCHED_WTIME_IDLE; + + + } + + ERTS_THR_WRITE_MEMORY_BARRIER; + mod++; + erts_atomic32_set_nob(&esdp->sched_wall_time.u.mod, mod); + +#if 0 + if (!working) { + ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_BUSY_WAIT); + } else { + ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_OTHER); + } +#endif +} + +#endif /* ERTS_SMP */ + +static void sched_wall_time_change(ErtsSchedulerData *esdp, int working) { - if (esdp->sched_wall_time.need) { + if (esdp->sched_wall_time.u.need) { Uint64 ts = sched_wall_time_ts(); #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT - update_avg_sched_util(esdp, ts, working); + update_avg_sched_util(esdp, ts, working); #endif if (esdp->sched_wall_time.enabled) { if (working) { -#ifdef DEBUG - ASSERT(!esdp->sched_wall_time.working.currently); - esdp->sched_wall_time.working.currently = 1; -#endif + ASSERT(esdp->sched_wall_time.working.start + == ERTS_SCHED_WTIME_IDLE); ts -= esdp->sched_wall_time.start; esdp->sched_wall_time.working.start = ts; } else { -#ifdef DEBUG - ASSERT(esdp->sched_wall_time.working.currently); - esdp->sched_wall_time.working.currently = 0; -#endif + ASSERT(esdp->sched_wall_time.working.start + != ERTS_SCHED_WTIME_IDLE); ts -= esdp->sched_wall_time.start; ts -= esdp->sched_wall_time.working.start; esdp->sched_wall_time.working.total += ts; +#ifdef DEBUG + esdp->sched_wall_time.working.start + = ERTS_SCHED_WTIME_IDLE; +#endif } } } @@ -1047,6 +1174,10 @@ typedef struct { Eterm ref_heap[REF_THING_SIZE]; Uint req_sched; erts_smp_atomic32_t refc; +#ifdef ERTS_DIRTY_SCHEDULERS + int want_dirty_cpu; + int want_dirty_io; +#endif } ErtsSchedWallTimeReq; typedef struct { @@ -1068,6 +1199,7 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(screq, 5, ERTS_ALC_T_SYS_CHECK_REQ) + static void reply_sched_wall_time(void *vswtrp) { @@ -1091,23 +1223,23 @@ reply_sched_wall_time(void *vswtrp) #endif if (swtrp->set) { if (!swtrp->enable && esdp->sched_wall_time.enabled) { - esdp->sched_wall_time.need = erts_sched_balance_util; + esdp->sched_wall_time.u.need = erts_sched_balance_util; esdp->sched_wall_time.enabled = 0; } else if (swtrp->enable && !esdp->sched_wall_time.enabled) { Uint64 ts = sched_wall_time_ts(); - esdp->sched_wall_time.need = 1; + esdp->sched_wall_time.u.need = 1; esdp->sched_wall_time.enabled = 1; esdp->sched_wall_time.start = ts; esdp->sched_wall_time.working.total = 0; esdp->sched_wall_time.working.start = 0; - esdp->sched_wall_time.working.currently = 1; } } if (esdp->sched_wall_time.enabled) { Uint64 ts = sched_wall_time_ts(); - ASSERT(esdp->sched_wall_time.working.currently); + ASSERT(esdp->sched_wall_time.working.start + != ERTS_SCHED_WTIME_IDLE); ts -= esdp->sched_wall_time.start; total = ts; ts -= esdp->sched_wall_time.working.start; @@ -1118,30 +1250,117 @@ reply_sched_wall_time(void *vswtrp) hpp = NULL; szp = &sz; - while (1) { - if (hpp) - ref_copy = STORE_NC(hpp, ohp, swtrp->ref); - else - *szp += REF_THING_SIZE; +#ifdef ERTS_DIRTY_SCHEDULERS + if (esdp->sched_wall_time.enabled + && swtrp->req_sched == esdp->no + && (swtrp->want_dirty_cpu || swtrp->want_dirty_io)) { + /* Reply with info about this scheduler and all dirty schedulers... */ + ErtsDirtySchedWallTime *dswt; + int ix, no_dirty_scheds, want_dcpu, want_dio, soffset; + + want_dcpu = swtrp->want_dirty_cpu; + want_dio = swtrp->want_dirty_io; + + no_dirty_scheds = 0; + if (want_dcpu) + no_dirty_scheds += erts_no_dirty_cpu_schedulers; + if (want_dio) + no_dirty_scheds += erts_no_dirty_io_schedulers; + + ASSERT(no_dirty_scheds); + + dswt = erts_alloc(ERTS_ALC_T_TMP, + sizeof(ErtsDirtySchedWallTime) + * no_dirty_scheds); + + for (ix = 0; ix < no_dirty_scheds; ix++) { + ErtsSchedulerData *esdp; + if (want_dcpu && ix < erts_no_dirty_cpu_schedulers) + esdp = &erts_aligned_dirty_cpu_scheduler_data[ix].esd; + else { + int dio_ix = ix - erts_no_dirty_cpu_schedulers; + esdp = &erts_aligned_dirty_io_scheduler_data[dio_ix].esd; + } + read_dirty_sched_wall_time(esdp, &dswt[ix]); + } - if (swtrp->set) - msg = ref_copy; - else { - msg = (!esdp->sched_wall_time.enabled - ? am_notsup - : erts_bld_tuple(hpp, szp, 3, - make_small(esdp->no), - erts_bld_uint64(hpp, szp, working), - erts_bld_uint64(hpp, szp, total))); + soffset = erts_no_schedulers + 1; - msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg); - } - if (hpp) - break; + if (!want_dcpu) { + ASSERT(want_dio); + soffset += erts_no_dirty_cpu_schedulers; + } + + while (1) { + if (hpp) + ref_copy = STORE_NC(hpp, ohp, swtrp->ref); + else + *szp += REF_THING_SIZE; + + ASSERT(!swtrp->set); + + /* info about dirty schedulers... */ + msg = NIL; + for (ix = no_dirty_scheds-1; ix >= 0; ix--) { + msg = erts_bld_cons(hpp, szp, + erts_bld_tuple(hpp, szp, 3, + make_small(ix+soffset), + erts_bld_uint64(hpp, szp, + dswt[ix].working), + erts_bld_uint64(hpp, szp, + dswt[ix].total)), + msg); + } + /* info about this scheduler... */ + msg = erts_bld_cons(hpp, szp, + erts_bld_tuple(hpp, szp, 3, + make_small(esdp->no), + erts_bld_uint64(hpp, szp, working), + erts_bld_uint64(hpp, szp, total)), + msg); + + msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg); + + if (hpp) + break; + + mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); + szp = NULL; + hpp = &hp; + } - mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); - szp = NULL; - hpp = &hp; + erts_free(ERTS_ALC_T_TMP, dswt); + } + else +#endif + { + /* Reply with info about this scheduler only... */ + + while (1) { + if (hpp) + ref_copy = STORE_NC(hpp, ohp, swtrp->ref); + else + *szp += REF_THING_SIZE; + + if (swtrp->set) + msg = ref_copy; + else { + msg = (!esdp->sched_wall_time.enabled + ? am_undefined + : erts_bld_tuple(hpp, szp, 3, + make_small(esdp->no), + erts_bld_uint64(hpp, szp, working), + erts_bld_uint64(hpp, szp, total))); + + msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg); + } + if (hpp) + break; + + mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); + szp = NULL; + hpp = &hp; + } } erts_queue_message(rp, rp_locks, mp, msg, am_system); @@ -1159,7 +1378,8 @@ reply_sched_wall_time(void *vswtrp) } Eterm -erts_sched_wall_time_request(Process *c_p, int set, int enable) +erts_sched_wall_time_request(Process *c_p, int set, int enable, + int want_dirty_cpu, int want_dirty_io) { ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); Eterm ref; @@ -1181,6 +1401,10 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable) swtrp->proc = c_p; swtrp->ref = STORE_NC(&hp, NULL, ref); swtrp->req_sched = esdp->no; +#ifdef ERTS_DIRTY_SCHEDULERS + swtrp->want_dirty_cpu = want_dirty_cpu; + swtrp->want_dirty_io = want_dirty_io; +#endif erts_smp_atomic32_init_nob(&swtrp->refc, (erts_aint32_t) erts_no_schedulers); @@ -3106,8 +3330,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) tse_wait: - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && thr_prgr_active != working) - sched_wall_time_change(esdp, thr_prgr_active); + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) + dirty_sched_wall_time_change(esdp, working = 0); + else if (thr_prgr_active != working) + sched_wall_time_change(esdp, working = thr_prgr_active); while (1) { ErtsMonotonicTime current_time = 0; @@ -3218,10 +3444,12 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) (ERTS_SSI_FLG_SUSPENDED | ERTS_SSI_FLG_MSB_EXEC)); - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) { + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) + dirty_sched_wall_time_change(esdp, working = 1); + else if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); sched_wall_time_change(esdp, 1); - } + } erts_smp_runq_lock(rq); sched_active(esdp->no, rq); @@ -5712,7 +5940,8 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, ErtsSchedulerSleepInfo* ssi, ErtsRunQueue* runq, char** daww_ptr, size_t daww_sz, - Process *shadow_proc) + Process *shadow_proc, + Uint64 time_stamp) { esdp->timer_wheel = NULL; #ifdef ERTS_SMP @@ -5794,7 +6023,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, esdp->reductions = 0; - init_sched_wall_time(&esdp->sched_wall_time); + init_sched_wall_time(esdp, time_stamp); erts_port_task_handle_init(&esdp->nosuspend_port_task_handle); } @@ -6008,17 +6237,18 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online erts_aligned_scheduler_data = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, - n*sizeof(ErtsAlignedSchedulerData)); + n*sizeof(ErtsAlignedSchedulerData)); for (ix = 0; ix < n; ix++) { ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix); init_scheduler_data(esdp, ix+1, ERTS_SCHED_SLEEP_INFO_IX(ix), ERTS_RUNQ_IX(ix), &daww_ptr, daww_sz, - NULL); + NULL, 0); } #ifdef ERTS_DIRTY_SCHEDULERS { + Uint64 ts = sched_wall_time_ts(); int dirty_scheds = no_dirty_cpu_schedulers + no_dirty_io_schedulers; int adspix = 0; ErtsAlignedDirtyShadowProcess *adsp = @@ -6038,13 +6268,13 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); init_scheduler_data(esdp, ix+1, ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix), ERTS_DIRTY_CPU_RUNQ, NULL, 0, - &adsp[adspix++].dsp); + &adsp[adspix++].dsp, ts); } for (ix = 0; ix < no_dirty_io_schedulers; ix++) { ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix); init_scheduler_data(esdp, ix+1, ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix), ERTS_DIRTY_IO_RUNQ, NULL, 0, - &adsp[adspix++].dsp); + &adsp[adspix++].dsp, ts); } } #endif @@ -7506,8 +7736,10 @@ suspend_scheduler(ErtsSchedulerData *esdp) #endif - if (sched_type != ERTS_SCHED_NORMAL) + if (sched_type != ERTS_SCHED_NORMAL) { erts_smp_runq_unlock(esdp->run_queue); + dirty_sched_wall_time_change(esdp, 0); + } else { if (no != 1) evacuate_run_queue(esdp->run_queue, &sbp); @@ -7518,8 +7750,8 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_inactive); - } + erts_smp_mtx_lock(&schdlr_sspnd.mtx); flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); @@ -7797,15 +8029,17 @@ suspend_scheduler(ErtsSchedulerData *esdp) ASSERT(curr_online); - if (sched_type == ERTS_SCHED_NORMAL) { - (void) erts_get_monotonic_time(esdp); - if (erts_system_profile_flags.scheduler) - profile_scheduler(make_small(esdp->no), am_active); + if (sched_type != ERTS_SCHED_NORMAL) + dirty_sched_wall_time_change(esdp, 1); + else { + (void) erts_get_monotonic_time(esdp); + if (erts_system_profile_flags.scheduler) + profile_scheduler(make_small(esdp->no), am_active); - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); - } + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } } erts_smp_runq_lock(esdp->run_queue); @@ -8518,6 +8752,8 @@ sched_dirty_cpu_thread_func(void *vesdp) callbacks.wait = NULL; callbacks.finalize_wait = NULL; + dirty_sched_wall_time_change(esdp, 1); + esdp->thr_id += erts_no_schedulers; erts_msacc_init_thread("dirty_cpu_scheduler", no, 0); @@ -8565,6 +8801,8 @@ sched_dirty_io_thread_func(void *vesdp) callbacks.wait = NULL; callbacks.finalize_wait = NULL; + dirty_sched_wall_time_change(esdp, 1); + esdp->thr_id += erts_no_schedulers + erts_no_dirty_cpu_schedulers; erts_msacc_init_thread("dirty_io_scheduler", no, 0); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index fc2055332b..ce0989883c 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -550,13 +550,15 @@ do { \ } while (0) typedef struct { - int need; /* "+sbu true" or scheduler_wall_time enabled */ + union { + erts_atomic32_t mod; /* on dirty schedulers */ + int need; /* "+sbu true" or scheduler_wall_time enabled */ + } u; int enabled; Uint64 start; struct { Uint64 total; Uint64 start; - int currently; } working; } ErtsSchedWallTime; @@ -1598,7 +1600,8 @@ void erts_init_scheduling(int, int void erts_execute_dirty_system_task(Process *c_p); #endif int erts_set_gc_state(Process *c_p, int enable); -Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable); +Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable, + int dirty_cpu, int want_dirty_io); Eterm erts_system_check_request(Process *c_p); Eterm erts_gc_info_request(Process *c_p); Uint64 erts_get_proc_interval(void); -- cgit v1.2.3 From 505305a4b6a6372921808c4e5ec56db51b6a308b Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 18 Jan 2017 13:30:20 +0100 Subject: Remove double check of NifExport when checking process code --- erts/emulator/beam/beam_bif_load.c | 5 ----- 1 file changed, 5 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index c8dde8caf8..a5891a0ec2 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1117,11 +1117,6 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) *redsp += 1; - if (erts_check_nif_export_in_area(rp, mod_start, mod_size)) - return am_true; - - *redsp += 1; - if (erts_check_nif_export_in_area(rp, mod_start, mod_size)) return am_true; -- cgit v1.2.3 From bfcf88311828b93a833ce96ad1a518b8eca08552 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 18 Jan 2017 14:40:25 +0100 Subject: Change exception for enif_schedule_nif() with dirty flags --- erts/emulator/beam/erl_nif.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index b860759fa2..af3ca3afa3 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2506,10 +2506,13 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, if (flags == 0) result = schedule(env, execute_nif, fp, proc->current->module, fun_name_atom, argc, argv); + else if (!(flags & ~(ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND))) { #ifdef ERTS_DIRTY_SCHEDULERS - else if (!(flags & ~(ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND))) result = schedule_dirty_nif(env, flags, fp, fun_name_atom, argc, argv); +#else + result = enif_raise_exception(env, am_notsup); #endif + } else result = enif_make_badarg(env); -- cgit v1.2.3 From 40c82769def0cfa59e75669ff1c6fc4abcecd764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Thu, 15 Dec 2016 18:07:13 +0100 Subject: erts: Handle SIGTERM via signal service instead --- erts/emulator/beam/atom.names | 1 + 1 file changed, 1 insertion(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 363b17e27c..52a3413796 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -592,6 +592,7 @@ atom set_tcw_fake atom separate atom shared atom sighup +atom sigterm atom silent atom size atom sl_alloc -- cgit v1.2.3 From 2d3bf84d8167b50728b0a5411a4e2dfa71d52c10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 16 Dec 2016 12:24:13 +0100 Subject: erts: Handle SIGUSR1 via signal service instead --- erts/emulator/beam/atom.names | 1 + erts/emulator/beam/sys.h | 14 -------------- 2 files changed, 1 insertion(+), 14 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 52a3413796..668abfaaee 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -593,6 +593,7 @@ atom separate atom shared atom sighup atom sigterm +atom sigusr1 atom silent atom size atom sl_alloc diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 4b3ac594a0..ceea794cc4 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -487,20 +487,6 @@ extern volatile int erts_break_requested; void erts_do_break_handling(void); #endif -#ifdef ERTS_WANT_GOT_SIGUSR1 -# ifndef UNIX -# define ERTS_GOT_SIGUSR1 0 -# else -# ifdef ERTS_SMP -extern erts_smp_atomic32_t erts_got_sigusr1; -# define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic32_read_mb(&erts_got_sigusr1)) -# else -extern volatile int erts_got_sigusr1; -# define ERTS_GOT_SIGUSR1 erts_got_sigusr1 -# endif -# endif -#endif - #ifdef ERTS_SMP extern erts_smp_atomic32_t erts_writing_erl_crash_dump; extern erts_tsd_key_t erts_is_crash_dumping_key; -- cgit v1.2.3 From 6bebb90f7c94e1aa95aff06fc5c40dd07696edc2 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 19 Jan 2017 16:28:35 +0100 Subject: Introduce erts_proc_lookup_inc_refc() --- erts/emulator/beam/erl_process_lock.c | 35 +++++++++++++++++++++++++++++++++++ erts/emulator/beam/erl_process_lock.h | 2 ++ 2 files changed, 37 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index 180df229eb..a93f1755c8 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -1006,6 +1006,41 @@ erts_pid2proc_opt(Process *c_p, return proc; } +static ERTS_INLINE +Process *proc_lookup_inc_refc(Eterm pid, int allow_exit) +{ + Process *proc; +#ifdef ERTS_SMP + ErtsThrPrgrDelayHandle dhndl; + + dhndl = erts_thr_progress_unmanaged_delay(); +#endif + + proc = erts_proc_lookup_raw(pid); + if (proc) { + if (!allow_exit && ERTS_PROC_IS_EXITING(proc)) + proc = NULL; + else + erts_proc_inc_refc(proc); + } + +#ifdef ERTS_SMP + erts_thr_progress_unmanaged_continue(dhndl); +#endif + + return proc; +} + +Process *erts_proc_lookup_inc_refc(Eterm pid) +{ + return proc_lookup_inc_refc(pid, 0); +} + +Process *erts_proc_lookup_raw_inc_refc(Eterm pid) +{ + return proc_lookup_inc_refc(pid, 1); +} + void erts_proc_lock_init(Process *p) { diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index 2cccf0697a..773529384f 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -940,6 +940,8 @@ void erts_proc_safelock(Process *a_proc, #define erts_pid2proc(PROC, HL, PID, NL) \ erts_pid2proc_opt((PROC), (HL), (PID), (NL), 0) +Process *erts_proc_lookup_inc_refc(Eterm pid); +Process *erts_proc_lookup_raw_inc_refc(Eterm pid); ERTS_GLB_INLINE Process *erts_pix2proc(int ix); ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid); -- cgit v1.2.3 From eaecd838ebc3410efcbe0f3b1717543d6a4650b0 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 19 Jan 2017 17:24:14 +0100 Subject: Fix unused warning --- erts/emulator/beam/erl_process.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index f80ebdf31b..641a8fb3e8 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -127,11 +127,13 @@ runq_got_work_to_execute_flags(Uint32 flags) return !ERTS_IS_RUNQ_EMPTY_FLGS(flags); } +#ifdef ERTS_SMP static ERTS_INLINE int runq_got_work_to_execute(ErtsRunQueue *rq) { return runq_got_work_to_execute_flags(ERTS_RUNQ_FLGS_GET_NOB(rq)); } +#endif #undef RUNQ_READ_RQ #undef RUNQ_SET_RQ -- cgit v1.2.3 From 1623d7a85908221a118356bec64693901405659d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 20 Jan 2017 16:28:42 +0100 Subject: erts: Fix thread suspend in crashdump * move signal handler setup --- erts/emulator/beam/break.c | 2 ++ erts/emulator/beam/erl_init.c | 1 - erts/emulator/beam/sys.h | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index dfbe1ced47..61907cc7ff 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -717,6 +717,8 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) * We have to be very very careful when doing this as the schedulers * could be anywhere. */ + sys_init_suspend_handler(); + for (i = 0; i < erts_no_schedulers; i++) { erts_tid_t tid = ERTS_SCHEDULER_IX(i)->tid; if (!erts_equal_tids(tid,erts_thr_self())) diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 2fd97208cc..070cc3f2d0 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -2220,7 +2220,6 @@ erl_start(int argc, char **argv) init_break_handler(); if (replace_intr) erts_replace_intr(); - sys_init_suspend_handler(); #endif boot_argc = argc - i; /* Number of arguments to init */ diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 7740dd4373..41a7ff4c16 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -859,9 +859,12 @@ int erts_sys_unsetenv(char *key); char *erts_read_env(char *key); void erts_free_read_env(void *value); +#if defined(ERTS_SMP) #if defined(ERTS_THR_HAVE_SIG_FUNCS) && !defined(ETHR_UNUSABLE_SIGUSRX) extern void sys_thr_resume(erts_tid_t tid); extern void sys_thr_suspend(erts_tid_t tid); +#define ERTS_SYS_SUSPEND_SIGNAL SIGUSR2 +#endif #endif /* utils.c */ -- cgit v1.2.3 From e27119948fc6ab28bea81019720bddaac5b655a7 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 23 Jan 2017 15:13:19 +0100 Subject: erts: Fix binary_to_term for compressed and zlib >= v1.2.9 Problem: z_stream was incorrectly copied with memcpy which just happened to work with zlib < v1.2.9. Solution: Avoid copying z_stream. --- erts/emulator/beam/external.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 656de7c49a..4491d48683 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1193,6 +1193,7 @@ typedef struct B2TContext_t { } u; } B2TContext; +static B2TContext* b2t_export_context(Process*, B2TContext* src); static uLongf binary2term_uncomp_size(byte* data, Sint size) { @@ -1225,7 +1226,7 @@ static uLongf binary2term_uncomp_size(byte* data, Sint size) static ERTS_INLINE int binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size, - B2TContext* ctx) + B2TContext** ctxp, Process* p) { byte *bytes = data; Sint size = data_size; @@ -1239,8 +1240,8 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size, size--; if (size < 5 || *bytes != COMPRESSED) { state->extp = bytes; - if (ctx) - ctx->state = B2TSizeInit; + if (ctxp) + (*ctxp)->state = B2TSizeInit; } else { uLongf dest_len = (Uint32) get_int32(bytes+1); @@ -1257,16 +1258,26 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size, return -1; } state->extp = erts_alloc(ERTS_ALC_T_EXT_TERM_DATA, dest_len); - ctx->reds -= dest_len; + if (ctxp) + (*ctxp)->reds -= dest_len; } state->exttmp = 1; - if (ctx) { + if (ctxp) { + /* + * Start decompression by exporting trap context + * so we don't have to deal with deep-copying z_stream. + */ + B2TContext* ctx = b2t_export_context(p, *ctxp); + ASSERT(state = &(*ctxp)->b2ts); + state = &ctx->b2ts; + if (erl_zlib_inflate_start(&ctx->u.uc.stream, bytes, size) != Z_OK) return -1; ctx->u.uc.dbytes = state->extp; ctx->u.uc.dleft = dest_len; ctx->state = B2TUncompressChunk; + *ctxp = ctx; } else { uLongf dlen = dest_len; @@ -1308,7 +1319,7 @@ erts_binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size { Sint res; - if (binary2term_prepare(state, data, data_size, NULL) < 0 || + if (binary2term_prepare(state, data, data_size, NULL, NULL) < 0 || (res=decoded_size(state->extp, state->extp + state->extsize, 0, NULL)) < 0) { if (state->exttmp) @@ -1435,7 +1446,7 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con if (ctx->aligned_alloc) { ctx->reds -= bin_size / 8; } - if (binary2term_prepare(&ctx->b2ts, bytes, bin_size, ctx) < 0) { + if (binary2term_prepare(&ctx->b2ts, bytes, bin_size, &ctx, p) < 0) { ctx->state = B2TBadArg; } break; -- cgit v1.2.3 From c5d9b970fb5b3a7143ec8bb2e8194514b5b04e25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Mon, 1 Aug 2016 14:35:04 +0200 Subject: erts: Remove broken hash from Erlang erlang:hash/2 has been deprecated for a while, time to remove it. --- erts/emulator/beam/bif.c | 19 --- erts/emulator/beam/bif.tab | 6 - erts/emulator/beam/erl_utils.h | 1 - erts/emulator/beam/utils.c | 264 +---------------------------------------- 4 files changed, 5 insertions(+), 285 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 65c370c55b..a3acf87000 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -4723,25 +4723,6 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) /**********************************************************************/ -BIF_RETTYPE hash_2(BIF_ALIST_2) -{ - Uint32 hash; - Sint range; - - if (is_not_small(BIF_ARG_2)) { - BIF_ERROR(BIF_P, BADARG); - } - if ((range = signed_val(BIF_ARG_2)) <= 0) { /* [1..MAX_SMALL] */ - BIF_ERROR(BIF_P, BADARG); - } -#if defined(ARCH_64) - if (range > ((1L << 27) - 1)) - BIF_ERROR(BIF_P, BADARG); -#endif - hash = make_broken_hash(BIF_ARG_1); - BIF_RET(make_small(1 + (hash % range))); /* [1..range] */ -} - BIF_RETTYPE phash_2(BIF_ALIST_2) { Uint32 hash; diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 47fdcfa7a4..476c4b9632 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -670,9 +670,3 @@ gcbif erlang:ceil/1 bif math:floor/1 bif math:ceil/1 bif math:fmod/2 - -# -# Obsolete -# - -bif erlang:hash/2 diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index 81800752f0..47289a0af1 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -117,7 +117,6 @@ int erts_fit_in_bits_int32(Sint32); int erts_fit_in_bits_uint(Uint); Sint erts_list_length(Eterm); int erts_is_builtin(Eterm, Eterm, int); -Uint32 make_broken_hash(Eterm); Uint32 block_hash(byte *, unsigned, Uint32); Uint32 make_hash2(Eterm); Uint32 make_hash(Eterm); diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 3fa48da1ec..74a0681dd5 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -700,12 +700,7 @@ erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length, /* make a hash index from an erlang term */ /* -** There are three hash functions. -** make_broken_hash: the one used for backward compatibility -** is called from the bif erlang:hash/2. Should never be used -** as it a) hashes only a part of binaries, b) hashes bignums really poorly, -** c) hashes bignums differently on different endian processors and d) hashes -** small integers with different weights on different bytes. +** There are two hash functions. ** ** make_hash: A hash function that will give the same values for the same ** terms regardless of the internal representation. Small integers are @@ -1045,6 +1040,10 @@ tail_recur: DESTROY_WSTACK(stack); return hash; +#undef MAKE_HASH_TUPLE_OP +#undef MAKE_HASH_TERM_ARRAY_OP +#undef MAKE_HASH_CDR_PRE_OP +#undef MAKE_HASH_CDR_POST_OP #undef UINT32_HASH_STEP #undef UINT32_HASH_RET } @@ -1954,259 +1953,6 @@ make_internal_hash(Eterm term) #undef HCONST #undef MIX - -Uint32 make_broken_hash(Eterm term) -{ - Uint32 hash = 0; - DECLARE_WSTACK(stack); - unsigned op; -tail_recur: - op = tag_val_def(term); - for (;;) { - switch (op) { - case NIL_DEF: - hash = hash*FUNNY_NUMBER3 + 1; - break; - case ATOM_DEF: - hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(term))->slot.bucket.hvalue); - break; - case SMALL_DEF: -#if defined(ARCH_64) - { - Sint y1 = signed_val(term); - Uint y2 = y1 < 0 ? -(Uint)y1 : y1; - Uint32 y3 = (Uint32) (y2 >> 32); - int arity = 1; - -#if defined(WORDS_BIGENDIAN) - if (!IS_SSMALL28(y1)) - { /* like a bignum */ - Uint32 y4 = (Uint32) y2; - hash = hash*FUNNY_NUMBER2 + ((y4 << 16) | (y4 >> 16)); - if (y3) { - hash = hash*FUNNY_NUMBER2 + ((y3 << 16) | (y3 >> 16)); - arity++; - } - hash = hash * (y1 < 0 ? FUNNY_NUMBER3 : FUNNY_NUMBER2) + arity; - } else { - hash = hash*FUNNY_NUMBER2 + (((Uint) y1) & 0xfffffff); - } -#else - if (!IS_SSMALL28(y1)) - { /* like a bignum */ - hash = hash*FUNNY_NUMBER2 + ((Uint32) y2); - if (y3) - { - hash = hash*FUNNY_NUMBER2 + y3; - arity++; - } - hash = hash * (y1 < 0 ? FUNNY_NUMBER3 : FUNNY_NUMBER2) + arity; - } else { - hash = hash*FUNNY_NUMBER2 + (((Uint) y1) & 0xfffffff); - } -#endif - } -#else - hash = hash*FUNNY_NUMBER2 + unsigned_val(term); -#endif - break; - - case BINARY_DEF: - { - size_t sz = binary_size(term); - size_t i = (sz < 15) ? sz : 15; - - hash = hash_binary_bytes(term, i, hash); - hash = hash*FUNNY_NUMBER4 + sz; - break; - } - - case EXPORT_DEF: - { - Export* ep = *((Export **) (export_val(term) + 1)); - - hash = hash * FUNNY_NUMBER11 + ep->info.mfa.arity; - hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(ep->info.mfa.module))->slot.bucket.hvalue); - hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(ep->info.mfa.function))->slot.bucket.hvalue); - break; - } - - case FUN_DEF: - { - ErlFunThing* funp = (ErlFunThing *) fun_val(term); - Uint num_free = funp->num_free; - - hash = hash * FUNNY_NUMBER10 + num_free; - hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(funp->fe->module))->slot.bucket.hvalue); - hash = hash*FUNNY_NUMBER2 + funp->fe->old_index; - hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq; - if (num_free > 0) { - if (num_free > 1) { - WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_TERM_ARRAY_OP); - } - term = funp->env[0]; - goto tail_recur; - } - break; - } - - case PID_DEF: - hash = hash*FUNNY_NUMBER5 + internal_pid_number(term); - break; - case EXTERNAL_PID_DEF: - hash = hash*FUNNY_NUMBER5 + external_pid_number(term); - break; - case PORT_DEF: - hash = hash*FUNNY_NUMBER9 + internal_port_number(term); - break; - case EXTERNAL_PORT_DEF: - hash = hash*FUNNY_NUMBER9 + external_port_number(term); - break; - case REF_DEF: - hash = hash*FUNNY_NUMBER9 + internal_ref_numbers(term)[0]; - break; - case EXTERNAL_REF_DEF: - hash = hash*FUNNY_NUMBER9 + external_ref_numbers(term)[0]; - break; - case FLOAT_DEF: - { - FloatDef ff; - GET_DOUBLE(term, ff); - if (ff.fd == 0.0f) { - /* ensure positive 0.0 */ - ff.fd = erts_get_positive_zero_float(); - } - hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]); - } - break; - case MAKE_HASH_CDR_PRE_OP: - term = (Eterm) WSTACK_POP(stack); - if (is_not_list(term)) { - WSTACK_PUSH(stack, (UWord) MAKE_HASH_CDR_POST_OP); - goto tail_recur; - } - /*fall through*/ - case LIST_DEF: - { - Eterm* list = list_val(term); - WSTACK_PUSH2(stack, (UWord) CDR(list), - (UWord) MAKE_HASH_CDR_PRE_OP); - term = CAR(list); - goto tail_recur; - } - - case MAKE_HASH_CDR_POST_OP: - hash *= FUNNY_NUMBER8; - break; - - case BIG_DEF: - { - Eterm* ptr = big_val(term); - int is_neg = BIG_SIGN(ptr); - Uint arity = BIG_ARITY(ptr); - Uint i = arity; - ptr++; -#if D_EXP == 16 - /* hash over 32 bit LE */ - - while(i--) { - hash = hash*FUNNY_NUMBER2 + *ptr++; - } -#elif D_EXP == 32 - -#if defined(WORDS_BIGENDIAN) - while(i--) { - Uint d = *ptr++; - hash = hash*FUNNY_NUMBER2 + ((d << 16) | (d >> 16)); - } -#else - while(i--) { - hash = hash*FUNNY_NUMBER2 + *ptr++; - } -#endif - -#elif D_EXP == 64 - { - Uint32 h = 0, l; -#if defined(WORDS_BIGENDIAN) - while(i--) { - Uint d = *ptr++; - l = d & 0xffffffff; - h = d >> 32; - hash = hash*FUNNY_NUMBER2 + ((l << 16) | (l >> 16)); - if (h || i) - hash = hash*FUNNY_NUMBER2 + ((h << 16) | (h >> 16)); - } -#else - while(i--) { - Uint d = *ptr++; - l = d & 0xffffffff; - h = d >> 32; - hash = hash*FUNNY_NUMBER2 + l; - if (h || i) - hash = hash*FUNNY_NUMBER2 + h; - } -#endif - /* adjust arity to match 32 bit mode */ - arity = (arity << 1) - (h == 0); - } - -#else -#error "unsupported D_EXP size" -#endif - hash = hash * (is_neg ? FUNNY_NUMBER3 : FUNNY_NUMBER2) + arity; - } - break; - - case MAP_DEF: - hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term); - break; - case TUPLE_DEF: - { - Eterm* ptr = tuple_val(term); - Uint arity = arityval(*ptr); - - WSTACK_PUSH3(stack, (UWord) arity, (UWord) (ptr+1), (UWord) arity); - op = MAKE_HASH_TUPLE_OP; - }/*fall through*/ - case MAKE_HASH_TUPLE_OP: - case MAKE_HASH_TERM_ARRAY_OP: - { - Uint i = (Uint) WSTACK_POP(stack); - Eterm* ptr = (Eterm*) WSTACK_POP(stack); - if (i != 0) { - term = *ptr; - WSTACK_PUSH3(stack, (UWord)(ptr+1), (UWord) i-1, (UWord) op); - goto tail_recur; - } - if (op == MAKE_HASH_TUPLE_OP) { - Uint32 arity = (UWord) WSTACK_POP(stack); - hash = hash*FUNNY_NUMBER9 + arity; - } - break; - } - - default: - erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_broken_hash\n"); - return 0; - } - if (WSTACK_ISEMPTY(stack)) break; - op = (Uint) WSTACK_POP(stack); - } - - DESTROY_WSTACK(stack); - return hash; - -#undef MAKE_HASH_TUPLE_OP -#undef MAKE_HASH_TERM_ARRAY_OP -#undef MAKE_HASH_CDR_PRE_OP -#undef MAKE_HASH_CDR_POST_OP -} - static Eterm do_allocate_logger_message(Eterm gleader, Eterm **hp, ErlOffHeap **ohp, ErlHeapFragment **bp, Process **p, Uint sz) -- cgit v1.2.3 From 26b59dfe67ef551cd94765557cdd8c79794bcc38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Tue, 31 May 2016 14:28:54 +0200 Subject: Add new AtU8 beam chunk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new chunk stores atoms encoded in UTF-8. beam_lib has also been modified to handle the new 'utf8_atoms' attribute while the 'atoms' attribute may be a missing chunk from now on. The binary_to_atom/2 BIF can now encode any utf8 binary with up to 255 characters. The list_to_atom/1 BIF can now accept codepoints higher than 255 with up to 255 characters (thanks to Björn Gustavsson). --- erts/emulator/beam/atom.c | 35 +++++++++++------ erts/emulator/beam/atom.h | 3 ++ erts/emulator/beam/beam_load.c | 66 +++++++++++++++++++++++--------- erts/emulator/beam/bif.c | 14 +++---- erts/emulator/beam/erl_unicode.c | 83 ++++++++++++++++------------------------ erts/emulator/beam/global.h | 1 + erts/emulator/beam/utils.c | 62 ++++++++++++++++++++++++++++++ 7 files changed, 178 insertions(+), 86 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index 2b5ad097a0..2055c29190 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -233,10 +233,10 @@ need_convertion: } /* - * erts_atom_put() may fail. If it fails THE_NON_VALUE is returned! + * erts_atom_put_index() may fail. Returns negative indexes for errors. */ -Eterm -erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc) +int +erts_atom_put_index(const byte *name, int len, ErtsAtomEncoding enc, int trunc) { byte utf8_copy[MAX_ATOM_SZ_FROM_LATIN1]; const byte *text = name; @@ -253,7 +253,7 @@ erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc) if (trunc) tlen = 0; else - return THE_NON_VALUE; + return ATOM_MAX_CHARS_ERROR; } switch (enc) { @@ -262,7 +262,7 @@ erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc) if (trunc) tlen = MAX_ATOM_CHARACTERS; else - return THE_NON_VALUE; + return ATOM_MAX_CHARS_ERROR; } #ifdef DEBUG for (aix = 0; aix < len; aix++) { @@ -276,7 +276,7 @@ erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc) if (trunc) tlen = MAX_ATOM_CHARACTERS; else - return THE_NON_VALUE; + return ATOM_MAX_CHARS_ERROR; } no_latin1_chars = tlen; latin1_to_utf8(utf8_copy, &text, &tlen); @@ -284,7 +284,7 @@ erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc) case ERTS_ATOM_ENC_UTF8: /* First sanity check; need to verify later */ if (tlen > MAX_ATOM_SZ_LIMIT && !trunc) - return THE_NON_VALUE; + return ATOM_MAX_CHARS_ERROR; break; } @@ -295,7 +295,7 @@ erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc) atom_read_unlock(); if (aix >= 0) { /* Already in table no need to verify it */ - return make_atom(aix); + return aix; } if (enc == ERTS_ATOM_ENC_UTF8) { @@ -314,13 +314,13 @@ erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc) case ERTS_UTF8_OK_MAX_CHARS: /* Truncated... */ if (!trunc) - return THE_NON_VALUE; + return ATOM_MAX_CHARS_ERROR; ASSERT(no_chars == MAX_ATOM_CHARACTERS); tlen = err_pos - text; break; default: /* Bad utf8... */ - return THE_NON_VALUE; + return ATOM_BAD_ENCODING_ERROR; } } @@ -333,7 +333,20 @@ erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc) atom_write_lock(); aix = index_put(&erts_atom_table, (void*) &a); atom_write_unlock(); - return make_atom(aix); + return aix; +} + +/* + * erts_atom_put() may fail. If it fails THE_NON_VALUE is returned! + */ +Eterm +erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc) +{ + int aix = erts_atom_put_index(name, len, enc, trunc); + if (aix >= 0) + return make_atom(aix); + else + return THE_NON_VALUE; } Eterm diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h index abd3b44993..be998a46bd 100644 --- a/erts/emulator/beam/atom.h +++ b/erts/emulator/beam/atom.h @@ -29,6 +29,8 @@ #define MAX_ATOM_SZ_LIMIT (4*MAX_ATOM_CHARACTERS) /* theoretical byte limit */ #define ATOM_LIMIT (1024*1024) #define MIN_ATOM_TABLE_SIZE 8192 +#define ATOM_BAD_ENCODING_ERROR -1 +#define ATOM_MAX_CHARS_ERROR -2 #ifndef ARCH_32 /* Internal atom cache needs MAX_ATOM_TABLE_SIZE to be less than an @@ -133,6 +135,7 @@ int atom_table_sz(void); /* table size in bytes, excluding stored objects */ Eterm am_atom_put(const char*, int); /* ONLY 7-bit ascii! */ Eterm erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc); +int erts_atom_put_index(const byte *name, int len, ErtsAtomEncoding enc, int trunc); void init_atom_table(void); void atom_info(fmtfn_t, void *); void dump_atoms(fmtfn_t, void *); diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 8f1faa6719..1899ffb079 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -157,13 +157,15 @@ typedef struct { #define STR_CHUNK 2 #define IMP_CHUNK 3 #define EXP_CHUNK 4 -#define NUM_MANDATORY 5 +#define MIN_MANDATORY 1 +#define MAX_MANDATORY 5 #define LAMBDA_CHUNK 5 #define LITERAL_CHUNK 6 #define ATTR_CHUNK 7 #define COMPILE_CHUNK 8 #define LINE_CHUNK 9 +#define UTF8_ATOM_CHUNK 10 #define NUM_CHUNK_TYPES (sizeof(chunk_types)/sizeof(chunk_types[0])) @@ -173,9 +175,13 @@ typedef struct { static Uint chunk_types[] = { /* - * Mandatory chunk types -- these MUST be present. + * Atom chunk types -- Atom or AtU8 MUST be present. */ MakeIffId('A', 't', 'o', 'm'), /* 0 */ + + /* + * Mandatory chunk types -- these MUST be present. + */ MakeIffId('C', 'o', 'd', 'e'), /* 1 */ MakeIffId('S', 't', 'r', 'T'), /* 2 */ MakeIffId('I', 'm', 'p', 'T'), /* 3 */ @@ -189,6 +195,7 @@ static Uint chunk_types[] = { MakeIffId('A', 't', 't', 'r'), /* 7 */ MakeIffId('C', 'I', 'n', 'f'), /* 8 */ MakeIffId('L', 'i', 'n', 'e'), /* 9 */ + MakeIffId('A', 't', 'U', '8'), /* 10 */ }; /* @@ -490,9 +497,9 @@ static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, #endif static int init_iff_file(LoaderState* stp, byte* code, Uint size); static int scan_iff_file(LoaderState* stp, Uint* chunk_types, - Uint num_types, Uint num_mandatory); + Uint num_types); static int verify_chunks(LoaderState* stp); -static int load_atom_table(LoaderState* stp); +static int load_atom_table(LoaderState* stp, ErtsAtomEncoding enc); static int load_import_table(LoaderState* stp); static int read_export_table(LoaderState* stp); static int is_bif(Eterm mod, Eterm func, unsigned arity); @@ -629,7 +636,7 @@ erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader, CHKALLOC(); CHKBLK(ERTS_ALC_T_CODE,stp->code); if (!init_iff_file(stp, code, unloaded_size) || - !scan_iff_file(stp, chunk_types, NUM_CHUNK_TYPES, NUM_MANDATORY) || + !scan_iff_file(stp, chunk_types, NUM_CHUNK_TYPES) || !verify_chunks(stp)) { goto load_error; } @@ -674,9 +681,16 @@ erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader, */ CHKBLK(ERTS_ALC_T_CODE,stp->code); - define_file(stp, "atom table", ATOM_CHUNK); - if (!load_atom_table(stp)) { - goto load_error; + if (stp->chunks[UTF8_ATOM_CHUNK].size > 0) { + define_file(stp, "utf8 atom table", UTF8_ATOM_CHUNK); + if (!load_atom_table(stp, ERTS_ATOM_ENC_UTF8)) { + goto load_error; + } + } else { + define_file(stp, "atom table", ATOM_CHUNK); + if (!load_atom_table(stp, ERTS_ATOM_ENC_LATIN1)) { + goto load_error; + } } /* @@ -1212,7 +1226,7 @@ init_iff_file(LoaderState* stp, byte* code, Uint size) * Scan the IFF file. The header should have been verified by init_iff_file(). */ static int -scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory) +scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types) { Uint count; Uint id; @@ -1291,7 +1305,16 @@ verify_chunks(LoaderState* stp) MD5_CTX context; MD5Init(&context); - for (i = 0; i < NUM_MANDATORY; i++) { + + if (stp->chunks[UTF8_ATOM_CHUNK].start != NULL) { + MD5Update(&context, stp->chunks[UTF8_ATOM_CHUNK].start, stp->chunks[UTF8_ATOM_CHUNK].size); + } else if (stp->chunks[ATOM_CHUNK].start != NULL) { + MD5Update(&context, stp->chunks[ATOM_CHUNK].start, stp->chunks[ATOM_CHUNK].size); + } else { + LoadError0(stp, "mandatory chunk of type 'Atom' or 'AtU8' not found\n"); + } + + for (i = MIN_MANDATORY; i < MAX_MANDATORY; i++) { if (stp->chunks[i].start != NULL) { MD5Update(&context, stp->chunks[i].start, stp->chunks[i].size); } else { @@ -1352,7 +1375,7 @@ verify_chunks(LoaderState* stp) } static int -load_atom_table(LoaderState* stp) +load_atom_table(LoaderState* stp, ErtsAtomEncoding enc) { unsigned int i; @@ -1371,7 +1394,7 @@ load_atom_table(LoaderState* stp) GetByte(stp, n); GetString(stp, atom, n); - stp->atom[i] = erts_atom_put(atom, n, ERTS_ATOM_ENC_LATIN1, 1); + stp->atom[i] = erts_atom_put(atom, n, enc, 1); } /* @@ -5937,7 +5960,7 @@ code_get_chunk_2(BIF_ALIST_2) goto error; } if (!init_iff_file(stp, start, binary_size(Bin)) || - !scan_iff_file(stp, &chunk, 1, 1) || + !scan_iff_file(stp, &chunk, 1) || stp->chunks[0].start == NULL) { res = am_undefined; goto done; @@ -5986,7 +6009,7 @@ code_module_md5_1(BIF_ALIST_1) } stp->module = THE_NON_VALUE; /* Suppress diagnostiscs */ if (!init_iff_file(stp, bytes, binary_size(Bin)) || - !scan_iff_file(stp, chunk_types, NUM_CHUNK_TYPES, NUM_MANDATORY) || + !scan_iff_file(stp, chunk_types, NUM_CHUNK_TYPES) || !verify_chunks(stp)) { res = am_undefined; goto done; @@ -6335,7 +6358,7 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) if (!init_iff_file(stp, bytes, size)) { goto error; } - if (!scan_iff_file(stp, chunk_types, NUM_CHUNK_TYPES, NUM_MANDATORY) || + if (!scan_iff_file(stp, chunk_types, NUM_CHUNK_TYPES) || !verify_chunks(stp)) { goto error; } @@ -6343,9 +6366,16 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) if (!read_code_header(stp)) { goto error; } - define_file(stp, "atom table", ATOM_CHUNK); - if (!load_atom_table(stp)) { - goto error; + if (stp->chunks[UTF8_ATOM_CHUNK].size > 0) { + define_file(stp, "utf8 atom table", UTF8_ATOM_CHUNK); + if (!load_atom_table(stp, ERTS_ATOM_ENC_UTF8)) { + goto error; + } + } else { + define_file(stp, "atom table", ATOM_CHUNK); + if (!load_atom_table(stp, ERTS_ATOM_ENC_LATIN1)) { + goto error; + } } define_file(stp, "export table", EXP_CHUNK); if (!stub_read_export_table(stp)) { diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index d886c2985e..95bf13c07c 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -3022,8 +3022,8 @@ BIF_RETTYPE atom_to_list_1(BIF_ALIST_1) BIF_RETTYPE list_to_atom_1(BIF_ALIST_1) { Eterm res; - char *buf = (char *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_CHARACTERS); - Sint i = intlist_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS); + byte *buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_SZ_LIMIT); + Sint i = erts_unicode_list_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS); if (i < 0) { erts_free(ERTS_ALC_T_TMP, (void *) buf); @@ -3033,7 +3033,7 @@ BIF_RETTYPE list_to_atom_1(BIF_ALIST_1) } BIF_ERROR(BIF_P, BADARG); } - res = erts_atom_put((byte *) buf, i, ERTS_ATOM_ENC_LATIN1, 1); + res = erts_atom_put(buf, i, ERTS_ATOM_ENC_UTF8, 1); ASSERT(is_atom(res)); erts_free(ERTS_ALC_T_TMP, (void *) buf); BIF_RET(res); @@ -3043,17 +3043,17 @@ BIF_RETTYPE list_to_atom_1(BIF_ALIST_1) BIF_RETTYPE list_to_existing_atom_1(BIF_ALIST_1) { - Sint i; - char *buf = (char *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_CHARACTERS); + byte *buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_SZ_LIMIT); + Sint i = erts_unicode_list_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS); - if ((i = intlist_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS)) < 0) { + if (i < 0) { error: erts_free(ERTS_ALC_T_TMP, (void *) buf); BIF_ERROR(BIF_P, BADARG); } else { Eterm a; - if (erts_atom_get(buf, i, &a, ERTS_ATOM_ENC_LATIN1)) { + if (erts_atom_get((char *) buf, i, &a, ERTS_ATOM_ENC_UTF8)) { erts_free(ERTS_ALC_T_TMP, (void *) buf); BIF_RET(a); } else { diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index bd5e1482fb..8919898181 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -1890,74 +1890,57 @@ binary_to_atom(Process* proc, Eterm bin, Eterm enc, int must_exist) byte* bytes; byte *temp_alloc = NULL; Uint bin_size; + Eterm a; if ((bytes = erts_get_aligned_binary_bytes(bin, &temp_alloc)) == 0) { BIF_ERROR(proc, BADARG); } bin_size = binary_size(bin); + if (enc == am_latin1) { - Eterm a; - if (bin_size > MAX_ATOM_CHARACTERS) { - system_limit: - erts_free_aligned_binary_bytes(temp_alloc); - BIF_ERROR(proc, SYSTEM_LIMIT); - } if (!must_exist) { - a = erts_atom_put((byte *) bytes, - bin_size, - ERTS_ATOM_ENC_LATIN1, - 0); - erts_free_aligned_binary_bytes(temp_alloc); - if (is_non_value(a)) - goto badarg; - BIF_RET(a); - } else if (erts_atom_get((char *)bytes, bin_size, &a, ERTS_ATOM_ENC_LATIN1)) { - erts_free_aligned_binary_bytes(temp_alloc); - BIF_RET(a); - } else { + int lix = erts_atom_put_index((byte *) bytes, + bin_size, + ERTS_ATOM_ENC_LATIN1, + 0); + if (lix == ATOM_BAD_ENCODING_ERROR) { + badarg: + erts_free_aligned_binary_bytes(temp_alloc); + BIF_ERROR(proc, BADARG); + } else if (lix == ATOM_MAX_CHARS_ERROR) { + system_limit: + erts_free_aligned_binary_bytes(temp_alloc); + BIF_ERROR(proc, SYSTEM_LIMIT); + } + + a = make_atom(lix); + } else if (!erts_atom_get((char *)bytes, bin_size, &a, ERTS_ATOM_ENC_LATIN1)) { goto badarg; } - } else if (enc == am_utf8 || enc == am_unicode) { - Eterm res; - Uint num_chars = 0; - const byte* p = bytes; - Uint left = bin_size; - while (left) { - if (++num_chars > MAX_ATOM_CHARACTERS) { + } else if (enc == am_utf8 || enc == am_unicode) { + if (!must_exist) { + int uix = erts_atom_put_index((byte *) bytes, + bin_size, + ERTS_ATOM_ENC_UTF8, + 0); + if (uix == ATOM_BAD_ENCODING_ERROR) { + goto badarg; + } else if (uix == ATOM_MAX_CHARS_ERROR) { goto system_limit; } - if ((p[0] & 0x80) == 0) { - ++p; - --left; - } - else if (left >= 2 - && (p[0] & 0xFE) == 0xC2 /* only allow latin1 subset */ - && (p[1] & 0xC0) == 0x80) { - p += 2; - left -= 2; - } - else goto badarg; - } - if (!must_exist) { - res = erts_atom_put((byte *) bytes, - bin_size, - ERTS_ATOM_ENC_UTF8, - 0); + a = make_atom(uix); } - else if (!erts_atom_get((char*)bytes, bin_size, &res, ERTS_ATOM_ENC_UTF8)) { + else if (!erts_atom_get((char*)bytes, bin_size, &a, ERTS_ATOM_ENC_UTF8)) { goto badarg; } - erts_free_aligned_binary_bytes(temp_alloc); - if (is_non_value(res)) - goto badarg; - BIF_RET(res); } else { - badarg: - erts_free_aligned_binary_bytes(temp_alloc); - BIF_ERROR(proc, BADARG); + goto badarg; } + + erts_free_aligned_binary_bytes(temp_alloc); + BIF_RET(a); } BIF_RETTYPE binary_to_atom_2(BIF_ALIST_2) diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 2b2f3c5cdc..9f2b43d216 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1373,6 +1373,7 @@ int erts_utf8_to_latin1(byte* dest, const byte* source, int slen); void bin_write(fmtfn_t, void*, byte*, size_t); Sint intlist_to_buf(Eterm, char*, Sint); /* most callers pass plain char*'s */ +Sint erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len); struct Sint_buf { #if defined(ARCH_64) diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index ec502d5a78..36b818505c 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -3923,6 +3923,68 @@ intlist_to_buf(Eterm list, char *buf, Sint len) return -2; /* not enough space */ } +/* Fill buf with the contents of the unicode list. + * Return the number of bytes in the buffer, + * or -1 for type error, + * or -2 for not enough buffer space (buffer contains truncated result). + */ +Sint +erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len) +{ + Eterm* listptr; + Sint sz = 0; + + if (is_nil(list)) { + return 0; + } + if (is_not_list(list)) { + return -1; + } + listptr = list_val(list); + + while (len-- > 0) { + Sint val; + + if (is_not_small(CAR(listptr))) { + return -1; + } + val = signed_val(CAR(listptr)); + if (0 <= val && val < 0x80) { + buf[sz] = val; + sz++; + } else if (val < 0x800) { + buf[sz+0] = 0xC0 | (val >> 6); + buf[sz+1] = 0x80 | (val & 0x3F); + sz += 2; + } else if (val < 0x10000UL) { + if (0xD800 <= val && val <= 0xDFFF) { + return -1; + } + buf[sz+0] = 0xE0 | (val >> 12); + buf[sz+1] = 0x80 | ((val >> 6) & 0x3F); + buf[sz+2] = 0x80 | (val & 0x3F); + sz += 3; + } else if (val < 0x110000) { + buf[sz+0] = 0xF0 | (val >> 18); + buf[sz+1] = 0x80 | ((val >> 12) & 0x3F); + buf[sz+2] = 0x80 | ((val >> 6) & 0x3F); + buf[sz+3] = 0x80 | (val & 0x3F); + sz += 4; + } else { + return -1; + } + list = CDR(listptr); + if (is_nil(list)) { + return sz; + } + if (is_not_list(list)) { + return -1; + } + listptr = list_val(list); + } + return -2; /* not enough space */ +} + /* ** Convert an integer to a byte list ** return pointer to converted stuff (need not to be at start of buf!) -- cgit v1.2.3 From 120f04387ade07ef5b8b6d20a04de7d21e0c40ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Thu, 5 Jan 2017 17:17:52 +0100 Subject: erts: Use generic signal handler --- erts/emulator/beam/atom.names | 14 +++++++++++++- erts/emulator/beam/bif.c | 14 +++++++++++++- erts/emulator/beam/bif.tab | 1 + erts/emulator/beam/global.h | 3 +++ 4 files changed, 30 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 668abfaaee..72a7b9e312 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -194,6 +194,7 @@ atom current_stacktrace atom data atom debug_flags atom decimals +atom default atom delay_trap atom dexit atom depth @@ -306,6 +307,7 @@ atom global atom Gt='>' atom grun atom group_leader +atom handle atom have_dt_utag atom heap_block_size atom heap_size @@ -566,7 +568,7 @@ atom running_procs atom runtime atom safe atom save_calls -atom scheduler +atom scheduler atom scheduler_id atom schedulers_online atom scheme @@ -594,6 +596,16 @@ atom shared atom sighup atom sigterm atom sigusr1 +atom sigusr2 +atom sigill +atom sigchld +atom sigabrt +atom sigalrm +atom sigstop +atom sigint +atom sigsegv +atom sigtstp +atom sigquit atom silent atom size atom sl_alloc diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index d886c2985e..452bfef71a 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -5273,7 +5273,7 @@ BIF_RETTYPE dt_restore_tag_1(BIF_ALIST_1) SEQ_TRACE_TOKEN(BIF_P) = am_have_dt_utag; } } -#else +#else if (BIF_ARG_1 != am_true) { BIF_ERROR(BIF_P,BADARG); } @@ -5281,4 +5281,16 @@ BIF_RETTYPE dt_restore_tag_1(BIF_ALIST_1) BIF_RET(am_true); } +BIF_RETTYPE erts_internal_set_signal_2(BIF_ALIST_2) { + if (is_atom(BIF_ARG_1) && ((BIF_ARG_2 == am_ignore) || + (BIF_ARG_2 == am_default) || + (BIF_ARG_2 == am_handle))) { + if (!erts_set_signal(BIF_ARG_1, BIF_ARG_2)) + goto error; + + BIF_RET(am_true); + } +error: + BIF_ERROR(BIF_P, BADARG); +} diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 32600f4338..318a4fd264 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -668,6 +668,7 @@ gcbif erlang:ceil/1 bif math:floor/1 bif math:ceil/1 bif math:fmod/2 +bif erts_internal:set_signal/2 # # Obsolete diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 2b2f3c5cdc..f0f959e97a 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1070,6 +1070,9 @@ void print_process_info(fmtfn_t, void *, Process*); void info(fmtfn_t, void *); void loaded(fmtfn_t, void *); +/* sighandler sys.c */ +int erts_set_signal(Eterm signal, Eterm type); + /* erl_arith.c */ double erts_get_positive_zero_float(void); -- cgit v1.2.3 From 314fddc2e2e4cd8587ce00dfce328245a315832f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 20 Jan 2017 16:28:42 +0100 Subject: erts: Fix thread suspend in crashdump * move signal handler setup --- erts/emulator/beam/break.c | 2 ++ erts/emulator/beam/erl_init.c | 1 - erts/emulator/beam/sys.h | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 6e1e94b95b..2fc61ab436 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -717,6 +717,8 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) * We have to be very very careful when doing this as the schedulers * could be anywhere. */ + sys_init_suspend_handler(); + for (i = 0; i < erts_no_schedulers; i++) { erts_tid_t tid = ERTS_SCHEDULER_IX(i)->tid; if (!erts_equal_tids(tid,erts_thr_self())) diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index c5904b375e..88bd002a8c 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -2220,7 +2220,6 @@ erl_start(int argc, char **argv) init_break_handler(); if (replace_intr) erts_replace_intr(); - sys_init_suspend_handler(); #endif boot_argc = argc - i; /* Number of arguments to init */ diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index ceea794cc4..a11fb16b39 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -851,9 +851,12 @@ int erts_sys_unsetenv(char *key); char *erts_read_env(char *key); void erts_free_read_env(void *value); +#if defined(ERTS_SMP) #if defined(ERTS_THR_HAVE_SIG_FUNCS) && !defined(ETHR_UNUSABLE_SIGUSRX) extern void sys_thr_resume(erts_tid_t tid); extern void sys_thr_suspend(erts_tid_t tid); +#define ERTS_SYS_SUSPEND_SIGNAL SIGUSR2 +#endif #endif /* utils.c */ -- cgit v1.2.3 From d4bd17da9759af54227891a90d9fb83d5bfe6d7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Tue, 24 Jan 2017 10:59:30 +0100 Subject: erts: Use os module instead of erts_internal for set_signal/2 * Add specs * Change return signature to 'ok' instead of 'true' --- erts/emulator/beam/bif.c | 14 -------------- erts/emulator/beam/bif.tab | 2 +- erts/emulator/beam/erl_bif_os.c | 14 ++++++++++++++ 3 files changed, 15 insertions(+), 15 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 452bfef71a..1048300cf7 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -5280,17 +5280,3 @@ BIF_RETTYPE dt_restore_tag_1(BIF_ALIST_1) #endif BIF_RET(am_true); } - -BIF_RETTYPE erts_internal_set_signal_2(BIF_ALIST_2) { - if (is_atom(BIF_ARG_1) && ((BIF_ARG_2 == am_ignore) || - (BIF_ARG_2 == am_default) || - (BIF_ARG_2 == am_handle))) { - if (!erts_set_signal(BIF_ARG_1, BIF_ARG_2)) - goto error; - - BIF_RET(am_true); - } - -error: - BIF_ERROR(BIF_P, BADARG); -} diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 318a4fd264..f90fae6041 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -668,7 +668,7 @@ gcbif erlang:ceil/1 bif math:floor/1 bif math:ceil/1 bif math:fmod/2 -bif erts_internal:set_signal/2 +bif os:set_signal/2 # # Obsolete diff --git a/erts/emulator/beam/erl_bif_os.c b/erts/emulator/beam/erl_bif_os.c index 46777d3aa5..edc3c82b23 100644 --- a/erts/emulator/beam/erl_bif_os.c +++ b/erts/emulator/beam/erl_bif_os.c @@ -203,3 +203,17 @@ BIF_RETTYPE os_unsetenv_1(BIF_ALIST_1) } BIF_RET(am_true); } + +BIF_RETTYPE os_set_signal_2(BIF_ALIST_2) { + if (is_atom(BIF_ARG_1) && ((BIF_ARG_2 == am_ignore) || + (BIF_ARG_2 == am_default) || + (BIF_ARG_2 == am_handle))) { + if (!erts_set_signal(BIF_ARG_1, BIF_ARG_2)) + goto error; + + BIF_RET(am_ok); + } + +error: + BIF_ERROR(BIF_P, BADARG); +} -- cgit v1.2.3 From 808b2f4d53e446aed07f85716c5c4b85abb3d18a Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 2 Feb 2017 16:08:38 +0100 Subject: erts: Add size of literals to module code size in crash dump and (l)oaded command in break menu. --- erts/emulator/beam/break.c | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 61907cc7ff..5d34c83c1a 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -379,6 +379,16 @@ info(fmtfn_t to, void *to_arg) } +static int code_size(struct erl_module_instance* modi) +{ + ErtsLiteralArea* lit = modi->code_hdr->literal_area; + int size = modi->code_length; + if (lit) { + size += (lit->end - lit->start) * sizeof(Eterm); + } + return size; +} + void loaded(fmtfn_t to, void *to_arg) { @@ -399,9 +409,9 @@ loaded(fmtfn_t to, void *to_arg) if ((modp = module_code(i, code_ix)) != NULL && ((modp->curr.code_length != 0) || (modp->old.code_length != 0))) { - cur += modp->curr.code_length; + cur += code_size(&modp->curr); if (modp->old.code_length != 0) { - old += modp->old.code_length; + old += code_size(&modp->old); } } } @@ -422,12 +432,12 @@ loaded(fmtfn_t to, void *to_arg) ((modp->curr.code_length != 0) || (modp->old.code_length != 0))) { erts_print(to, to_arg, "%T", make_atom(modp->module)); - cur += modp->curr.code_length; - erts_print(to, to_arg, " %d", modp->curr.code_length ); + cur += code_size(&modp->curr); + erts_print(to, to_arg, " %d", code_size(&modp->curr)); if (modp->old.code_length != 0) { erts_print(to, to_arg, " (%d old)", - modp->old.code_length ); - old += modp->old.code_length; + code_size(&modp->old)); + old += code_size(&modp->old); } erts_print(to, to_arg, "\n"); } @@ -442,7 +452,7 @@ loaded(fmtfn_t to, void *to_arg) erts_print(to, to_arg, "%T", make_atom(modp->module)); erts_print(to, to_arg, "\n"); erts_print(to, to_arg, "Current size: %d\n", - modp->curr.code_length); + code_size(&modp->curr)); code = modp->curr.code_hdr; if (code != NULL && code->attr_ptr) { erts_print(to, to_arg, "Current attributes: "); @@ -456,7 +466,7 @@ loaded(fmtfn_t to, void *to_arg) } if (modp->old.code_length != 0) { - erts_print(to, to_arg, "Old size: %d\n", modp->old.code_length); + erts_print(to, to_arg, "Old size: %d\n", code_size(&modp->old)); code = modp->old.code_hdr; if (code->attr_ptr) { erts_print(to, to_arg, "Old attributes: "); -- cgit v1.2.3 From 9d927001ba16517042d3099612a2c9b130c6f62a Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 2 Feb 2017 19:21:36 +0100 Subject: Dirty schedulers should not touch scheduler data pointed to by process struct --- erts/emulator/beam/erl_process.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index b345c35a7e..cfa2b62061 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -9598,7 +9598,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) esdp->current_process = NULL; #ifdef ERTS_SMP - p->scheduler_data = NULL; + if (is_normal_sched) + p->scheduler_data = NULL; #endif erts_smp_proc_unlock(p, (ERTS_PROC_LOCK_MAIN @@ -10013,8 +10014,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) state = erts_smp_atomic32_read_nob(&p->state); - ASSERT(!p->scheduler_data); #ifndef ERTS_DIRTY_SCHEDULERS + ASSERT(!p->scheduler_data); p->scheduler_data = esdp; #else /* ERTS_DIRTY_SCHEDULERS */ if (is_normal_sched) { @@ -10025,6 +10026,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); goto sched_out_proc; } + ASSERT(!p->scheduler_data); p->scheduler_data = esdp; } else { -- cgit v1.2.3 From ea530357b27635278ae8a3260e735ea39df5c283 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 3 Feb 2017 12:05:49 +0100 Subject: Fix merge commit This fixes commit f0867aa2ccbbf5677e0577bba08f8b7bc53ec0ed --- erts/emulator/beam/erl_init.c | 1 - 1 file changed, 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 2fd97208cc..070cc3f2d0 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -2220,7 +2220,6 @@ erl_start(int argc, char **argv) init_break_handler(); if (replace_intr) erts_replace_intr(); - sys_init_suspend_handler(); #endif boot_argc = argc - i; /* Number of arguments to init */ -- cgit v1.2.3 From 7eff21e2b0b86f798cce29835cd85a414d3d1be5 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 2 Feb 2017 18:19:49 +0100 Subject: Use a hole-marker that cannot be mistaken for a valid term on the heap --- erts/emulator/beam/erl_init.c | 5 +++++ erts/emulator/beam/erl_vm.h | 17 +++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 2fd97208cc..6c744553c3 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -117,6 +117,11 @@ const int etp_big_endian = 1; const int etp_big_endian = 0; #endif const Eterm etp_the_non_value = THE_NON_VALUE; +#ifdef ERTS_HOLE_MARKER +const Eterm etp_hole_marker = ERTS_HOLE_MARKER; +#else +const Eterm etp_hole_marker = 0; +#endif /* * Note about VxWorks: All variables must be initialized by executable code, diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index f97716d030..899e2a275a 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -110,8 +110,21 @@ #define HeapWordsLeft(p) (HEAP_LIMIT(p) - HEAP_TOP(p)) #if defined(DEBUG) || defined(CHECK_FOR_HOLES) -# define ERTS_HOLE_MARKER (((0xdeadbeef << 24) << 8) | 0xdeadbeef) -#endif /* egil: 32-bit ? */ + +/* + * ERTS_HOLE_MARKER must *not* be mistaken for a valid term + * on the heap... + */ +# ifdef ARCH_64 +# define ERTS_HOLE_MARKER \ + make_catch(UWORD_CONSTANT(0xdeadbeaf00000000) >> _TAG_IMMED2_SIZE) +/* Will (at the time of writing) appear as 0xdeadbeaf0000001b */ +# else +# define ERTS_HOLE_MARKER \ + make_catch(UWORD_CONSTANT(0xdead0000) >> _TAG_IMMED2_SIZE) +/* Will (at the time of writing) appear as 0xdead001b */ +# endif +#endif /* * Allocate heap memory on the ordinary heap, NEVER in a heap -- cgit v1.2.3 From 3fe6f3c0caecec522c7e14353eda6bbb86c9e8d6 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 3 Feb 2017 15:22:48 +0100 Subject: erts: Add deallocation veto for magic destructors A magic destructor can return 0 and thereby take control and prolong the lifetime of a magic binary. --- erts/emulator/beam/beam_load.c | 5 +++-- erts/emulator/beam/binary.c | 6 ++++-- erts/emulator/beam/dist.c | 4 +++- erts/emulator/beam/dist.h | 2 +- erts/emulator/beam/erl_bif_binary.c | 15 ++++++++------- erts/emulator/beam/erl_bif_re.c | 3 ++- erts/emulator/beam/erl_binary.h | 16 ++++++++++------ erts/emulator/beam/erl_db_util.c | 5 +++-- erts/emulator/beam/erl_db_util.h | 2 +- erts/emulator/beam/erl_map.c | 3 ++- erts/emulator/beam/erl_nif.c | 3 ++- erts/emulator/beam/erl_ptab.c | 6 ++++-- erts/emulator/beam/erl_unicode.c | 3 ++- erts/emulator/beam/external.c | 6 ++++-- erts/emulator/beam/global.h | 2 +- 15 files changed, 50 insertions(+), 31 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 309465bcd3..119c004f1f 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -481,7 +481,7 @@ typedef struct LoaderState { static void free_loader_state(Binary* magic); static ErlHeapFragment* new_literal_fragment(Uint size); static void free_literal_fragment(ErlHeapFragment*); -static void loader_state_dtor(Binary* magic); +static int loader_state_dtor(Binary* magic); #ifdef HIPE static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, @@ -1012,7 +1012,7 @@ static void free_literal_fragment(ErlHeapFragment* bp) /* * This destructor function can safely be called multiple times. */ -static void +static int loader_state_dtor(Binary* magic) { LoaderState* stp = ERTS_MAGIC_BIN_DATA(magic); @@ -1097,6 +1097,7 @@ loader_state_dtor(Binary* magic) */ ASSERT(stp->genop_blocks == 0); + return 1; } #ifdef HIPE diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index 071a356260..9894992b70 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -355,9 +355,10 @@ typedef struct { Uint bitoffs; } ErtsB2LState; -static void b2l_state_destructor(Binary *mbp) +static int b2l_state_destructor(Binary *mbp) { ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == b2l_state_destructor); + return 1; } static BIF_RETTYPE @@ -729,12 +730,13 @@ list_to_binary_engine(ErtsL2BState *sp) } } -static void +static int l2b_state_destructor(Binary *mbp) { ErtsL2BState *sp = ERTS_MAGIC_BIN_DATA(mbp); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == l2b_state_destructor); DESTROY_SAVED_ESTACK(&sp->buf.iolist.estack); + return 1; } static ERTS_INLINE Eterm diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 990edb274e..9e8f853279 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -713,7 +713,7 @@ static void clear_dist_entry(DistEntry *dep) } } -void erts_dsend_context_dtor(Binary* ctx_bin) +int erts_dsend_context_dtor(Binary* ctx_bin) { ErtsSendContext* ctx = ERTS_MAGIC_BIN_DATA(ctx_bin); switch (ctx->dss.phase) { @@ -730,6 +730,8 @@ void erts_dsend_context_dtor(Binary* ctx_bin) } if (ctx->dep_to_deref) erts_deref_dist_entry(ctx->dep_to_deref); + + return 1; } Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx) diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index e82b416286..8799e54057 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -375,7 +375,7 @@ extern int erts_dsig_send_monitor(ErtsDSigData *, Eterm, Eterm, Eterm); extern int erts_dsig_send_m_exit(ErtsDSigData *, Eterm, Eterm, Eterm, Eterm); extern int erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx); -extern void erts_dsend_context_dtor(Binary*); +extern int erts_dsend_context_dtor(Binary*); extern Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx); extern int erts_dist_command(Port *prt, int reds); diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index 6e10980b6b..24debedce2 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -239,13 +239,13 @@ static void dump_ac_node(ACNode *node, int indent, int ch); /* * Callback for the magic binary */ -static void cleanup_my_data_ac(Binary *bp) +static int cleanup_my_data_ac(Binary *bp) { - return; + return 1; } -static void cleanup_my_data_bm(Binary *bp) +static int cleanup_my_data_bm(Binary *bp) { - return; + return 1; } /* @@ -2066,7 +2066,7 @@ static int do_search_backward(CommonData *cd, Uint *posp, Uint *redsp) } } -static void cleanup_common_data(Binary *bp) +static int cleanup_common_data(Binary *bp) { int i; CommonData *cd; @@ -2083,7 +2083,7 @@ static void cleanup_common_data(Binary *bp) break; } } - return; + return 1; } static BIF_RETTYPE do_longest_common(Process *p, Eterm list, int direction) @@ -2563,7 +2563,7 @@ typedef struct { #define BINARY_COPY_LOOP_FACTOR 100 -static void cleanup_copy_bin_state(Binary *bp) +static int cleanup_copy_bin_state(Binary *bp) { CopyBinState *cbs = (CopyBinState *) ERTS_MAGIC_BIN_DATA(bp); if (cbs->result != NULL) { @@ -2583,6 +2583,7 @@ static void cleanup_copy_bin_state(Binary *bp) break; } cbs->source_type = BC_TYPE_EMPTY; + return 1; } /* diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index ff7746ce1d..391fee5753 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -600,10 +600,11 @@ static void cleanup_restart_context(RestartContext *rc) } } -static void cleanup_restart_context_bin(Binary *bp) +static int cleanup_restart_context_bin(Binary *bp) { RestartContext *rc = ERTS_MAGIC_BIN_DATA(bp); cleanup_restart_context(rc); + return 1; } /* diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index fdc5efea98..52efa06816 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -189,10 +189,10 @@ ERTS_GLB_INLINE Binary *erts_bin_realloc_fnf(Binary *bp, Uint size); ERTS_GLB_INLINE Binary *erts_bin_realloc(Binary *bp, Uint size); ERTS_GLB_INLINE void erts_bin_free(Binary *bp); ERTS_GLB_INLINE Binary *erts_create_magic_binary_x(Uint size, - void (*destructor)(Binary *), + int (*destructor)(Binary *), int unaligned); ERTS_GLB_INLINE Binary *erts_create_magic_binary(Uint size, - void (*destructor)(Binary *)); + int (*destructor)(Binary *)); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -320,8 +320,12 @@ erts_bin_realloc(Binary *bp, Uint size) ERTS_GLB_INLINE void erts_bin_free(Binary *bp) { - if (bp->flags & BIN_FLAG_MAGIC) - ERTS_MAGIC_BIN_DESTRUCTOR(bp)(bp); + if (bp->flags & BIN_FLAG_MAGIC) { + if (!ERTS_MAGIC_BIN_DESTRUCTOR(bp)(bp)) { + /* Destructor took control of the deallocation */ + return; + } + } if (bp->flags & BIN_FLAG_DRV) erts_free(ERTS_ALC_T_DRV_BINARY, (void *) bp); else @@ -329,7 +333,7 @@ erts_bin_free(Binary *bp) } ERTS_GLB_INLINE Binary * -erts_create_magic_binary_x(Uint size, void (*destructor)(Binary *), +erts_create_magic_binary_x(Uint size, int (*destructor)(Binary *), int unaligned) { Uint bsize = unaligned ? ERTS_MAGIC_BIN_UNALIGNED_SIZE(size) @@ -348,7 +352,7 @@ erts_create_magic_binary_x(Uint size, void (*destructor)(Binary *), } ERTS_GLB_INLINE Binary * -erts_create_magic_binary(Uint size, void (*destructor)(Binary *)) +erts_create_magic_binary(Uint size, int (*destructor)(Binary *)) { return erts_create_magic_binary_x(size, destructor, 0); } diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 4e987d5bee..991d63a18b 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1702,17 +1702,18 @@ error: /* Here is were we land when compilation failed. */ /* ** Free a match program (in a binary) */ -void erts_db_match_prog_destructor(Binary *bprog) +int erts_db_match_prog_destructor(Binary *bprog) { MatchProg *prog; if (bprog == NULL) - return; + return 1; prog = Binary2MatchProg(bprog); if (prog->term_save != NULL) { free_message_buffer(prog->term_save); } if (prog->saved_program_buf != NULL) free_message_buffer(prog->saved_program_buf); + return 1; } void diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 49e5f6b4cf..3e29fdcbcf 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -355,7 +355,7 @@ Eterm db_add_counter(Eterm** hpp, Wterm counter, Eterm incr); Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags); Binary *db_match_set_compile(Process *p, Eterm matchexpr, Uint flags); -void erts_db_match_prog_destructor(Binary *); +int erts_db_match_prog_destructor(Binary *); typedef struct match_prog { ErlHeapFragment *term_save; /* Only if needed, a list of message diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index 979a0040b0..3015158b24 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -1188,12 +1188,13 @@ typedef struct HashmapMergeContext_ { #endif } HashmapMergeContext; -static void hashmap_merge_ctx_destructor(Binary* ctx_bin) +static int hashmap_merge_ctx_destructor(Binary* ctx_bin) { HashmapMergeContext* ctx = (HashmapMergeContext*) ERTS_MAGIC_BIN_DATA(ctx_bin); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(ctx_bin) == hashmap_merge_ctx_destructor); PSTACK_DESTROY_SAVED(&ctx->pstack); + return 1; } BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1) { diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 85fa53a886..fbe94708a5 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2109,7 +2109,7 @@ static void rollback_opened_resource_types(void) } -static void nif_resource_dtor(Binary* bin) +static int nif_resource_dtor(Binary* bin) { ErlNifResource* resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(bin); ErlNifResourceType* type = resource->type; @@ -2128,6 +2128,7 @@ static void nif_resource_dtor(Binary* bin) steal_resource_type(type); erts_free(ERTS_ALC_T_NIF, type); } + return 1; } void erts_resource_stop(ErlNifResource* resource, ErlNifEvent e, diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c index 22830a19c4..578ae14b9f 100644 --- a/erts/emulator/beam/erl_ptab.c +++ b/erts/emulator/beam/erl_ptab.c @@ -733,7 +733,7 @@ erts_ptab_delete_element(ErtsPTab *ptab, * erts_ptab_list() implements BIFs listing the content of the table, * e.g. erlang:processes/0. */ -static void cleanup_ptab_list_bif_data(Binary *bp); +static int cleanup_ptab_list_bif_data(Binary *bp); static int ptab_list_bif_engine(Process *c_p, Eterm *res_accp, Binary *mbp); @@ -787,7 +787,7 @@ erts_ptab_list(Process *c_p, ErtsPTab *ptab) return ret_val; } -static void +static int cleanup_ptab_list_bif_data(Binary *bp) { ErtsPTabListBifData *ptlbdp = ERTS_MAGIC_BIN_DATA(bp); @@ -875,6 +875,8 @@ cleanup_ptab_list_bif_data(Binary *bp) ERTS_PTAB_LIST_DBG_TRACE(ptlbdp->debug.caller, return); ERTS_PTAB_LIST_DBG_CLEANUP(ptlbdp); + + return 1; } static int diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index bd5e1482fb..3167ca5eef 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -123,10 +123,11 @@ static void cleanup_restart_context(RestartContext *rc) } } -static void cleanup_restart_context_bin(Binary *bp) +static int cleanup_restart_context_bin(Binary *bp) { RestartContext *rc = ERTS_MAGIC_BIN_DATA(bp); cleanup_restart_context(rc); + return 1; } static RestartContext *get_rc_from_bin(Eterm bin) diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 10e452fa25..cb3486fde2 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1387,12 +1387,13 @@ static void b2t_destroy_context(B2TContext* context) } } -static void b2t_context_destructor(Binary *context_bin) +static int b2t_context_destructor(Binary *context_bin) { B2TContext* ctx = (B2TContext*) ERTS_MAGIC_BIN_DATA(context_bin); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor); b2t_destroy_context(ctx); + return 1; } static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1) @@ -1797,7 +1798,7 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) { #endif #define TERM_TO_BINARY_MEMCPY_FACTOR 8 -static void ttb_context_destructor(Binary *context_bin) +static int ttb_context_destructor(Binary *context_bin) { TTBContext *context = ERTS_MAGIC_BIN_DATA(context_bin); if (context->alive) { @@ -1831,6 +1832,7 @@ static void ttb_context_destructor(Binary *context_bin) break; } } + return 1; } static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint flags, diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 2158f54a49..91e5bbbd98 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -274,7 +274,7 @@ typedef struct binary { typedef struct { ERTS_INTERNAL_BINARY_FIELDS SWord orig_size; - void (*destructor)(Binary *); + int (*destructor)(Binary *); union { struct { ERTS_BINARY_STRUCT_ALIGNMENT -- cgit v1.2.3 From 7111434c0eb1fdd6576a99ca94cdc2b20be9b9af Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 3 Feb 2017 17:40:45 +0100 Subject: erts: Rename ErlNifResource as ErtsResource as it's not part of the API --- erts/emulator/beam/erl_nif.c | 24 ++++++++++++------------ erts/emulator/beam/global.h | 8 ++++---- 2 files changed, 16 insertions(+), 16 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index fbe94708a5..b7425b9e45 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1913,7 +1913,7 @@ int enif_snprintf(char *buffer, size_t size, const char* format, ...) /* dummy node in circular list */ struct enif_resource_type_t resource_type_list; -#define SIZEOF_ErlNifResource(SIZE) (offsetof(ErlNifResource,data) + (SIZE)) +#define SIZEOF_ErlNifResource(SIZE) (offsetof(ErtsResource,data) + (SIZE)) static ErlNifResourceType* find_resource_type(Eterm module, Eterm name) { @@ -2111,7 +2111,7 @@ static void rollback_opened_resource_types(void) static int nif_resource_dtor(Binary* bin) { - ErlNifResource* resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(bin); + ErtsResource* resource = (ErtsResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(bin); ErlNifResourceType* type = resource->type; ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); @@ -2131,7 +2131,7 @@ static int nif_resource_dtor(Binary* bin) return 1; } -void erts_resource_stop(ErlNifResource* resource, ErlNifEvent e, +void erts_resource_stop(ErtsResource* resource, ErlNifEvent e, int is_direct_call) { struct enif_msg_environment_t msg_env; @@ -2146,7 +2146,7 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t size) Binary* bin = erts_create_magic_binary_x(SIZEOF_ErlNifResource(size), &nif_resource_dtor, 1); /* unaligned */ - ErlNifResource* resource = ERTS_MAGIC_BIN_UNALIGNED_DATA(bin); + ErtsResource* resource = ERTS_MAGIC_BIN_UNALIGNED_DATA(bin); ASSERT(type->owner && type->next && type->prev); /* not allowed in load/upgrade */ resource->type = type; @@ -2160,7 +2160,7 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t size) void enif_release_resource(void* obj) { - ErlNifResource* resource = DATA_TO_RESOURCE(obj); + ErtsResource* resource = DATA_TO_RESOURCE(obj); ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); @@ -2174,7 +2174,7 @@ void enif_release_resource(void* obj) void enif_keep_resource(void* obj) { - ErlNifResource* resource = DATA_TO_RESOURCE(obj); + ErtsResource* resource = DATA_TO_RESOURCE(obj); ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); @@ -2186,7 +2186,7 @@ void enif_keep_resource(void* obj) ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj) { - ErlNifResource* resource = DATA_TO_RESOURCE(obj); + ErtsResource* resource = DATA_TO_RESOURCE(obj); ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); Eterm* hp = alloc_heap(env,PROC_BIN_SIZE); return erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary); @@ -2207,7 +2207,7 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ { ProcBin* pb; Binary* mbin; - ErlNifResource* resource; + ErtsResource* resource; if (!ERTS_TERM_IS_MAGIC_BINARY(term)) { return 0; } @@ -2216,7 +2216,7 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ return 0; / * Or should we allow "resource binaries" as handles? * / }*/ mbin = pb->val; - resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(mbin); + resource = (ErtsResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(mbin); if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor || resource->type != type) { return 0; @@ -2227,9 +2227,9 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ size_t enif_sizeof_resource(void* obj) { - ErlNifResource* resource = DATA_TO_RESOURCE(obj); + ErtsResource* resource = DATA_TO_RESOURCE(obj); Binary* bin = &ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource)->binary; - return ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(bin) - offsetof(ErlNifResource,data); + return ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(bin) - offsetof(ErtsResource,data); } @@ -3303,7 +3303,7 @@ erts_unload_nif(struct erl_module_nif* lib) void erl_nif_init() { - ERTS_CT_ASSERT((offsetof(ErlNifResource,data) % 8) == ERTS_MAGIC_BIN_BYTES_TO_ALIGN); + ERTS_CT_ASSERT((offsetof(ErtsResource,data) % 8) == ERTS_MAGIC_BIN_BYTES_TO_ALIGN); resource_type_list.next = &resource_type_list; resource_type_list.prev = &resource_type_list; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 91e5bbbd98..86c38f8e8c 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -71,7 +71,7 @@ struct enif_resource_type_t Eterm module; Eterm name; }; -typedef struct enif_resource_t +typedef struct { struct enif_resource_type_t* type; #ifdef DEBUG @@ -82,14 +82,14 @@ typedef struct enif_resource_t #endif char data[1]; -}ErlNifResource; +}ErtsResource; -#define DATA_TO_RESOURCE(PTR) ((ErlNifResource*)((char*)(PTR) - offsetof(ErlNifResource,data))) +#define DATA_TO_RESOURCE(PTR) ((ErtsResource*)((char*)(PTR) - offsetof(ErtsResource,data))) extern void erts_pre_nif(struct enif_environment_t*, Process*, struct erl_module_nif*, Process* tracee); extern void erts_post_nif(struct enif_environment_t* env); -extern void erts_resource_stop(ErlNifResource*, ErlNifEvent, int is_direct_call); +extern void erts_resource_stop(ErtsResource*, ErlNifEvent, int is_direct_call); extern Eterm erts_nif_taints(Process* p); extern void erts_print_nif_taints(fmtfn_t to, void* to_arg); void erts_unload_nif(struct erl_module_nif* nif); -- cgit v1.2.3 From fc0477a67641b9ba344de595b7fec2431208f8e6 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 24 Jan 2017 19:50:25 +0100 Subject: Atomic reference count of binaries also in non-SMP NIF resources was not handled in a thread-safe manner in the runtime system without SMP support. As a consequence of this fix, the following driver functions are now thread-safe also in the runtime system without SMP support: - driver_free_binary() - driver_realloc_binary() - driver_binary_get_refc() - driver_binary_inc_refc() - driver_binary_dec_refc() --- erts/emulator/beam/beam_bp.c | 18 ++--- erts/emulator/beam/beam_bp.h | 6 +- erts/emulator/beam/beam_emu.c | 2 +- erts/emulator/beam/beam_load.c | 4 +- erts/emulator/beam/break.c | 2 +- erts/emulator/beam/copy.c | 12 ++-- erts/emulator/beam/dist.c | 4 +- erts/emulator/beam/erl_bif_ddll.c | 22 +++--- erts/emulator/beam/erl_bif_info.c | 2 +- erts/emulator/beam/erl_db.c | 20 +++--- erts/emulator/beam/erl_db_util.c | 2 +- erts/emulator/beam/erl_db_util.h | 4 +- erts/emulator/beam/erl_fun.c | 20 +++--- erts/emulator/beam/erl_fun.h | 2 +- erts/emulator/beam/erl_gc.c | 6 +- erts/emulator/beam/erl_message.c | 2 +- erts/emulator/beam/erl_monitors.c | 2 +- erts/emulator/beam/erl_node_tables.c | 50 ++++++------- erts/emulator/beam/erl_node_tables.h | 8 +-- erts/emulator/beam/erl_process_dump.c | 2 +- erts/emulator/beam/external.c | 2 +- erts/emulator/beam/global.h | 2 +- erts/emulator/beam/io.c | 31 ++------ erts/emulator/beam/sys.h | 128 +++++++++++++++++++++++++++++++--- erts/emulator/beam/utils.c | 2 +- 25 files changed, 219 insertions(+), 136 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index bbb2e4f34f..4e4429d1dc 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -347,17 +347,17 @@ consolidate_bp_data(Module* modp, BeamInstr* pc, int local) } if (flags & ERTS_BPF_META_TRACE) { dst->meta_tracer = src->meta_tracer; - erts_refc_inc(&dst->meta_tracer->refc, 1); + erts_smp_refc_inc(&dst->meta_tracer->refc, 1); dst->meta_ms = src->meta_ms; MatchSetRef(dst->meta_ms); } if (flags & ERTS_BPF_COUNT) { dst->count = src->count; - erts_refc_inc(&dst->count->refc, 1); + erts_smp_refc_inc(&dst->count->refc, 1); } if (flags & ERTS_BPF_TIME_TRACE) { dst->time = src->time; - erts_refc_inc(&dst->time->refc, 1); + erts_smp_refc_inc(&dst->time->refc, 1); ASSERT(dst->time->hash); } } @@ -1498,7 +1498,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, MatchSetRef(match_spec); bp->meta_ms = match_spec; bmt = Alloc(sizeof(BpMetaTracer)); - erts_refc_init(&bmt->refc, 1); + erts_smp_refc_init(&bmt->refc, 1); erts_tracer_update(&meta_tracer, tracer); /* copy tracer */ erts_smp_atomic_init_nob(&bmt->tracer, (erts_aint_t)meta_tracer); bp->meta_tracer = bmt; @@ -1507,7 +1507,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, ASSERT((bp->flags & ERTS_BPF_COUNT) == 0); bcp = Alloc(sizeof(BpCount)); - erts_refc_init(&bcp->refc, 1); + erts_smp_refc_init(&bcp->refc, 1); erts_smp_atomic_init_nob(&bcp->acount, 0); bp->count = bcp; } else if (break_flags & ERTS_BPF_TIME_TRACE) { @@ -1516,7 +1516,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, ASSERT((bp->flags & ERTS_BPF_TIME_TRACE) == 0); bdt = Alloc(sizeof(BpDataTime)); - erts_refc_init(&bdt->refc, 1); + erts_smp_refc_init(&bdt->refc, 1); bdt->n = erts_no_total_schedulers; bdt->hash = Alloc(sizeof(bp_time_hash_t)*(bdt->n)); for (i = 0; i < bdt->n; i++) { @@ -1583,7 +1583,7 @@ clear_function_break(BeamInstr *pc, Uint break_flags) static void bp_meta_unref(BpMetaTracer* bmt) { - if (erts_refc_dectest(&bmt->refc, 0) <= 0) { + if (erts_smp_refc_dectest(&bmt->refc, 0) <= 0) { ErtsTracer trc = erts_smp_atomic_read_nob(&bmt->tracer); ERTS_TRACER_CLEAR(&trc); Free(bmt); @@ -1593,7 +1593,7 @@ bp_meta_unref(BpMetaTracer* bmt) static void bp_count_unref(BpCount* bcp) { - if (erts_refc_dectest(&bcp->refc, 0) <= 0) { + if (erts_smp_refc_dectest(&bcp->refc, 0) <= 0) { Free(bcp); } } @@ -1601,7 +1601,7 @@ bp_count_unref(BpCount* bcp) static void bp_time_unref(BpDataTime* bdt) { - if (erts_refc_dectest(&bdt->refc, 0) <= 0) { + if (erts_smp_refc_dectest(&bdt->refc, 0) <= 0) { Uint i = 0; Uint j = 0; Process *h_p = NULL; diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 7206ef471a..27194e31be 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -41,7 +41,7 @@ typedef struct { typedef struct bp_data_time { /* Call time */ Uint n; bp_time_hash_t *hash; - erts_refc_t refc; + erts_smp_refc_t refc; } BpDataTime; typedef struct { @@ -51,12 +51,12 @@ typedef struct { typedef struct { erts_smp_atomic_t acount; - erts_refc_t refc; + erts_smp_refc_t refc; } BpCount; typedef struct { erts_smp_atomic_t tracer; - erts_refc_t refc; + erts_smp_refc_t refc; } BpMetaTracer; typedef struct generic_bp_data { diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 59a9ea1417..9413bc582d 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -6678,7 +6678,7 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) p->htop = hp + needed; funp = (ErlFunThing *) hp; hp = funp->env; - erts_refc_inc(&fe->refc, 2); + erts_smp_refc_inc(&fe->refc, 2); funp->thing_word = HEADER_FUN; funp->next = MSO(p).first; MSO(p).first = (struct erl_off_heap_header*) funp; diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 3f2bdf3f9d..88b6e89a02 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -4867,7 +4867,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) /* * We are hiding a pointer into older code. */ - erts_refc_dec(&fe->refc, 1); + erts_smp_refc_dec(&fe->refc, 1); } fe->address = code_ptr; #ifdef HIPE @@ -6287,7 +6287,7 @@ patch_funentries(Eterm Patchlist) * * Reproduced on a debug emulator with stdlib_test/qlc_SUITE:join_merge * - * erts_refc_dec(&fe->refc, 1); + * erts_smp_refc_dec(&fe->refc, 1); */ if (!patch(Addresses, (Uint) fe)) diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index dfbe1ced47..8020dd484c 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -647,7 +647,7 @@ bin_check(void) erts_printf("%p orig_size: %bpd, norefs = %bpd\n", bp->val, bp->val->orig_size, - erts_smp_atomic_read_nob(&bp->val->refc)); + erts_refc_read(&bp->val->refc, 1)); } } if (printed) { diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index ccc4cbad43..78db141cfc 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -826,7 +826,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint funp = (ErlFunThing *) tp; funp->next = off_heap->first; off_heap->first = (struct erl_off_heap_header*) funp; - erts_refc_inc(&funp->fe->refc, 2); + erts_smp_refc_inc(&funp->fe->refc, 2); *argp = make_fun(tp); } break; @@ -845,7 +845,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint etp->next = off_heap->first; off_heap->first = (struct erl_off_heap_header*)etp; - erts_refc_inc(&etp->node->refc, 2); + erts_smp_refc_inc(&etp->node->refc, 2); *argp = make_external(tp); } @@ -1491,7 +1491,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, } funp->next = off_heap->first; off_heap->first = (struct erl_off_heap_header*) funp; - erts_refc_inc(&funp->fe->refc, 2); + erts_smp_refc_inc(&funp->fe->refc, 2); goto cleanup_next; } case MAP_SUBTAG: @@ -1626,7 +1626,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, } etp->next = off_heap->first; off_heap->first = (struct erl_off_heap_header*) etp; - erts_refc_inc(&etp->node->refc, 2); + erts_smp_refc_inc(&etp->node->refc, 2); goto cleanup_next; } default: @@ -1802,7 +1802,7 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) case FUN_SUBTAG: { ErlFunThing* funp = (ErlFunThing *) (tp-1); - erts_refc_inc(&funp->fe->refc, 2); + erts_smp_refc_inc(&funp->fe->refc, 2); } goto off_heap_common; case EXTERNAL_PID_SUBTAG: @@ -1810,7 +1810,7 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) case EXTERNAL_REF_SUBTAG: { ExternalThing* etp = (ExternalThing *) (tp-1); - erts_refc_inc(&etp->node->refc, 2); + erts_smp_refc_inc(&etp->node->refc, 2); } off_heap_common: { diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index d79245e0e6..52b2174609 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -2081,7 +2081,7 @@ erts_dist_command(Port *prt, int reds_limit) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - erts_refc_inc(&dep->refc, 1); /* Otherwise dist_entry might be + erts_smp_refc_inc(&dep->refc, 1); /* Otherwise dist_entry might be removed if port command fails */ erts_smp_atomic_set_mb(&dep->dist_cmd_scheduled, 0); @@ -2514,7 +2514,7 @@ info_dist_entry(fmtfn_t to, void *arg, DistEntry *dep, int visible, int connecte erts_print(to, arg, "Name: %T", dep->sysname); #ifdef DEBUG - erts_print(to, arg, " (refc=%d)", erts_refc_read(&dep->refc, 0)); + erts_print(to, arg, " (refc=%d)", erts_smp_refc_read(&dep->refc, 0)); #endif erts_print(to, arg, "\n"); if (!connected && is_nil(dep->cid)) { diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index ef77201544..bace51bbbb 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1109,25 +1109,25 @@ void erts_ddll_decrement_port_count(DE_Handle *dh) static void first_ddll_reference(DE_Handle *dh) { assert_drv_list_rwlocked(); - erts_refc_init(&(dh->refc),1); + erts_smp_refc_init(&(dh->refc),1); } void erts_ddll_reference_driver(DE_Handle *dh) { assert_drv_list_locked(); - if (erts_refc_inctest(&(dh->refc),1) == 1) { - erts_refc_inc(&(dh->refc),2); /* add a reference for the scheduled operation */ + if (erts_smp_refc_inctest(&(dh->refc),1) == 1) { + erts_smp_refc_inc(&(dh->refc),2); /* add a reference for the scheduled operation */ } } void erts_ddll_reference_referenced_driver(DE_Handle *dh) { - erts_refc_inc(&(dh->refc),2); + erts_smp_refc_inc(&(dh->refc),2); } void erts_ddll_dereference_driver(DE_Handle *dh) { - if (erts_refc_dectest(&(dh->refc),0) == 0) { + if (erts_smp_refc_dectest(&(dh->refc),0) == 0) { /* No lock here, but if the driver is referenced again, the scheduled deletion is added as a reference too, see above */ erts_schedule_misc_op(ddll_no_more_references, (void *) dh); @@ -1150,11 +1150,11 @@ static void restore_process_references(DE_Handle *dh) { DE_ProcEntry *p; assert_drv_list_rwlocked(); - ASSERT(erts_refc_read(&(dh->refc),0) == 0); + ASSERT(erts_smp_refc_read(&(dh->refc),0) == 0); for(p = dh->procs;p != NULL; p = p->next) { if (p->awaiting_status == ERL_DE_PROC_LOADED) { ASSERT(p->flags & ERL_DE_FL_DEREFERENCED); - erts_refc_inc(&(dh->refc),1); + erts_smp_refc_inc(&(dh->refc),1); p->flags &= ~ERL_DE_FL_DEREFERENCED; } } @@ -1176,9 +1176,9 @@ static void ddll_no_more_references(void *vdh) lock_drv_list(); - x = erts_refc_read(&(dh->refc),0); + x = erts_smp_refc_read(&(dh->refc),0); if (x > 0) { - x = erts_refc_dectest(&(dh->refc),0); /* delete the reference added for me */ + x = erts_smp_refc_dectest(&(dh->refc),0); /* delete the reference added for me */ } @@ -1643,7 +1643,7 @@ static int load_driver_entry(DE_Handle **dhp, char *path, char *name) dh->handle = NULL; dh->procs = NULL; erts_smp_atomic32_init_nob(&dh->port_count, 0); - erts_refc_init(&(dh->refc), (erts_aint_t) 0); + erts_smp_refc_init(&(dh->refc), (erts_aint_t) 0); dh->status = -1; dh->reload_full_path = NULL; dh->reload_driver_name = NULL; @@ -1681,7 +1681,7 @@ static int reload_driver_entry(DE_Handle *dh) dh->reload_full_path = NULL; dh->reload_driver_name = NULL; - ASSERT(erts_refc_read(&(dh->refc),0) == 0); + ASSERT(erts_smp_refc_read(&(dh->refc),0) == 0); ASSERT(dh->full_path != NULL); erts_free(ERTS_ALC_T_DDLL_HANDLE, (void *) dh->full_path); dh->full_path = NULL; diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 735aabbee3..adf9306029 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -174,7 +174,7 @@ bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh) if (szp) *szp += 4+2; if (hpp) { - Uint refc = (Uint) erts_smp_atomic_read_nob(&pb->val->refc); + Uint refc = (Uint) erts_refc_read(&pb->val->refc, 1); tuple = TUPLE3(*hpp, val, orig_size, make_small(refc)); res = CONS(*hpp + 4, tuple, res); *hpp += 4+2; diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index dceadc46f4..3cc2b21a20 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -266,7 +266,7 @@ static void schedule_free_dbtable(DbTable* tb) * Caller is *not* allowed to access the specialized part * (hash or tree) of *tb after this function has returned. */ - ASSERT(erts_refc_read(&tb->common.ref, 0) == 0); + ASSERT(erts_smp_refc_read(&tb->common.ref, 0) == 0); erts_schedule_thr_prgr_later_cleanup_op(free_dbtable, (void *) tb, &tb->release.data, @@ -600,11 +600,11 @@ done: */ static ERTS_INLINE void local_fix_table(DbTable* tb) { - erts_refc_inc(&tb->common.ref, 1); + erts_smp_refc_inc(&tb->common.ref, 1); } static ERTS_INLINE void local_unfix_table(DbTable* tb) { - if (erts_refc_dectest(&tb->common.ref, 0) == 0) { + if (erts_smp_refc_dectest(&tb->common.ref, 0) == 0) { ASSERT(IS_HASH_TABLE(tb->common.status)); db_unfix_table_hash(&(tb->hash)); } @@ -1487,7 +1487,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) tb->common.type = status & ERTS_ETS_TABLE_TYPES; /* Note, 'type' is *read only* from now on... */ #endif - erts_refc_init(&tb->common.ref, 0); + erts_smp_refc_init(&tb->common.ref, 0); db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ), "db_tab", "db_tab_fix"); tb->common.keypos = keypos; @@ -2990,7 +2990,7 @@ void init_db(ErtsDbSpinCount db_spin_count) meta_pid_to_tab->common.meth = &db_hash; meta_pid_to_tab->common.compress = 0; - erts_refc_init(&meta_pid_to_tab->common.ref, 0); + erts_smp_refc_init(&meta_pid_to_tab->common.ref, 0); /* Neither rwlock or fixlock used db_init_lock(meta_pid_to_tab, "meta_pid_to_tab", "meta_pid_to_tab_FIX");*/ @@ -3021,7 +3021,7 @@ void init_db(ErtsDbSpinCount db_spin_count) meta_pid_to_fixed_tab->common.meth = &db_hash; meta_pid_to_fixed_tab->common.compress = 0; - erts_refc_init(&meta_pid_to_fixed_tab->common.ref, 0); + erts_smp_refc_init(&meta_pid_to_fixed_tab->common.ref, 0); /* Neither rwlock or fixlock used db_init_lock(meta_pid_to_fixed_tab, "meta_pid_to_fixed_tab", "meta_pid_to_fixed_tab_FIX");*/ @@ -3382,7 +3382,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) if ((*pp)->pid == pid) { DbFixation* fix = *pp; erts_aint_t diff = -((erts_aint_t) fix->counter); - erts_refc_add(&tb->common.ref,diff,0); + erts_smp_refc_add(&tb->common.ref,diff,0); *pp = fix->next; erts_db_free(ERTS_ALC_T_DB_FIXATION, tb, fix, sizeof(DbFixation)); @@ -3458,7 +3458,7 @@ static void fix_table_locked(Process* p, DbTable* tb) #ifdef ERTS_SMP erts_smp_mtx_lock(&tb->common.fixlock); #endif - erts_refc_inc(&tb->common.ref,1); + erts_smp_refc_inc(&tb->common.ref,1); fix = tb->common.fixations; if (fix == NULL) { tb->common.time.monotonic @@ -3514,7 +3514,7 @@ static void unfix_table_locked(Process* p, DbTable* tb, for (pp = &tb->common.fixations; *pp != NULL; pp = &(*pp)->next) { if ((*pp)->pid == p->common.id) { DbFixation* fix = *pp; - erts_refc_dec(&tb->common.ref,0); + erts_smp_refc_dec(&tb->common.ref,0); --(fix->counter); ASSERT(fix->counter >= 0); if (fix->counter > 0) { @@ -3563,7 +3563,7 @@ static void free_fixations_locked(DbTable *tb) fix = tb->common.fixations; while (fix != NULL) { erts_aint_t diff = -((erts_aint_t) fix->counter); - erts_refc_add(&tb->common.ref,diff,0); + erts_smp_refc_add(&tb->common.ref,diff,0); next_fix = fix->next; db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC); db_erase_bag_exact2(meta_pid_to_fixed_tab, diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 6732b708a8..f4db7ca3dd 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -3107,7 +3107,7 @@ void db_cleanup_offheap_comp(DbTerm* obj) break; case FUN_SUBTAG: ASSERT(u.pb != &tmp); - if (erts_refc_dectest(&u.fun->fe->refc, 0) == 0) { + if (erts_smp_refc_dectest(&u.fun->fe->refc, 0) == 0) { erts_erase_fun_entry(u.fun->fe); } break; diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 49e5f6b4cf..3fb063797e 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -211,7 +211,7 @@ typedef struct db_fixation { */ typedef struct db_table_common { - erts_refc_t ref; /* fixation counter */ + erts_smp_refc_t ref; /* fixation counter */ #ifdef ERTS_SMP erts_smp_rwmtx_t rwlock; /* rw lock on table */ erts_smp_mtx_t fixlock; /* Protects fixations,megasec,sec,microsec */ @@ -260,7 +260,7 @@ typedef struct db_table_common { (DB_BAG | DB_SET | DB_DUPLICATE_BAG))) #define IS_TREE_TABLE(Status) (!!((Status) & \ DB_ORDERED_SET)) -#define NFIXED(T) (erts_refc_read(&(T)->common.ref,0)) +#define NFIXED(T) (erts_smp_refc_read(&(T)->common.ref,0)) #define IS_FIXED(T) (NFIXED(T) != 0) /* diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index d0a57f0ad0..fd4d5b1c5c 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -110,9 +110,9 @@ erts_put_fun_entry(Eterm mod, int uniq, int index) fe = (ErlFunEntry *) hash_put(&erts_fun_table, (void*) &template); sys_memset(fe->uniq, 0, sizeof(fe->uniq)); fe->index = 0; - refc = erts_refc_inctest(&fe->refc, 0); + refc = erts_smp_refc_inctest(&fe->refc, 0); if (refc < 2) /* New or pending delete */ - erts_refc_inc(&fe->refc, 1); + erts_smp_refc_inc(&fe->refc, 1); erts_fun_write_unlock(); return fe; } @@ -134,9 +134,9 @@ erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index, sys_memcpy(fe->uniq, uniq, sizeof(fe->uniq)); fe->index = index; fe->arity = arity; - refc = erts_refc_inctest(&fe->refc, 0); + refc = erts_smp_refc_inctest(&fe->refc, 0); if (refc < 2) /* New or pending delete */ - erts_refc_inc(&fe->refc, 1); + erts_smp_refc_inc(&fe->refc, 1); erts_fun_write_unlock(); return fe; } @@ -161,9 +161,9 @@ erts_get_fun_entry(Eterm mod, int uniq, int index) erts_fun_read_lock(); ret = (ErlFunEntry *) hash_get(&erts_fun_table, (void*) &template); if (ret) { - erts_aint_t refc = erts_refc_inctest(&ret->refc, 1); + erts_aint_t refc = erts_smp_refc_inctest(&ret->refc, 1); if (refc < 2) /* Pending delete */ - erts_refc_inc(&ret->refc, 1); + erts_smp_refc_inc(&ret->refc, 1); } erts_fun_read_unlock(); return ret; @@ -184,7 +184,7 @@ erts_erase_fun_entry(ErlFunEntry* fe) * We have to check refc again since someone might have looked up * the fun entry and incremented refc after last check. */ - if (erts_refc_dectest(&fe->refc, -1) <= 0) + if (erts_smp_refc_dectest(&fe->refc, -1) <= 0) #endif { if (fe->address != unloaded_fun) @@ -256,7 +256,7 @@ erts_fun_purge_complete(ErlFunEntry **funs, Uint no) for (ix = 0; ix < no; ix++) { ErlFunEntry *fe = funs[ix]; fe->pend_purge_address = NULL; - if (erts_refc_dectest(&fe->refc, 0) == 0) + if (erts_smp_refc_dectest(&fe->refc, 0) == 0) erts_erase_fun_entry(fe); } ERTS_SMP_WRITE_MEMORY_BARRIER; @@ -288,7 +288,7 @@ erts_dump_fun_entries(fmtfn_t to, void *to_arg) #ifdef HIPE erts_print(to, to_arg, "Native_address: %p\n", fe->native_address); #endif - erts_print(to, to_arg, "Refc: %ld\n", erts_refc_read(&fe->refc, 1)); + erts_print(to, to_arg, "Refc: %ld\n", erts_smp_refc_read(&fe->refc, 1)); b = b->next; } } @@ -319,7 +319,7 @@ fun_alloc(ErlFunEntry* template) obj->old_uniq = template->old_uniq; obj->old_index = template->old_index; obj->module = template->module; - erts_refc_init(&obj->refc, -1); + erts_smp_refc_init(&obj->refc, -1); obj->address = unloaded_fun; obj->pend_purge_address = NULL; #ifdef HIPE diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index caa55c730c..0fd0d62343 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -42,7 +42,7 @@ typedef struct erl_fun_entry { Uint arity; /* The arity of the fun. */ Eterm module; /* Tagged atom for module. */ - erts_refc_t refc; /* Reference count: One for code + one for each + erts_smp_refc_t refc; /* Reference count: One for code + one for each fun object in each process. */ BeamInstr *pend_purge_address; /* address stored during a pending purge */ } ErlFunEntry; diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index af799d09da..704c282869 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -2718,7 +2718,7 @@ sweep_off_heap(Process *p, int fullsweep) case FUN_SUBTAG: { ErlFunEntry* fe = ((ErlFunThing*)ptr)->fe; - if (erts_refc_dectest(&fe->refc, 0) == 0) { + if (erts_smp_refc_dectest(&fe->refc, 0) == 0) { erts_erase_fun_entry(fe); } break; @@ -3374,12 +3374,12 @@ erts_check_off_heap2(Process *p, Eterm *htop) refc = erts_refc_read(&u.pb->val->refc, 1); break; case FUN_SUBTAG: - refc = erts_refc_read(&u.fun->fe->refc, 1); + refc = erts_smp_refc_read(&u.fun->fe->refc, 1); break; case EXTERNAL_PID_SUBTAG: case EXTERNAL_PORT_SUBTAG: case EXTERNAL_REF_SUBTAG: - refc = erts_refc_read(&u.ext->node->refc, 1); + refc = erts_smp_refc_read(&u.ext->node->refc, 1); break; default: ASSERT(!"erts_check_off_heap2: Invalid thing_word"); diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index e4c696ae3b..13c597adce 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -172,7 +172,7 @@ erts_cleanup_offheap(ErlOffHeap *offheap) } break; case FUN_SUBTAG: - if (erts_refc_dectest(&u.fun->fe->refc, 0) == 0) { + if (erts_smp_refc_dectest(&u.fun->fe->refc, 0) == 0) { erts_erase_fun_entry(u.fun->fe); } break; diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c index 910598690d..c207dea10f 100644 --- a/erts/emulator/beam/erl_monitors.c +++ b/erts/emulator/beam/erl_monitors.c @@ -104,7 +104,7 @@ do { \ (*((Hp)++)) = boxed_val((From))[i__]; \ if (is_external((To))) { \ external_thing_ptr((To))->next = NULL; \ - erts_refc_inc(&(external_thing_ptr((To))->node->refc), 2);\ + erts_smp_refc_inc(&(external_thing_ptr((To))->node->refc), 2);\ } \ } \ } while (0) diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 70500ed6e1..467a05f950 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -98,7 +98,7 @@ dist_table_alloc(void *dep_tmpl) dist_entries++; dep->prev = NULL; - erts_refc_init(&dep->refc, -1); + erts_smp_refc_init(&dep->refc, -1); erts_smp_rwmtx_init_opt_x(&dep->rwmtx, &rwmtx_opt, "dist_entry", chnl_nr); dep->sysname = sysname; dep->cid = NIL; @@ -208,7 +208,7 @@ erts_channel_no_to_dist_entry(Uint cno) * to the node name is used as channel no. */ if(cno == ERST_INTERNAL_CHANNEL_NO) { - erts_refc_inc(&erts_this_dist_entry->refc, 2); + erts_smp_refc_inc(&erts_this_dist_entry->refc, 2); return erts_this_dist_entry; } @@ -231,16 +231,16 @@ erts_sysname_to_connected_dist_entry(Eterm sysname) de.sysname = sysname; if(erts_this_dist_entry->sysname == sysname) { - erts_refc_inc(&erts_this_dist_entry->refc, 2); + erts_smp_refc_inc(&erts_this_dist_entry->refc, 2); return erts_this_dist_entry; } erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); res_dep = (DistEntry *) hash_get(&erts_dist_table, (void *) &de); if (res_dep) { - erts_aint_t refc = erts_refc_inctest(&res_dep->refc, 1); + erts_aint_t refc = erts_smp_refc_inctest(&res_dep->refc, 1); if (refc < 2) /* Pending delete */ - erts_refc_inc(&res_dep->refc, 1); + erts_smp_refc_inc(&res_dep->refc, 1); } erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); if (res_dep) { @@ -267,9 +267,9 @@ DistEntry *erts_find_or_insert_dist_entry(Eterm sysname) de.sysname = sysname; erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); res = hash_put(&erts_dist_table, (void *) &de); - refc = erts_refc_inctest(&res->refc, 0); + refc = erts_smp_refc_inctest(&res->refc, 0); if (refc < 2) /* New or pending delete */ - erts_refc_inc(&res->refc, 1); + erts_smp_refc_inc(&res->refc, 1); erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); return res; } @@ -282,9 +282,9 @@ DistEntry *erts_find_dist_entry(Eterm sysname) erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); res = hash_get(&erts_dist_table, (void *) &de); if (res) { - erts_aint_t refc = erts_refc_inctest(&res->refc, 1); + erts_aint_t refc = erts_smp_refc_inctest(&res->refc, 1); if (refc < 2) /* Pending delete */ - erts_refc_inc(&res->refc, 1); + erts_smp_refc_inc(&res->refc, 1); } erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); return res; @@ -311,7 +311,7 @@ static void try_delete_dist_entry(void *vdep) * * If refc > 0, the entry is in use. Keep the entry. */ - refc = erts_refc_dectest(&dep->refc, -1); + refc = erts_smp_refc_dectest(&dep->refc, -1); if (refc == -1) (void) hash_erase(&erts_dist_table, (void *) dep); erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); @@ -518,7 +518,7 @@ node_table_alloc(void *venp_tmpl) node_entries++; - erts_refc_init(&enp->refc, -1); + erts_smp_refc_init(&enp->refc, -1); enp->creation = ((ErlNode *) venp_tmpl)->creation; enp->sysname = ((ErlNode *) venp_tmpl)->sysname; enp->dist_entry = erts_find_or_insert_dist_entry(((ErlNode *) venp_tmpl)->sysname); @@ -585,9 +585,9 @@ ErlNode *erts_find_or_insert_node(Eterm sysname, Uint32 creation) erts_smp_rwmtx_rlock(&erts_node_table_rwmtx); res = hash_get(&erts_node_table, (void *) &ne); if (res && res != erts_this_node) { - erts_aint_t refc = erts_refc_inctest(&res->refc, 0); + erts_aint_t refc = erts_smp_refc_inctest(&res->refc, 0); if (refc < 2) /* New or pending delete */ - erts_refc_inc(&res->refc, 1); + erts_smp_refc_inc(&res->refc, 1); } erts_smp_rwmtx_runlock(&erts_node_table_rwmtx); if (res) @@ -597,9 +597,9 @@ ErlNode *erts_find_or_insert_node(Eterm sysname, Uint32 creation) res = hash_put(&erts_node_table, (void *) &ne); ASSERT(res); if (res != erts_this_node) { - erts_aint_t refc = erts_refc_inctest(&res->refc, 0); + erts_aint_t refc = erts_smp_refc_inctest(&res->refc, 0); if (refc < 2) /* New or pending delete */ - erts_refc_inc(&res->refc, 1); + erts_smp_refc_inc(&res->refc, 1); } erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); return res; @@ -626,7 +626,7 @@ static void try_delete_node(void *venp) * * If refc > 0, the entry is in use. Keep the entry. */ - refc = erts_refc_dectest(&enp->refc, -1); + refc = erts_smp_refc_dectest(&enp->refc, -1); if (refc == -1) (void) hash_erase(&erts_node_table, (void *) enp); erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); @@ -672,7 +672,7 @@ static void print_node(void *venp, void *vpndp) erts_print(pndp->to, pndp->to_arg, " %d", enp->creation); #ifdef DEBUG erts_print(pndp->to, pndp->to_arg, " (refc=%ld)", - erts_refc_read(&enp->refc, 0)); + erts_smp_refc_read(&enp->refc, 0)); #endif pndp->no_sysname++; } @@ -715,19 +715,19 @@ void erts_set_this_node(Eterm sysname, Uint creation) { ERTS_SMP_LC_ASSERT(erts_thr_progress_is_blocking()); - ASSERT(erts_refc_read(&erts_this_dist_entry->refc, 2)); + ASSERT(erts_smp_refc_read(&erts_this_dist_entry->refc, 2)); - if (erts_refc_dectest(&erts_this_node->refc, 0) == 0) + if (erts_smp_refc_dectest(&erts_this_node->refc, 0) == 0) try_delete_node(erts_this_node); - if (erts_refc_dectest(&erts_this_dist_entry->refc, 0) == 0) + if (erts_smp_refc_dectest(&erts_this_dist_entry->refc, 0) == 0) try_delete_dist_entry(erts_this_dist_entry); erts_this_node = NULL; /* to make sure refc is bumped for this node */ erts_this_node = erts_find_or_insert_node(sysname, creation); erts_this_dist_entry = erts_this_node->dist_entry; - erts_refc_inc(&erts_this_dist_entry->refc, 2); + erts_smp_refc_inc(&erts_this_dist_entry->refc, 2); erts_this_node_sysname = erts_this_node_sysname_BUFFER; erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER), @@ -789,13 +789,13 @@ void erts_init_node_tables(int dd_sec) node_tmpl.creation = 0; erts_this_node = hash_put(&erts_node_table, &node_tmpl); /* +1 for erts_this_node */ - erts_refc_init(&erts_this_node->refc, 1); + erts_smp_refc_init(&erts_this_node->refc, 1); ASSERT(erts_this_node->dist_entry != NULL); erts_this_dist_entry = erts_this_node->dist_entry; /* +1 for erts_this_dist_entry */ /* +1 for erts_this_node->dist_entry */ - erts_refc_init(&erts_this_dist_entry->refc, 2); + erts_smp_refc_init(&erts_this_dist_entry->refc, 2); erts_this_node_sysname = erts_this_node_sysname_BUFFER; @@ -1623,7 +1623,7 @@ reference_table_term(Uint **hpp, Uint *szp) tup = MK_2TUP(referred_nodes[i].node->sysname, MK_UINT(referred_nodes[i].node->creation)); - tup = MK_3TUP(tup, MK_UINT(erts_refc_read(&referred_nodes[i].node->refc, 0)), nril); + tup = MK_3TUP(tup, MK_UINT(erts_smp_refc_read(&referred_nodes[i].node->refc, 0)), nril); nl = MK_CONS(tup, nl); } @@ -1684,7 +1684,7 @@ reference_table_term(Uint **hpp, Uint *szp) /* DistList = [{Dist, Refc, ReferenceIdList}] */ tup = MK_3TUP(referred_dists[i].dist->sysname, - MK_UINT(erts_refc_read(&referred_dists[i].dist->refc, 0)), + MK_UINT(erts_smp_refc_read(&referred_dists[i].dist->refc, 0)), dril); dl = MK_CONS(tup, dl); } diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index 47a6724c21..35051173d0 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -107,7 +107,7 @@ typedef struct dist_entry_ { HashBucket hash_bucket; /* Hash bucket */ struct dist_entry_ *next; /* Next entry in dist_table (not sorted) */ struct dist_entry_ *prev; /* Previous entry in dist_table (not sorted) */ - erts_refc_t refc; /* Reference count */ + erts_smp_refc_t refc; /* Reference count */ erts_smp_rwmtx_t rwmtx; /* Protects all fields below until lck_mtx. */ Eterm sysname; /* name@host atom for efficiency */ @@ -149,7 +149,7 @@ typedef struct dist_entry_ { typedef struct erl_node_ { HashBucket hash_bucket; /* Hash bucket */ - erts_refc_t refc; /* Reference count */ + erts_smp_refc_t refc; /* Reference count */ Eterm sysname; /* name@host atom for efficiency */ Uint32 creation; /* Creation */ DistEntry *dist_entry; /* Corresponding dist entry */ @@ -210,7 +210,7 @@ ERTS_GLB_INLINE void erts_deref_dist_entry(DistEntry *dep) { ASSERT(dep); - if (erts_refc_dectest(&dep->refc, 0) == 0) + if (erts_smp_refc_dectest(&dep->refc, 0) == 0) erts_schedule_delete_dist_entry(dep); } @@ -218,7 +218,7 @@ ERTS_GLB_INLINE void erts_deref_node_entry(ErlNode *np) { ASSERT(np); - if (erts_refc_dectest(&np->refc, 0) == 0) + if (erts_smp_refc_dectest(&np->refc, 0) == 0) erts_schedule_delete_node(np); } diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index d8bb00e8c6..c5c62c5438 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -443,7 +443,7 @@ heap_dump(fmtfn_t to, void *to_arg, Eterm x) ProcBin* pb = (ProcBin *) binary_val(x); Binary* val = pb->val; - if (erts_smp_atomic_xchg_nob(&val->refc, 0) != 0) { + if (erts_atomic_xchg_nob(&val->refc, 0) != 0) { val->flags = (UWord) all_binaries; all_binaries = val; } diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index beed847578..7963e745c7 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -616,7 +616,7 @@ erts_make_dist_ext_copy(ErtsDistExternal *edep, Uint xsize) sys_memcpy((void *) ep, (void *) edep, dist_ext_sz); ep += dist_ext_sz; if (new_edep->dep) - erts_refc_inc(&new_edep->dep->refc, 1); + erts_smp_refc_inc(&new_edep->dep->refc, 1); new_edep->extp = ep; new_edep->ext_endp = ep + ext_sz; new_edep->heap_size = -1; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index d6df85034c..26aa39b65a 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -127,7 +127,7 @@ typedef struct { void *handle; /* Handle for DLL or SO (for dyn. drivers). */ DE_ProcEntry *procs; /* List of pids that have loaded this driver, or that wait for it to change state */ - erts_refc_t refc; /* Number of ports/processes having + erts_smp_refc_t refc; /* Number of ports/processes having references to the driver */ erts_smp_atomic32_t port_count; /* Number of ports using the driver */ Uint flags; /* ERL_DE_FL_KILL_PORTS */ diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 4f131c74de..31396d06c6 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -6890,12 +6890,6 @@ ErlDrvSizeT driver_vec_to_buf(ErlIOVec *vec, char *buf, ErlDrvSizeT len) return (orig_len - len); } - -/* - * - driver_alloc_binary() is thread safe (efile driver depend on it). - * - driver_realloc_binary(), and driver_free_binary() are *not* thread safe. - */ - /* * reference count on driver binaries... */ @@ -6938,26 +6932,15 @@ driver_alloc_binary(ErlDrvSizeT size) return Binary2ErlDrvBinary(bin); } -/* Reallocate space hold by binary */ +/* Reallocate space held by binary */ ErlDrvBinary* driver_realloc_binary(ErlDrvBinary* bin, ErlDrvSizeT size) { Binary* oldbin; Binary* newbin; - if (!bin) { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, - "Bad use of driver_realloc_binary(%p, %lu): " - "called with ", - bin, (unsigned long)size); - if (!bin) { - erts_dsprintf(dsbufp, "NULL pointer as first argument"); - } - erts_send_warning_to_logger_nogl(dsbufp); - if (!bin) - return driver_alloc_binary(size); - } + if (!bin) + return driver_alloc_binary(size); oldbin = ErlDrvBinary2Binary(bin); newbin = (Binary *) erts_bin_realloc_fnf(oldbin, size); @@ -6971,14 +6954,8 @@ ErlDrvBinary* driver_realloc_binary(ErlDrvBinary* bin, ErlDrvSizeT size) void driver_free_binary(ErlDrvBinary* dbin) { Binary *bin; - if (!dbin) { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, - "Bad use of driver_free_binary(%p): called with " - "NULL pointer as argument", dbin); - erts_send_warning_to_logger_nogl(dsbufp); + if (!dbin) return; - } bin = ErlDrvBinary2Binary(dbin); if (erts_refc_dectest(&bin->refc, 0) == 0) diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 7740dd4373..acffef0d2c 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -894,7 +894,7 @@ void sys_alloc_stat(SysAllocStat *); #define ERTS_REFC_DEBUG #endif -typedef erts_smp_atomic_t erts_refc_t; +typedef erts_atomic_t erts_refc_t; ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, erts_aint_t val); ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val); @@ -913,27 +913,27 @@ ERTS_GLB_INLINE erts_aint_t erts_refc_read(erts_refc_t *refcp, ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, erts_aint_t val) { - erts_smp_atomic_init_nob((erts_smp_atomic_t *) refcp, val); + erts_atomic_init_nob((erts_atomic_t *) refcp, val); } ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val) { #ifdef ERTS_REFC_DEBUG - erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_atomic_inc_read_nob((erts_atomic_t *) refcp); if (val < min_val) erts_exit(ERTS_ABORT_EXIT, "erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #else - erts_smp_atomic_inc_nob((erts_smp_atomic_t *) refcp); + erts_atomic_inc_nob((erts_atomic_t *) refcp); #endif } ERTS_GLB_INLINE erts_aint_t erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val) { - erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_atomic_inc_read_nob((erts_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) erts_exit(ERTS_ABORT_EXIT, @@ -947,20 +947,20 @@ ERTS_GLB_INLINE void erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val) { #ifdef ERTS_REFC_DEBUG - erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_atomic_dec_read_nob((erts_atomic_t *) refcp); if (val < min_val) erts_exit(ERTS_ABORT_EXIT, "erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #else - erts_smp_atomic_dec_nob((erts_smp_atomic_t *) refcp); + erts_atomic_dec_nob((erts_atomic_t *) refcp); #endif } ERTS_GLB_INLINE erts_aint_t erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val) { - erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_atomic_dec_read_nob((erts_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) erts_exit(ERTS_ABORT_EXIT, @@ -974,20 +974,20 @@ ERTS_GLB_INLINE void erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val) { #ifdef ERTS_REFC_DEBUG - erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff); + erts_aint_t val = erts_atomic_add_read_nob((erts_atomic_t *) refcp, diff); if (val < min_val) erts_exit(ERTS_ABORT_EXIT, "erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n", diff, val, min_val); #else - erts_smp_atomic_add_nob((erts_smp_atomic_t *) refcp, diff); + erts_atomic_add_nob((erts_atomic_t *) refcp, diff); #endif } ERTS_GLB_INLINE erts_aint_t erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val) { - erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_atomic_read_nob((erts_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) erts_exit(ERTS_ABORT_EXIT, @@ -999,6 +999,112 @@ erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val) #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ +typedef erts_smp_atomic_t erts_smp_refc_t; + +ERTS_GLB_INLINE void erts_smp_refc_init(erts_smp_refc_t *refcp, erts_aint_t val); +ERTS_GLB_INLINE void erts_smp_refc_inc(erts_smp_refc_t *refcp, erts_aint_t min_val); +ERTS_GLB_INLINE erts_aint_t erts_smp_refc_inctest(erts_smp_refc_t *refcp, + erts_aint_t min_val); +ERTS_GLB_INLINE void erts_smp_refc_dec(erts_smp_refc_t *refcp, erts_aint_t min_val); +ERTS_GLB_INLINE erts_aint_t erts_smp_refc_dectest(erts_smp_refc_t *refcp, + erts_aint_t min_val); +ERTS_GLB_INLINE void erts_smp_refc_add(erts_smp_refc_t *refcp, erts_aint_t diff, + erts_aint_t min_val); +ERTS_GLB_INLINE erts_aint_t erts_smp_refc_read(erts_smp_refc_t *refcp, + erts_aint_t min_val); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE void +erts_smp_refc_init(erts_smp_refc_t *refcp, erts_aint_t val) +{ + erts_smp_atomic_init_nob((erts_smp_atomic_t *) refcp, val); +} + +ERTS_GLB_INLINE void +erts_smp_refc_inc(erts_smp_refc_t *refcp, erts_aint_t min_val) +{ +#ifdef ERTS_REFC_DEBUG + erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp); + if (val < min_val) + erts_exit(ERTS_ABORT_EXIT, + "erts_smp_refc_inc(): Bad refc found (refc=%ld < %ld)!\n", + val, min_val); +#else + erts_smp_atomic_inc_nob((erts_smp_atomic_t *) refcp); +#endif +} + +ERTS_GLB_INLINE erts_aint_t +erts_smp_refc_inctest(erts_smp_refc_t *refcp, erts_aint_t min_val) +{ + erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp); +#ifdef ERTS_REFC_DEBUG + if (val < min_val) + erts_exit(ERTS_ABORT_EXIT, + "erts_smp_refc_inctest(): Bad refc found (refc=%ld < %ld)!\n", + val, min_val); +#endif + return val; +} + +ERTS_GLB_INLINE void +erts_smp_refc_dec(erts_smp_refc_t *refcp, erts_aint_t min_val) +{ +#ifdef ERTS_REFC_DEBUG + erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp); + if (val < min_val) + erts_exit(ERTS_ABORT_EXIT, + "erts_smp_refc_dec(): Bad refc found (refc=%ld < %ld)!\n", + val, min_val); +#else + erts_smp_atomic_dec_nob((erts_smp_atomic_t *) refcp); +#endif +} + +ERTS_GLB_INLINE erts_aint_t +erts_smp_refc_dectest(erts_smp_refc_t *refcp, erts_aint_t min_val) +{ + erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp); +#ifdef ERTS_REFC_DEBUG + if (val < min_val) + erts_exit(ERTS_ABORT_EXIT, + "erts_smp_refc_dectest(): Bad refc found (refc=%ld < %ld)!\n", + val, min_val); +#endif + return val; +} + +ERTS_GLB_INLINE void +erts_smp_refc_add(erts_smp_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val) +{ +#ifdef ERTS_REFC_DEBUG + erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff); + if (val < min_val) + erts_exit(ERTS_ABORT_EXIT, + "erts_smp_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n", + diff, val, min_val); +#else + erts_smp_atomic_add_nob((erts_smp_atomic_t *) refcp, diff); +#endif +} + +ERTS_GLB_INLINE erts_aint_t +erts_smp_refc_read(erts_smp_refc_t *refcp, erts_aint_t min_val) +{ + erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp); +#ifdef ERTS_REFC_DEBUG + if (val < min_val) + erts_exit(ERTS_ABORT_EXIT, + "erts_smp_refc_read(): Bad refc found (refc=%ld < %ld)!\n", + val, min_val); +#endif + return val; +} + +#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ + + #ifdef ERTS_ENABLE_KERNEL_POLL extern int erts_use_kernel_poll; #endif diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 87ea4f05a1..7f7e38c22a 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -3842,7 +3842,7 @@ store_external_or_ref_(Uint **hpp, ErlOffHeap* oh, Eterm ns) for(i = 0; i < size; i++) to_hp[i] = from_hp[i]; - erts_refc_inc(&((ExternalThing *) to_hp)->node->refc, 2); + erts_smp_refc_inc(&((ExternalThing *) to_hp)->node->refc, 2); ((struct erl_off_heap_header*) to_hp)->next = oh->first; oh->first = (struct erl_off_heap_header*) to_hp; -- cgit v1.2.3 From aefe39da715130f3d1df10084495d3b7ee48337e Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 24 Jan 2017 20:03:57 +0100 Subject: Implement erts_refc_inc_unless() --- erts/emulator/beam/sys.h | 55 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 16b38b8c6f..c6ea8049c3 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -893,6 +893,9 @@ typedef erts_atomic_t erts_refc_t; ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, erts_aint_t val); ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val); +ERTS_GLB_INLINE erts_aint_t erts_refc_inc_unless(erts_refc_t *refcp, + erts_aint_t unless_val, + erts_aint_t min_val); ERTS_GLB_INLINE erts_aint_t erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val); ERTS_GLB_INLINE void erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val); @@ -925,6 +928,30 @@ erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val) #endif } +ERTS_GLB_INLINE erts_aint_t +erts_refc_inc_unless(erts_refc_t *refcp, + erts_aint_t unless_val, + erts_aint_t min_val) +{ + erts_aint_t val = erts_atomic_read_nob((erts_atomic_t *) refcp); + while (1) { + erts_aint_t exp, new; +#ifdef ERTS_REFC_DEBUG + if (val < 0) + erts_exit(ERTS_ABORT_EXIT, + "erts_refc_inc_unless(): Bad refc found (refc=%ld < %ld)!\n", + val, min_val); +#endif + if (val == unless_val) + return val; + new = val + 1; + exp = val; + val = erts_atomic_cmpxchg_nob((erts_atomic_t *) refcp, new, exp); + if (val == exp) + return new; + } +} + ERTS_GLB_INLINE erts_aint_t erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val) { @@ -998,6 +1025,9 @@ typedef erts_smp_atomic_t erts_smp_refc_t; ERTS_GLB_INLINE void erts_smp_refc_init(erts_smp_refc_t *refcp, erts_aint_t val); ERTS_GLB_INLINE void erts_smp_refc_inc(erts_smp_refc_t *refcp, erts_aint_t min_val); +ERTS_GLB_INLINE erts_aint_t erts_smp_refc_inc_unless(erts_smp_refc_t *refcp, + erts_aint_t unless_val, + erts_aint_t min_val); ERTS_GLB_INLINE erts_aint_t erts_smp_refc_inctest(erts_smp_refc_t *refcp, erts_aint_t min_val); ERTS_GLB_INLINE void erts_smp_refc_dec(erts_smp_refc_t *refcp, erts_aint_t min_val); @@ -1030,6 +1060,31 @@ erts_smp_refc_inc(erts_smp_refc_t *refcp, erts_aint_t min_val) #endif } +ERTS_GLB_INLINE erts_aint_t +erts_smp_refc_inc_unless(erts_smp_refc_t *refcp, + erts_aint_t unless_val, + erts_aint_t min_val) +{ + erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp); + while (1) { + erts_aint_t exp, new; +#ifdef ERTS_REFC_DEBUG + if (val < 0) + erts_exit(ERTS_ABORT_EXIT, + "erts_smp_refc_inc_unless(): Bad refc found (refc=%ld < %ld)!\n", + val, min_val); +#endif + if (val == unless_val) + return val; + new = val + 1; + exp = val; + val = erts_smp_atomic_cmpxchg_nob((erts_smp_atomic_t *) refcp, new, exp); + if (val == exp) + return new; + } +} + + ERTS_GLB_INLINE erts_aint_t erts_smp_refc_inctest(erts_smp_refc_t *refcp, erts_aint_t min_val) { -- cgit v1.2.3 From b079018e38272604ffacfece9b97924a9e39df5c Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 23 Jan 2017 17:10:18 +0100 Subject: Implement magic references Magic references are *intentionally* indistinguishable from ordinary references for the Erlang software. Magic references do not change the language, and are intended as a pure runtime internal optimization. An ordinary reference is typically used as a key in some table. A magic reference has a direct pointer to a reference counted magic binary. This makes it possible to implement various things without having to do lookups in a table, but instead access the data directly. Besides very fast lookups this can also improve scalability by removing a potentially contended table. A couple of examples of planned future usage of magic references are ETS table identifiers, and BIF timer identifiers. Besides future optimizations using magic references it should also be possible to replace the exposed magic binary cludge with magic references. That is, magic binaries that are exposed as empty binaries to the Erlang software. --- erts/emulator/beam/atom.names | 1 + erts/emulator/beam/bif.c | 15 +- erts/emulator/beam/binary.c | 9 +- erts/emulator/beam/copy.c | 60 ++++- erts/emulator/beam/erl_alloc.c | 19 +- erts/emulator/beam/erl_alloc.types | 4 + erts/emulator/beam/erl_bif_ddll.c | 14 +- erts/emulator/beam/erl_bif_info.c | 87 ++++++- erts/emulator/beam/erl_bif_port.c | 14 +- erts/emulator/beam/erl_bif_trace.c | 2 +- erts/emulator/beam/erl_bif_unique.c | 310 ++++++++++++++++++++++- erts/emulator/beam/erl_bif_unique.h | 342 ++++++++++++++++++++++++-- erts/emulator/beam/erl_binary.h | 159 +++++++++++- erts/emulator/beam/erl_db_util.c | 25 +- erts/emulator/beam/erl_debug.c | 2 +- erts/emulator/beam/erl_gc.c | 27 +- erts/emulator/beam/erl_hl_timer.c | 12 +- erts/emulator/beam/erl_init.c | 12 +- erts/emulator/beam/erl_lock_check.c | 1 + erts/emulator/beam/erl_message.c | 5 + erts/emulator/beam/erl_monitors.c | 25 +- erts/emulator/beam/erl_monitors.h | 2 +- erts/emulator/beam/erl_msacc.c | 10 +- erts/emulator/beam/erl_nif.c | 4 +- erts/emulator/beam/erl_node_container_utils.h | 37 +-- erts/emulator/beam/erl_node_tables.c | 5 +- erts/emulator/beam/erl_printf_term.c | 5 +- erts/emulator/beam/erl_process.c | 11 +- erts/emulator/beam/erl_term.c | 40 ++- erts/emulator/beam/erl_term.h | 248 +++++++++++++++---- erts/emulator/beam/erl_time_sup.c | 15 +- erts/emulator/beam/external.c | 56 +++-- erts/emulator/beam/global.h | 133 +--------- erts/emulator/beam/io.c | 72 +++--- erts/emulator/beam/utils.c | 96 ++++---- 35 files changed, 1436 insertions(+), 443 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 1efe5fe300..df2866b40e 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -379,6 +379,7 @@ atom long_schedule atom low atom Lt='<' atom machine +atom magic_ref atom major atom match atom match_limit diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index a2d2433ed8..bda7b9224c 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -199,7 +199,7 @@ BIF_RETTYPE link_1(BIF_ALIST_1) goto res_no_proc; case ERTS_PORT_OP_SCHEDULED: if (refp) { - ASSERT(is_internal_ref(ref)); + ASSERT(is_internal_ordinary_ref(ref)); BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true); } default: @@ -782,7 +782,7 @@ local_name_monitor(Process *self, Eterm type, Eterm target_name) case ERTS_PORT_OP_DONE: return ret; case ERTS_PORT_OP_SCHEDULED: { /* Scheduled a signal */ - ASSERT(is_internal_ref(ret)); + ASSERT(is_internal_ordinary_ref(ret)); BIF_TRAP3(await_port_send_result_trap, self, ret, am_true, ret); /* bif_trap returns */ @@ -1180,7 +1180,7 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) res = erts_port_unlink(BIF_P, prt, BIF_P->common.id, refp); if (refp && res == ERTS_PORT_OP_SCHEDULED) { - ASSERT(is_internal_ref(ref)); + ASSERT(is_internal_ordinary_ref(ref)); BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true); } } @@ -1586,7 +1586,7 @@ BIF_RETTYPE exit_2(BIF_ALIST_2) ERTS_BIF_CHK_EXITED(BIF_P); if (refp && res == ERTS_PORT_OP_SCHEDULED) { - ASSERT(is_internal_ref(ref)); + ASSERT(is_internal_ordinary_ref(ref)); BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true); } @@ -2221,7 +2221,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx) /* Fall through */ case ERTS_PORT_OP_SCHEDULED: if (is_not_nil(*refp)) { - ASSERT(is_internal_ref(*refp)); + ASSERT(is_internal_ordinary_ref(*refp)); ret_val = SEND_AWAIT_RESULT; } break; @@ -2409,7 +2409,7 @@ BIF_RETTYPE send_3(BIF_ALIST_3) ERTS_BIF_PREP_YIELD_RETURN(retval, p, am_ok); break; case SEND_AWAIT_RESULT: - ASSERT(is_internal_ref(ref)); + ASSERT(is_internal_ordinary_ref(ref)); ERTS_BIF_PREP_TRAP3(retval, await_port_send_result_trap, p, ref, am_nosuspend, am_ok); break; case SEND_BADARG: @@ -2526,7 +2526,7 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg) ERTS_BIF_PREP_YIELD_RETURN(retval, p, msg); break; case SEND_AWAIT_RESULT: - ASSERT(is_internal_ref(ref)); + ASSERT(is_internal_ordinary_ref(ref)); ERTS_BIF_PREP_TRAP3(retval, await_port_send_result_trap, p, ref, msg, msg); break; @@ -4109,6 +4109,7 @@ BIF_RETTYPE ref_to_list_1(BIF_ALIST_1) { if (is_not_ref(BIF_ARG_1)) BIF_ERROR(BIF_P, BADARG); + erts_magic_ref_save_bin(BIF_ARG_1); BIF_RET(term2list_dsprintf(BIF_P, BIF_ARG_1)); } diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index 071a356260..a09950ec40 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -47,13 +47,8 @@ void erts_init_binary(void) { /* Verify Binary alignment... */ - if ((((UWord) &((Binary *) 0)->orig_bytes[0]) % ((UWord) 8)) != 0) { - /* I assume that any compiler should be able to optimize this - away. If not, this test is not very expensive... */ - erts_exit(ERTS_ABORT_EXIT, - "Internal error: Address of orig_bytes[0] of a Binary" - " is *not* 8-byte aligned\n"); - } + ERTS_CT_ASSERT((offsetof(Binary,orig_bytes) % 8) == 0); + ERTS_CT_ASSERT((offsetof(ErtsMagicBinary,u.aligned.data) % 8) == 0); erts_init_trap_export(&binary_to_list_continue_export, am_erts_internal, am_binary_to_list_continue, 1, diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index d20102cb2c..85db23393c 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -858,20 +858,24 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint case EXTERNAL_PORT_SUBTAG: case EXTERNAL_REF_SUBTAG: { - ExternalThing *etp = (ExternalThing *) htop; - + ExternalThing *etp = (ExternalThing *) objp; + erts_smp_refc_inc(&etp->node->refc, 2); + } + L_off_heap_node_container_common: + { + struct erl_off_heap_header *ohhp; + ohhp = (struct erl_off_heap_header *) htop; i = thing_arityval(hdr) + 1; + *argp = make_boxed(htop); tp = htop; while (i--) { *htop++ = *objp++; } - etp->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)etp; - erts_smp_refc_inc(&etp->node->refc, 2); + ohhp->next = off_heap->first; + off_heap->first = ohhp; - *argp = make_external(tp); } break; case MAP_SUBTAG: @@ -899,6 +903,13 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint case BIN_MATCHSTATE_SUBTAG: erts_exit(ERTS_ABORT_EXIT, "copy_struct: matchstate term not allowed"); + case REF_SUBTAG: + if (is_magic_ref_thing(objp)) { + ErtsMRefThing *mreft = (ErtsMRefThing *) objp; + erts_refc_inc(&mreft->mb->refc, 2); + goto L_off_heap_node_container_common; + } + /* Fall through... */ default: i = thing_arityval(hdr)+1; hbot -= i; @@ -1649,20 +1660,33 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, } case EXTERNAL_PID_SUBTAG: case EXTERNAL_PORT_SUBTAG: - case EXTERNAL_REF_SUBTAG: { - ExternalThing *etp = (ExternalThing *) hp; + case EXTERNAL_REF_SUBTAG: + { + ExternalThing *etp = (ExternalThing *) ptr; + erts_smp_refc_inc(&etp->node->refc, 2); + } + off_heap_node_container_common: + { + struct erl_off_heap_header *ohhp; + ohhp = (struct erl_off_heap_header *) hp; sz = thing_arityval(hdr); - *resp = make_external(hp); + *resp = make_boxed(hp); *hp++ = hdr; ptr++; while (sz-- > 0) { *hp++ = *ptr++; } - etp->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*) etp; - erts_smp_refc_inc(&etp->node->refc, 2); + ohhp->next = off_heap->first; + off_heap->first = ohhp; goto cleanup_next; } + case REF_SUBTAG: + if (is_magic_ref_thing(ptr)) { + ErtsMRefThing *mreft = (ErtsMRefThing *) ptr; + erts_refc_inc(&mreft->mb->refc, 2); + goto off_heap_node_container_common; + } + /* Fall through... */ default: sz = thing_arityval(hdr); *resp = make_boxed(hp); @@ -1859,6 +1883,15 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) off_heap->first = ohh; } break; + case REF_SUBTAG: { + ErtsRefThing *rtp = (ErtsRefThing *) (tp - 1); + if (is_magic_ref_thing(rtp)) { + ErtsMRefThing *mreft = (ErtsMRefThing *) rtp; + erts_refc_inc(&mreft->mb->refc, 2); + goto off_heap_common; + } + /* Fall through... */ + } default: { int tari = header_arity(val); @@ -1959,6 +1992,9 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite ASSERT(ptr + header_arity(val) < end); MOVE_BOXED(ptr, val, hp, &dummy_ref); switch (val & _HEADER_SUBTAG_MASK) { + case REF_SUBTAG: + if (is_ordinary_ref_thing(hdr)) + break; case REFC_BINARY_SUBTAG: case FUN_SUBTAG: case EXTERNAL_PID_SUBTAG: diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 4d990a9c56..e433de0f35 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -48,6 +48,7 @@ #if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE) #include "erl_check_io.h" #endif +#include "erl_bif_unique.h" #define GET_ERL_GF_ALLOC_IMPL #include "erl_goodfit_alloc.h" @@ -152,8 +153,7 @@ typedef struct { int internal; Uint req_sched; Process *proc; - Eterm ref; - Eterm ref_heap[REF_THING_SIZE]; + ErtsIRefStorage iref; int allocs[ERTS_ALC_INFO_A_END - ERTS_ALC_A_MIN + 1]; } ErtsAllocInfoReq; @@ -682,6 +682,8 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) #endif fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_EXP_TRACE)] = sizeof(NifExportTrace); + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MREF_NSCHED_ENT)] + = sizeof(ErtsNSchedMagicRefTableEntry); #ifdef HARD_DEBUG hdbg_init(); @@ -3139,9 +3141,10 @@ reply_alloc_info(void *vair) while (1) { if (hpp) - ref_copy = STORE_NC(hpp, ohp, air->ref); + ref_copy = erts_iref_storage_make_ref(&air->iref, + hpp, ohp, 0); else - *szp += REF_THING_SIZE; + *szp += erts_iref_storage_heap_size(&air->iref); ai_list = NIL; for (i = 0; air->allocs[i] != ERTS_ALC_A_INVALID; i++); @@ -3370,8 +3373,10 @@ reply_alloc_info(void *vair) erts_smp_proc_unlock(rp, rp_locks); erts_proc_dec_refc(rp); - if (erts_smp_atomic32_dec_read_nob(&air->refc) == 0) + if (erts_smp_atomic32_dec_read_nob(&air->refc) == 0) { + erts_iref_storage_clean(&air->iref); aireq_free(air); + } } int @@ -3384,7 +3389,6 @@ erts_request_alloc_info(struct process *c_p, ErtsAllocInfoReq *air = aireq_alloc(); Eterm req_ai[ERTS_ALC_INFO_A_END] = {0}; Eterm alist; - Eterm *hp; int airix = 0, ai; air->req_sched = erts_get_scheduler_id(); @@ -3398,8 +3402,7 @@ erts_request_alloc_info(struct process *c_p, if (is_not_internal_ref(ref)) return 0; - hp = &air->ref_heap[0]; - air->ref = STORE_NC(&hp, NULL, ref); + erts_iref_storage_save(&air->iref, ref); if (is_not_list(allocs)) return 0; diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 70eca5b49c..c253ea82c8 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -280,6 +280,10 @@ type TRACE_MSG_QUEUE SHORT_LIVED SYSTEM trace_message_queue type SCHED_ASYNC_JOB SHORT_LIVED SYSTEM async_calls type DIRTY_START STANDARD PROCESSES dirty_start type DIRTY_SL SHORT_LIVED SYSTEM dirty_short_lived +type MREF_NSCHED_ENT FIXED_SIZE SYSTEM nsched_magic_ref_entry +type MREF_ENT STANDARD SYSTEM magic_ref_entry +type MREF_TAB_BKTS STANDARD SYSTEM magic_ref_table_buckets +type MREF_TAB LONG_LIVED SYSTEM magic_ref_table +if threads_no_smp # Need thread safe allocs, but std_alloc and fix_alloc are not; diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index af90cfe40e..2ce872d2bc 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1476,8 +1476,10 @@ static void add_proc_loaded_deref(DE_Handle *dh, Process *proc) static Eterm copy_ref(Eterm ref, Eterm *hp) { - RefThing *ptr = ref_thing_ptr(ref); - memcpy(hp, ptr, sizeof(RefThing)); + ErtsORefThing *ptr; + ASSERT(is_internal_ordinary_ref(ref)); + ptr = ordinary_ref_thing_ptr(ref); + memcpy(hp, ptr, sizeof(ErtsORefThing)); return (make_internal_ref(hp)); } @@ -1720,10 +1722,10 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type, Eterm e; mp = erts_alloc_message_heap(proc, &rp_locks, (6 /* tuple */ + 3 /* Error tuple */ + - REF_THING_SIZE + need), + ERTS_REF_THING_SIZE + need), &hp, &ohp); r = copy_ref(ref,hp); - hp += REF_THING_SIZE; + hp += ERTS_REF_THING_SIZE; e = build_load_error_hp(hp, errcode); hp += need; mess = TUPLE2(hp,tag,e); @@ -1731,10 +1733,10 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type, mess = TUPLE5(hp,type,r,am_driver,driver_name,mess); } else { mp = erts_alloc_message_heap(proc, &rp_locks, - 6 /* tuple */ + REF_THING_SIZE, + 6 /* tuple */ + ERTS_REF_THING_SIZE, &hp, &ohp); r = copy_ref(ref,hp); - hp += REF_THING_SIZE; + hp += ERTS_REF_THING_SIZE; mess = TUPLE5(hp,type,r,am_driver,driver_name,tag); } erts_queue_message(proc, rp_locks, mp, mess, am_system); diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 37dcb5e7a6..be113b7c88 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -187,6 +187,32 @@ bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh) return res; } +static Eterm +bld_magic_ref_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh) +{ + struct erl_off_heap_header* ohh; + Eterm res = NIL; + Eterm tuple; + + for (ohh = oh->first; ohh; ohh = ohh->next) { + if (is_ref_thing_header((*((Eterm *) ohh)))) { + ErtsMRefThing *mrtp = (ErtsMRefThing *) ohh; + Eterm val = erts_bld_uword(hpp, szp, (UWord) mrtp->mb); + Eterm orig_size = erts_bld_uint(hpp, szp, mrtp->mb->orig_size); + + if (szp) + *szp += 4+2; + if (hpp) { + Uint refc = (Uint) erts_refc_read(&mrtp->mb->refc, 1); + tuple = TUPLE3(*hpp, val, orig_size, make_small(refc)); + res = CONS(*hpp + 4, tuple, res); + *hpp += 4+2; + } + } + } + return res; +} + /* make_monitor_list: @@ -613,7 +639,8 @@ static Eterm pi_args[] = { am_current_location, am_current_stacktrace, am_message_queue_data, - am_garbage_collection_info + am_garbage_collection_info, + am_magic_ref }; #define ERTS_PI_ARGS ((int) (sizeof(pi_args)/sizeof(Eterm))) @@ -664,6 +691,7 @@ pi_arg2ix(Eterm arg) case am_current_stacktrace: return 31; case am_message_queue_data: return 32; case am_garbage_collection_info: return 33; + case am_magic_ref: return 34; default: return -1; } } @@ -1599,6 +1627,14 @@ process_info_aux(Process *BIF_P, hp = HAlloc(BIF_P, 3); break; + case am_magic_ref: { + Uint sz = 3; + (void) bld_magic_ref_bin_list(NULL, &sz, &MSO(rp)); + hp = HAlloc(BIF_P, sz); + res = bld_magic_ref_bin_list(&hp, NULL, &MSO(rp)); + break; + } + default: return THE_NON_VALUE; /* will produce badarg */ @@ -3531,6 +3567,11 @@ BIF_RETTYPE error_logger_warning_map_0(BIF_ALIST_0) static erts_smp_atomic_t available_internal_state; +static void empty_magic_ref_destructor(Binary *bin) +{ + +} + BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) { /* @@ -3896,6 +3937,34 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) } return make_atom(ix); } + else if (ERTS_IS_ATOM_STR("magic_ref", tp[1])) { + Binary *bin; + UWord bin_addr, refc; + Eterm bin_addr_term, refc_term, test_type; + Uint sz; + Eterm *hp; + if (!is_internal_magic_ref(tp[2])) { + if (is_internal_ordinary_ref(tp[2])) { + ErtsORefThing *rtp; + rtp = (ErtsORefThing *) internal_ref_val(tp[2]); + if (erts_is_ref_numbers_magic(rtp->num)) + BIF_RET(am_true); + } + BIF_RET(am_false); + } + bin = erts_magic_ref2bin(tp[2]); + refc = erts_refc_read(&bin->refc, 1); + bin_addr = (UWord) bin; + sz = 4; + erts_bld_uword(NULL, &sz, bin_addr); + erts_bld_uword(NULL, &sz, refc); + hp = HAlloc(BIF_P, sz); + bin_addr_term = erts_bld_uword(&hp, NULL, bin_addr); + refc_term = erts_bld_uword(&hp, NULL, refc); + test_type = (ERTS_MAGIC_BIN_DESTRUCTOR(bin) == empty_magic_ref_destructor + ? am_true : am_false); + BIF_RET(TUPLE3(hp, bin_addr_term, refc_term, test_type)); + } break; } @@ -3984,7 +4053,6 @@ static void broken_halt_test(Eterm bif_arg_2) erts_exit(ERTS_DUMP_EXIT, "%T", bif_arg_2); } - BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) { /* @@ -4314,6 +4382,21 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) /* Used by ets_SUITE (stdlib) */ BIF_RET(erts_ets_restore_meta_state(BIF_P, BIF_ARG_2)); } + else if (ERTS_IS_ATOM_STR("make", BIF_ARG_1)) { + if (ERTS_IS_ATOM_STR("magic_ref", BIF_ARG_2)) { + Binary *bin = erts_create_magic_binary(0, empty_magic_ref_destructor); + UWord bin_addr = (UWord) bin; + Eterm bin_addr_term, magic_ref, res; + Eterm *hp; + Uint sz = ERTS_MAGIC_REF_THING_SIZE + 3; + erts_bld_uword(NULL, &sz, bin_addr); + hp = HAlloc(BIF_P, sz); + bin_addr_term = erts_bld_uword(&hp, NULL, bin_addr); + magic_ref = erts_mk_magic_ref(&hp, &BIF_P->off_heap, bin); + res = TUPLE2(hp, magic_ref, bin_addr_term); + BIF_RET(res); + } + } } BIF_ERROR(BIF_P, BADARG); diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 90e78a9b0b..09d2c5376b 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -214,7 +214,7 @@ BIF_RETTYPE erts_internal_port_command_3(BIF_ALIST_3) ASSERT(!(flags & ERTS_PORT_SIG_FLG_FORCE)); /* Fall through... */ case ERTS_PORT_OP_SCHEDULED: - ASSERT(is_internal_ref(ref)); + ASSERT(is_internal_ordinary_ref(ref)); ERTS_BIF_PREP_RET(res, ref); break; case ERTS_PORT_OP_DONE: @@ -260,7 +260,7 @@ BIF_RETTYPE erts_internal_port_call_3(BIF_ALIST_3) retval = am_badarg; break; case ERTS_PORT_OP_SCHEDULED: - ASSERT(is_internal_ref(retval)); + ASSERT(is_internal_ordinary_ref(retval)); break; case ERTS_PORT_OP_DONE: ASSERT(is_not_internal_ref(retval)); @@ -310,7 +310,7 @@ BIF_RETTYPE erts_internal_port_control_3(BIF_ALIST_3) retval = am_badarg; break; case ERTS_PORT_OP_SCHEDULED: - ASSERT(is_internal_ref(retval)); + ASSERT(is_internal_ordinary_ref(retval)); break; case ERTS_PORT_OP_DONE: ASSERT(is_not_internal_ref(retval)); @@ -356,7 +356,7 @@ BIF_RETTYPE erts_internal_port_close_1(BIF_ALIST_1) case ERTS_PORT_OP_DROPPED: BIF_RET(am_badarg); case ERTS_PORT_OP_SCHEDULED: - ASSERT(is_internal_ref(ref)); + ASSERT(is_internal_ordinary_ref(ref)); BIF_RET(ref); case ERTS_PORT_OP_DONE: BIF_RET(am_true); @@ -389,7 +389,7 @@ BIF_RETTYPE erts_internal_port_connect_2(BIF_ALIST_2) case ERTS_PORT_OP_DROPPED: BIF_RET(am_badarg); case ERTS_PORT_OP_SCHEDULED: - ASSERT(is_internal_ref(ref)); + ASSERT(is_internal_ordinary_ref(ref)); BIF_RET(ref); break; case ERTS_PORT_OP_DONE: @@ -428,7 +428,7 @@ BIF_RETTYPE erts_internal_port_info_1(BIF_ALIST_1) case ERTS_PORT_OP_DROPPED: BIF_RET(am_undefined); case ERTS_PORT_OP_SCHEDULED: - ASSERT(is_internal_ref(retval)); + ASSERT(is_internal_ordinary_ref(retval)); BIF_RET(retval); case ERTS_PORT_OP_DONE: ASSERT(is_not_internal_ref(retval)); @@ -467,7 +467,7 @@ BIF_RETTYPE erts_internal_port_info_2(BIF_ALIST_2) case ERTS_PORT_OP_DROPPED: BIF_RET(am_undefined); case ERTS_PORT_OP_SCHEDULED: - ASSERT(is_internal_ref(retval)); + ASSERT(is_internal_ordinary_ref(retval)); BIF_RET(retval); case ERTS_PORT_OP_DONE: ASSERT(is_not_internal_ref(retval)); diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index a480754bdb..f471390501 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -2363,7 +2363,7 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2) typedef struct { Process *proc; Eterm ref; - Eterm ref_heap[REF_THING_SIZE]; + Eterm ref_heap[ERTS_REF_THING_SIZE]; Eterm target; erts_smp_atomic32_t refc; } ErtsTraceDeliveredAll; diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c index 7c70217d8d..3fd4c87094 100644 --- a/erts/emulator/beam/erl_bif_unique.c +++ b/erts/emulator/beam/erl_bif_unique.c @@ -22,12 +22,15 @@ # include "config.h" #endif +#define ERL_BIF_UNIQUE_C__ #include "sys.h" #include "erl_vm.h" #include "erl_alloc.h" #include "export.h" #include "bif.h" #include "erl_bif_unique.h" +#include "hash.h" +#include "erl_binary.h" /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * Reference * @@ -58,9 +61,20 @@ static union { static Uint32 max_thr_id; #endif +static void init_magic_ref_tables(void); + +static Uint64 ref_init_value; + static void init_reference(void) { + SysTimeval tv; + sys_gettimeofday(&tv); + ref_init_value = 0; + ref_init_value |= (Uint64) tv.tv_sec; + ref_init_value |= ((Uint64) tv.tv_usec) << 32; + ref_init_value *= (Uint64) 268438039; + ref_init_value += (Uint64) tv.tv_usec; #ifdef DEBUG max_thr_id = (Uint32) erts_no_schedulers; #ifdef ERTS_DIRTY_SCHEDULERS @@ -68,11 +82,13 @@ init_reference(void) max_thr_id += (Uint32) erts_no_dirty_io_schedulers; #endif #endif - erts_atomic64_init_nob(&global_reference.count, 0); + erts_atomic64_init_nob(&global_reference.count, + (erts_aint64_t) ref_init_value); + init_magic_ref_tables(); } static ERTS_INLINE void -global_make_ref_in_array(Uint32 thr_id, Uint32 ref[ERTS_MAX_REF_NUMBERS]) +global_make_ref_in_array(Uint32 thr_id, Uint32 ref[ERTS_REF_NUMBERS]) { Uint64 value; @@ -82,7 +98,7 @@ global_make_ref_in_array(Uint32 thr_id, Uint32 ref[ERTS_MAX_REF_NUMBERS]) } static ERTS_INLINE void -make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]) +make_ref_in_array(Uint32 ref[ERTS_REF_NUMBERS]) { ErtsSchedulerData *esdp = erts_get_scheduler_data(); if (esdp) @@ -92,15 +108,23 @@ make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]) } void -erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]) +erts_make_ref_in_array(Uint32 ref[ERTS_REF_NUMBERS]) +{ + make_ref_in_array(ref); +} + +void +erts_make_magic_ref_in_array(Uint32 ref[ERTS_REF_NUMBERS]) { make_ref_in_array(ref); + ASSERT(!(ref[1] & ERTS_REF1_MAGIC_MARKER_BIT__)); + ref[1] |= ERTS_REF1_MAGIC_MARKER_BIT__; } -Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]) +Eterm erts_make_ref_in_buffer(Eterm buffer[ERTS_REF_THING_SIZE]) { Eterm* hp = buffer; - Uint32 ref[ERTS_MAX_REF_NUMBERS]; + Uint32 ref[ERTS_REF_NUMBERS]; make_ref_in_array(ref); write_ref_thing(hp, ref[0], ref[1], ref[2]); @@ -110,11 +134,11 @@ Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]) Eterm erts_make_ref(Process *c_p) { Eterm* hp; - Uint32 ref[ERTS_MAX_REF_NUMBERS]; + Uint32 ref[ERTS_REF_NUMBERS]; ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); - hp = HAlloc(c_p, REF_THING_SIZE); + hp = HAlloc(c_p, ERTS_REF_THING_SIZE); make_ref_in_array(ref); write_ref_thing(hp, ref[0], ref[1], ref[2]); @@ -122,6 +146,272 @@ Eterm erts_make_ref(Process *c_p) return make_internal_ref(hp); } +/* + * Magic reference tables + */ + +typedef struct { + HashBucket hash; + ErtsMagicBinary *mb; + Uint64 value; + Uint32 thr_id; +} ErtsMagicRefTableEntry; + +typedef struct { + erts_rwmtx_t rwmtx; + Hash hash; + char name[32]; +} ErtsMagicRefTable; + +typedef struct { + union { + ErtsMagicRefTable table; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsMagicRefTable))]; + } u; +} ErtsAlignedMagicRefTable; + +ErtsAlignedMagicRefTable *magic_ref_table; + +ErtsMagicBinary * +erts_magic_ref_lookup_bin__(Uint32 refn[ERTS_REF_NUMBERS]) +{ + ErtsMagicRefTableEntry tmpl; + ErtsMagicRefTableEntry *tep; + ErtsMagicBinary *mb; + ErtsMagicRefTable *tblp; + + ASSERT(erts_is_ref_numbers_magic(refn)); + + tmpl.value = erts_get_ref_numbers_value(refn); + tmpl.thr_id = erts_get_ref_numbers_thr_id(refn); + if (tmpl.thr_id > erts_no_schedulers) + tblp = &magic_ref_table[0].u.table; + else + tblp = &magic_ref_table[tmpl.thr_id].u.table; + + erts_rwmtx_rlock(&tblp->rwmtx); + + tep = (ErtsMagicRefTableEntry *) hash_get(&tblp->hash, &tmpl); + if (!tep) + mb = NULL; + else { + erts_aint_t refc; + mb = tep->mb; + refc = erts_refc_inc_unless(&mb->refc, 0, 0); + if (refc == 0) + mb = NULL; + } + + erts_rwmtx_runlock(&tblp->rwmtx); + + return mb; +} + +void +erts_magic_ref_save_bin__(Eterm ref) +{ + ErtsMagicRefTableEntry tmpl; + ErtsMagicRefTableEntry *tep; + ErtsMRefThing *mrtp; + ErtsMagicRefTable *tblp; + Uint32 *refn; + + ASSERT(is_internal_magic_ref(ref)); + + mrtp = (ErtsMRefThing *) internal_ref_val(ref); + refn = mrtp->mb->refn; + + tmpl.value = erts_get_ref_numbers_value(refn); + tmpl.thr_id = erts_get_ref_numbers_thr_id(refn); + + if (tmpl.thr_id > erts_no_schedulers) + tblp = &magic_ref_table[0].u.table; + else + tblp = &magic_ref_table[tmpl.thr_id].u.table; + + erts_rwmtx_rlock(&tblp->rwmtx); + + tep = (ErtsMagicRefTableEntry *) hash_get(&tblp->hash, &tmpl); + + erts_rwmtx_runlock(&tblp->rwmtx); + + if (!tep) { + ErtsMagicRefTableEntry *used_tep; + + ASSERT(tmpl.value == erts_get_ref_numbers_value(refn)); + ASSERT(tmpl.thr_id == erts_get_ref_numbers_thr_id(refn)); + + if (tblp != &magic_ref_table[0].u.table) { + tep = erts_alloc(ERTS_ALC_T_MREF_NSCHED_ENT, + sizeof(ErtsNSchedMagicRefTableEntry)); + } + else { + tep = erts_alloc(ERTS_ALC_T_MREF_ENT, + sizeof(ErtsMagicRefTableEntry)); + tep->thr_id = tmpl.thr_id; + } + + tep->value = tmpl.value; + tep->mb = mrtp->mb; + + erts_rwmtx_rwlock(&tblp->rwmtx); + + used_tep = hash_put(&tblp->hash, tep); + + erts_rwmtx_rwunlock(&tblp->rwmtx); + + if (used_tep != tep) { + if (tblp != &magic_ref_table[0].u.table) + erts_free(ERTS_ALC_T_MREF_NSCHED_ENT, (void *) tep); + else + erts_free(ERTS_ALC_T_MREF_ENT, (void *) tep); + } + } +} + +void +erts_magic_ref_remove_bin(Uint32 refn[ERTS_REF_NUMBERS]) +{ + ErtsMagicRefTableEntry tmpl; + ErtsMagicRefTableEntry *tep; + ErtsMagicRefTable *tblp; + + tmpl.value = erts_get_ref_numbers_value(refn); + tmpl.thr_id = erts_get_ref_numbers_thr_id(refn); + + if (tmpl.thr_id > erts_no_schedulers) + tblp = &magic_ref_table[0].u.table; + else + tblp = &magic_ref_table[tmpl.thr_id].u.table; + + erts_rwmtx_rlock(&tblp->rwmtx); + + tep = (ErtsMagicRefTableEntry *) hash_get(&tblp->hash, &tmpl); + + erts_rwmtx_runlock(&tblp->rwmtx); + + if (tep) { + + ASSERT(tmpl.value == erts_get_ref_numbers_value(refn)); + ASSERT(tmpl.thr_id == erts_get_ref_numbers_thr_id(refn)); + + erts_rwmtx_rwlock(&tblp->rwmtx); + + tep = hash_remove(&tblp->hash, &tmpl); + ASSERT(tep); + + erts_rwmtx_rwunlock(&tblp->rwmtx); + + if (tblp != &magic_ref_table[0].u.table) + erts_free(ERTS_ALC_T_MREF_NSCHED_ENT, (void *) tep); + else + erts_free(ERTS_ALC_T_MREF_ENT, (void *) tep); + } +} + +static int nsched_mreft_cmp(void *ve1, void *ve2) +{ + ErtsNSchedMagicRefTableEntry *e1 = ve1; + ErtsNSchedMagicRefTableEntry *e2 = ve2; + return e1->value != e2->value; +} + +static int non_nsched_mreft_cmp(void *ve1, void *ve2) +{ + ErtsMagicRefTableEntry *e1 = ve1; + ErtsMagicRefTableEntry *e2 = ve2; + return e1->value != e2->value || e1->thr_id != e2->thr_id; +} + +static HashValue nsched_mreft_hash(void *ve) +{ + ErtsNSchedMagicRefTableEntry *e = ve; + return (HashValue) e->value; +} + +static HashValue non_nsched_mreft_hash(void *ve) +{ + ErtsMagicRefTableEntry *e = ve; + HashValue h; + h = (HashValue) e->thr_id; + h *= 268440163; + h += (HashValue) e->value; + return h; +} + +static void *mreft_alloc(void *ve) +{ + /* + * We allocate the element before + * hash_put() and pass it as + * template which we get as + * input... + */ + return ve; +} + +static void mreft_free(void *ve) +{ + /* + * We free the element ourselves + * after hash_remove()... + */ +} + +static void *mreft_meta_alloc(int i, size_t size) +{ + return erts_alloc(ERTS_ALC_T_MREF_TAB_BKTS, size); +} + +static void mreft_meta_free(int i, void *ptr) +{ + erts_free(ERTS_ALC_T_MREF_TAB_BKTS, ptr); +} + +static void +init_magic_ref_tables(void) +{ + HashFunctions hash_funcs; + int i; + ErtsMagicRefTable *tblp; + + magic_ref_table = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_MREF_TAB, + (sizeof(ErtsAlignedMagicRefTable) + * (erts_no_schedulers + 1))); + + hash_funcs.hash = non_nsched_mreft_hash; + hash_funcs.cmp = non_nsched_mreft_cmp; + + hash_funcs.alloc = mreft_alloc; + hash_funcs.free = mreft_free; + hash_funcs.meta_alloc = mreft_meta_alloc; + hash_funcs.meta_free = mreft_meta_free; + hash_funcs.meta_print = erts_print; + + tblp = &magic_ref_table[0].u.table; + erts_snprintf(&tblp->name[0], sizeof(tblp->name), + "magic_ref_table_0"); + hash_init(0, &tblp->hash, &tblp->name[0], 1, hash_funcs); + erts_rwmtx_init(&tblp->rwmtx, "magic_ref_table"); + + hash_funcs.hash = nsched_mreft_hash; + hash_funcs.cmp = nsched_mreft_cmp; + + for (i = 1; i <= erts_no_schedulers; i++) { + ErtsMagicRefTable *tblp = &magic_ref_table[i].u.table; + erts_snprintf(&tblp->name[0], sizeof(tblp->name), + "magic_ref_table_%d", i); + hash_init(0, &tblp->hash, &tblp->name[0], 1, hash_funcs); + erts_rwmtx_init(&tblp->rwmtx, "magic_ref_table"); + } +} + +void erts_ref_bin_free(ErtsMagicBinary *mb) +{ + erts_bin_free((Binary *) mb); +} + + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * Unique Integer * \* */ @@ -498,7 +788,7 @@ void erts_sched_bif_unique_init(ErtsSchedulerData *esdp) { esdp->unique = (Uint64) 0; - esdp->ref = (Uint64) 0; + esdp->ref = (Uint64) ref_init_value; } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ @@ -513,7 +803,7 @@ BIF_RETTYPE make_ref_0(BIF_ALIST_0) ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P)); - hp = HAlloc(BIF_P, REF_THING_SIZE); + hp = HAlloc(BIF_P, ERTS_REF_THING_SIZE); res = erts_sched_make_ref_in_buffer(erts_proc_sched_data(BIF_P), hp); diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h index c6481864d0..6cd6cc3e09 100644 --- a/erts/emulator/beam/erl_bif_unique.h +++ b/erts/emulator/beam/erl_bif_unique.h @@ -21,16 +21,25 @@ #ifndef ERTS_BIF_UNIQUE_H__ #define ERTS_BIF_UNIQUE_H__ +#include "erl_term.h" #include "erl_process.h" #include "big.h" +#define ERTS_BINARY_TYPES_ONLY__ +#include "erl_binary.h" +#undef ERTS_BINARY_TYPES_ONLY__ void erts_bif_unique_init(void); void erts_sched_bif_unique_init(ErtsSchedulerData *esdp); /* reference */ Eterm erts_make_ref(Process *); -Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]); -void erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]); +Eterm erts_make_ref_in_buffer(Eterm buffer[ERTS_REF_THING_SIZE]); +void erts_make_ref_in_array(Uint32 ref[ERTS_REF_NUMBERS]); +void erts_make_magic_ref_in_array(Uint32 ref[ERTS_REF_NUMBERS]); +void erts_magic_ref_remove_bin(Uint32 refn[ERTS_REF_NUMBERS]); +void erts_magic_ref_save_bin__(Eterm ref); +ErtsMagicBinary *erts_magic_ref_lookup_bin__(Uint32 refn[ERTS_REF_NUMBERS]); + /* strict monotonic counter */ @@ -67,19 +76,35 @@ Eterm erts_debug_make_unique_integer(Process *c_p, Eterm etval1); -ERTS_GLB_INLINE void erts_set_ref_numbers(Uint32 ref[ERTS_MAX_REF_NUMBERS], +ERTS_GLB_INLINE void erts_set_ref_numbers(Uint32 ref[ERTS_REF_NUMBERS], Uint32 thr_id, Uint64 value); -ERTS_GLB_INLINE Uint32 erts_get_ref_numbers_thr_id(Uint32 ref[ERTS_MAX_REF_NUMBERS]); -ERTS_GLB_INLINE Uint64 erts_get_ref_numbers_value(Uint32 ref[ERTS_MAX_REF_NUMBERS]); +ERTS_GLB_INLINE Uint32 erts_get_ref_numbers_thr_id(Uint32 ref[ERTS_REF_NUMBERS]); +ERTS_GLB_INLINE int erts_is_ref_numbers_magic(Uint32 ref[ERTS_REF_NUMBERS]); +ERTS_GLB_INLINE Uint64 erts_get_ref_numbers_value(Uint32 ref[ERTS_REF_NUMBERS]); ERTS_GLB_INLINE void erts_sched_make_ref_in_array(ErtsSchedulerData *esdp, - Uint32 ref[ERTS_MAX_REF_NUMBERS]); + Uint32 ref[ERTS_REF_NUMBERS]); +ERTS_GLB_INLINE void erts_sched_make_magic_ref_in_array(ErtsSchedulerData *esdp, + Uint32 ref[ERTS_REF_NUMBERS]); ERTS_GLB_INLINE Eterm erts_sched_make_ref_in_buffer(ErtsSchedulerData *esdp, - Eterm buffer[REF_THING_SIZE]); + Eterm buffer[ERTS_REF_THING_SIZE]); +ERTS_GLB_INLINE Eterm erts_mk_magic_ref(Eterm **hpp, ErlOffHeap *ohp, Binary *mbp); +ERTS_GLB_INLINE Binary *erts_magic_ref2bin(Eterm mref); +ERTS_GLB_INLINE void erts_magic_ref_save_bin(Eterm ref); +ERTS_GLB_INLINE ErtsMagicBinary *erts_magic_ref_lookup_bin(Uint32 ref[ERTS_REF_NUMBERS]); + +#define ERTS_REF1_MAGIC_MARKER_BIT_NO__ \ + (_REF_NUM_SIZE-1) +#define ERTS_REF1_MAGIC_MARKER_BIT__ \ + (((Uint32) 1) << ERTS_REF1_MAGIC_MARKER_BIT_NO__) +#define ERTS_REF1_THR_ID_MASK__ \ + (ERTS_REF1_MAGIC_MARKER_BIT__-1) +#define ERTS_REF1_NUM_MASK__ \ + (~(ERTS_REF1_THR_ID_MASK__|ERTS_REF1_MAGIC_MARKER_BIT__)) #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void -erts_set_ref_numbers(Uint32 ref[ERTS_MAX_REF_NUMBERS], Uint32 thr_id, Uint64 value) +erts_set_ref_numbers(Uint32 ref[ERTS_REF_NUMBERS], Uint32 thr_id, Uint64 value) { /* * We cannot use thread id in the first 18-bit word since @@ -87,30 +112,39 @@ erts_set_ref_numbers(Uint32 ref[ERTS_MAX_REF_NUMBERS], Uint32 thr_id, Uint64 val * we did, we would get really poor hash values. Instead * we have to shuffle the bits a bit. */ - ASSERT(thr_id == (thr_id & ((Uint32) 0x3ffff))); - ref[0] = (Uint32) (value & ((Uint64) 0x3ffff)); - ref[1] = (((Uint32) (value & ((Uint64) 0xfffc0000))) - | (thr_id & ((Uint32) 0x3ffff))); + ASSERT(thr_id == (thr_id & ((Uint32) ERTS_REF1_THR_ID_MASK__))); + ref[0] = (Uint32) (value & ((Uint64) REF_MASK)); + ref[1] = (((Uint32) (value & ((Uint64) ERTS_REF1_NUM_MASK__))) + | (thr_id & ((Uint32) ERTS_REF1_THR_ID_MASK__))); ref[2] = (Uint32) ((value >> 32) & ((Uint64) 0xffffffff)); } ERTS_GLB_INLINE Uint32 -erts_get_ref_numbers_thr_id(Uint32 ref[ERTS_MAX_REF_NUMBERS]) +erts_get_ref_numbers_thr_id(Uint32 ref[ERTS_REF_NUMBERS]) { - return ref[1] & ((Uint32) 0x3ffff); + return ref[1] & ((Uint32) ERTS_REF1_THR_ID_MASK__); +} + +ERTS_GLB_INLINE int +erts_is_ref_numbers_magic(Uint32 ref[ERTS_REF_NUMBERS]) +{ + return !!(ref[1] & ERTS_REF1_MAGIC_MARKER_BIT__); } ERTS_GLB_INLINE Uint64 -erts_get_ref_numbers_value(Uint32 ref[ERTS_MAX_REF_NUMBERS]) +erts_get_ref_numbers_value(Uint32 ref[ERTS_REF_NUMBERS]) { + ERTS_CT_ASSERT((ERTS_REF1_NUM_MASK__ | REF_MASK) == 0xffffffff); + ERTS_CT_ASSERT((ERTS_REF1_NUM_MASK__ & REF_MASK) == 0); + return (((((Uint64) ref[2]) & ((Uint64) 0xffffffff)) << 32) - | (((Uint64) ref[1]) & ((Uint64) 0xfffc0000)) - | (((Uint64) ref[0]) & ((Uint64) 0x3ffff))); + | (((Uint64) ref[1]) & ((Uint64) ERTS_REF1_NUM_MASK__)) + | (((Uint64) ref[0]) & ((Uint64) REF_MASK))); } ERTS_GLB_INLINE void erts_sched_make_ref_in_array(ErtsSchedulerData *esdp, - Uint32 ref[ERTS_MAX_REF_NUMBERS]) + Uint32 ref[ERTS_REF_NUMBERS]) { Uint64 value; @@ -119,18 +153,286 @@ erts_sched_make_ref_in_array(ErtsSchedulerData *esdp, erts_set_ref_numbers(ref, (Uint32) esdp->thr_id, value); } +ERTS_GLB_INLINE void +erts_sched_make_magic_ref_in_array(ErtsSchedulerData *esdp, + Uint32 ref[ERTS_REF_NUMBERS]) +{ + erts_sched_make_ref_in_array(esdp, ref); + ASSERT(!(ref[1] & ERTS_REF1_MAGIC_MARKER_BIT__)); + ref[1] |= ERTS_REF1_MAGIC_MARKER_BIT__; +} + ERTS_GLB_INLINE Eterm erts_sched_make_ref_in_buffer(ErtsSchedulerData *esdp, - Eterm buffer[REF_THING_SIZE]) + Eterm buffer[ERTS_REF_THING_SIZE]) { Eterm* hp = buffer; - Uint32 ref[ERTS_MAX_REF_NUMBERS]; + Uint32 ref[ERTS_REF_NUMBERS]; erts_sched_make_ref_in_array(esdp, ref); write_ref_thing(hp, ref[0], ref[1], ref[2]); return make_internal_ref(hp); } +ERTS_GLB_INLINE Eterm +erts_mk_magic_ref(Eterm **hpp, ErlOffHeap *ohp, Binary *bp) +{ + Eterm *hp = *hpp; + ASSERT(bp->flags & BIN_FLAG_MAGIC); + write_magic_ref_thing(hp, ohp, (ErtsMagicBinary *) bp); + *hpp += ERTS_MAGIC_REF_THING_SIZE; + erts_refc_inc(&bp->refc, 1); + return make_internal_ref(hp); +} + +ERTS_GLB_INLINE Binary * +erts_magic_ref2bin(Eterm mref) +{ + ErtsMRefThing *mrtp; + ASSERT(is_internal_magic_ref(mref)); + mrtp = (ErtsMRefThing *) internal_ref_val(mref); + return (Binary *) mrtp->mb; +} + +/* + * Save the magic binary of a ref when the + * ref is exposed to the outside world... + */ +ERTS_GLB_INLINE void +erts_magic_ref_save_bin(Eterm ref) +{ + if (is_internal_magic_ref(ref)) + erts_magic_ref_save_bin__(ref); +} + +/* + * Look up the magic binary of a magic ref + * when the ref comes from the outside world... + */ +ERTS_GLB_INLINE ErtsMagicBinary * +erts_magic_ref_lookup_bin(Uint32 ref[ERTS_REF_NUMBERS]) +{ + if (!erts_is_ref_numbers_magic(ref)) + return NULL; + return erts_magic_ref_lookup_bin__(ref); +} + + #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +/* + * Storage of internal refs in misc structures... + */ + +#include "erl_message.h" + +#if ERTS_REF_NUMBERS != 3 +# error fix this... +#endif + +ERTS_GLB_INLINE int erts_internal_ref_number_cmp(Uint32 num1[ERTS_REF_NUMBERS], + Uint32 num2[ERTS_REF_NUMBERS]); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE int +erts_internal_ref_number_cmp(Uint32 num1[ERTS_REF_NUMBERS], + Uint32 num2[ERTS_REF_NUMBERS]) +{ + if (num1[2] != num2[2]) + return (int) ((Sint64) num1[2] - (Sint64) num2[2]); + if (num1[1] != num2[1]) + return (int) ((Sint64) num1[1] - (Sint64) num2[1]); + if (num1[0] != num2[0]) + return (int) ((Sint64) num1[0] - (Sint64) num2[0]); + return 0; +} + +#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +/* Iref storage for all internal references... */ +typedef struct { + Uint32 is_magic; + union { + ErtsMagicBinary *mb; + Uint32 num[ERTS_REF_NUMBERS]; + } u; +} ErtsIRefStorage; + +void erts_ref_bin_free(ErtsMagicBinary *mb); + +ERTS_GLB_INLINE void erts_iref_storage_save(ErtsIRefStorage *iref, Eterm ref); +ERTS_GLB_INLINE void erts_iref_storage_clean(ErtsIRefStorage *iref); +ERTS_GLB_INLINE Uint erts_iref_storage_heap_size(ErtsIRefStorage *iref); +ERTS_GLB_INLINE Eterm erts_iref_storage_make_ref(ErtsIRefStorage *iref, + Eterm **hpp, ErlOffHeap *ohp, + int clean_storage); +ERTS_GLB_INLINE int erts_iref_storage_cmp(ErtsIRefStorage *iref1, + ErtsIRefStorage *iref2); + + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE void +erts_iref_storage_save(ErtsIRefStorage *iref, Eterm ref) +{ + Eterm *hp; + + ERTS_CT_ASSERT(ERTS_REF_NUMBERS == 3); + ASSERT(is_internal_ref(ref)); + + hp = boxed_val(ref); + + if (is_ordinary_ref_thing(hp)) { + ErtsORefThing *rtp = (ErtsORefThing *) hp; + iref->is_magic = 0; + iref->u.num[0] = rtp->num[0]; + iref->u.num[1] = rtp->num[1]; + iref->u.num[2] = rtp->num[2]; + } + else { + ErtsMRefThing *mrtp = (ErtsMRefThing *) hp; + ASSERT(is_magic_ref_thing(hp)); + iref->is_magic = 1; + iref->u.mb = mrtp->mb; + erts_refc_inc(&mrtp->mb->refc, 1); + } +} + +ERTS_GLB_INLINE void +erts_iref_storage_clean(ErtsIRefStorage *iref) +{ + if (iref->is_magic && erts_refc_dectest(&iref->u.mb->refc, 0) == 0) + erts_ref_bin_free(iref->u.mb); +#ifdef DEBUG + memset((void *) iref, 0xf, sizeof(ErtsIRefStorage)); +#endif +} + +ERTS_GLB_INLINE Uint +erts_iref_storage_heap_size(ErtsIRefStorage *iref) +{ + return iref->is_magic ? ERTS_MAGIC_REF_THING_SIZE : ERTS_REF_THING_SIZE; +} + +ERTS_GLB_INLINE Eterm +erts_iref_storage_make_ref(ErtsIRefStorage *iref, + Eterm **hpp, ErlOffHeap *ohp, + int clean_storage) +{ + Eterm *hp = *hpp; + if (!iref->is_magic) { + write_ref_thing(hp, iref->u.num[0], iref->u.num[1], + iref->u.num[2]); + *hpp += ERTS_REF_THING_SIZE; + } + else { + write_magic_ref_thing(hp, ohp, iref->u.mb); + *hpp += ERTS_MAGIC_REF_THING_SIZE; + /* + * If we clean storage, the term inherits the + * refc increment of the cleaned storage... + */ + if (!clean_storage) + erts_refc_inc(&iref->u.mb->refc, 1); + } + +#ifdef DEBUG + if (clean_storage) + memset((void *) iref, 0xf, sizeof(ErtsIRefStorage)); +#endif + + return make_internal_ref(hp); +} + +ERTS_GLB_INLINE int +erts_iref_storage_cmp(ErtsIRefStorage *iref1, + ErtsIRefStorage *iref2) +{ + Uint32 *num1 = iref1->is_magic ? iref1->u.mb->refn : iref1->u.num; + Uint32 *num2 = iref2->is_magic ? iref2->u.mb->refn : iref2->u.num; + return erts_internal_ref_number_cmp(num1, num2); +} + +#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ + + +/* OIref storage for ordinary internal references only... */ +typedef struct { + Uint32 num[ERTS_REF_NUMBERS]; +} ErtsOIRefStorage; + +ERTS_GLB_INLINE void erts_oiref_storage_save(ErtsOIRefStorage *oiref, + Eterm ref); +ERTS_GLB_INLINE Eterm erts_oiref_storage_make_ref(ErtsOIRefStorage *oiref, + Eterm **hpp); +ERTS_GLB_INLINE int erts_oiref_storage_cmp(ErtsOIRefStorage *oiref1, + ErtsOIRefStorage *oiref2); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE void +erts_oiref_storage_save(ErtsOIRefStorage *oiref, Eterm ref) +{ + ErtsORefThing *rtp; + ERTS_CT_ASSERT(ERTS_REF_NUMBERS == 3); + ASSERT(is_internal_ordinary_ref(ref)); + + rtp = (ErtsORefThing *) internal_ref_val(ref); + + oiref->num[0] = rtp->num[0]; + oiref->num[1] = rtp->num[1]; + oiref->num[2] = rtp->num[2]; +} + +ERTS_GLB_INLINE Eterm +erts_oiref_storage_make_ref(ErtsOIRefStorage *oiref, Eterm **hpp) +{ + Eterm *hp = *hpp; + ERTS_CT_ASSERT(ERTS_REF_NUMBERS == 3); + write_ref_thing(hp, oiref->num[0], oiref->num[1], oiref->num[2]); + *hpp += ERTS_REF_THING_SIZE; + return make_internal_ref(hp); +} + +ERTS_GLB_INLINE int +erts_oiref_storage_cmp(ErtsOIRefStorage *oiref1, + ErtsOIRefStorage *oiref2) +{ + return erts_internal_ref_number_cmp(oiref1->num, oiref2->num); +} + +#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ + + +ERTS_GLB_INLINE Eterm +erts_proc_store_ref(Process *c_p, Uint32 ref[ERTS_MAX_REF_NUMBERS]); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE Eterm +erts_proc_store_ref(Process *c_p, Uint32 ref[ERTS_MAX_REF_NUMBERS]) +{ + Eterm *hp = HAlloc(c_p, ERTS_REF_THING_SIZE); + write_ref_thing(hp, ref[0], ref[1], ref[2]); + return make_internal_ref(hp); +} + +#endif + #endif /* ERTS_BIF_UNIQUE_H__ */ + +#if (defined(ERTS_ALLOC_C__) || defined(ERL_BIF_UNIQUE_C__)) \ + && !defined(ERTS_BIF_UNIQUE_H__FIX_ALLOC_TYPES__) +#define ERTS_BIF_UNIQUE_H__FIX_ALLOC_TYPES__ + +#include "hash.h" + +typedef struct { + HashBucket hash; + ErtsMagicBinary *mb; + Uint64 value; +} ErtsNSchedMagicRefTableEntry; + +#endif diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index fdc5efea98..c811a002ce 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -18,11 +18,146 @@ * %CopyrightEnd% */ -#ifndef __ERL_BINARY_H -#define __ERL_BINARY_H +#ifndef ERL_BINARY_H__TYPES__ +#define ERL_BINARY_H__TYPES__ + +/* +** Just like the driver binary but with initial flags +** Note that the two structures Binary and ErlDrvBinary HAVE to +** be equal except for extra fields in the beginning of the struct. +** ErlDrvBinary is defined in erl_driver.h. +** When driver_alloc_binary is called, a Binary is allocated, but +** the pointer returned is to the address of the first element that +** also occurs in the ErlDrvBinary struct (driver.*binary takes care if this). +** The driver need never know about additions to the internal Binary of the +** emulator. One should however NEVER be sloppy when mixing ErlDrvBinary +** and Binary, the macros below can convert one type to the other, as they both +** in reality are equal. +*/ + +#ifdef ARCH_32 + /* *DO NOT USE* only for alignment. */ +#define ERTS_BINARY_STRUCT_ALIGNMENT Uint32 align__; +#else +#define ERTS_BINARY_STRUCT_ALIGNMENT +#endif + +/* Add fields in ERTS_INTERNAL_BINARY_FIELDS, otherwise the drivers crash */ +#define ERTS_INTERNAL_BINARY_FIELDS \ + UWord flags; \ + erts_refc_t refc; \ + ERTS_BINARY_STRUCT_ALIGNMENT + +typedef struct binary { + ERTS_INTERNAL_BINARY_FIELDS + SWord orig_size; + char orig_bytes[1]; /* to be continued */ +} Binary; + +#define ERTS_SIZEOF_Binary(Sz) \ + (offsetof(Binary,orig_bytes) + (Sz)) + +#if ERTS_REF_NUMBERS != 3 +#error "Update ErtsMagicBinary" +#endif + +typedef struct magic_binary ErtsMagicBinary; +struct magic_binary { + ERTS_INTERNAL_BINARY_FIELDS + SWord orig_size; + void (*destructor)(Binary *); + Uint32 refn[ERTS_REF_NUMBERS]; + ErtsAlcType_t alloc_type; + union { + struct { + ERTS_BINARY_STRUCT_ALIGNMENT + char data[1]; + } aligned; + struct { + char data[1]; + } unaligned; + } u; +}; + +#ifdef ARCH_32 +#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 4 +#else +#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 0 +#endif + +typedef union { + Binary binary; + ErtsMagicBinary magic_binary; + struct { + ERTS_INTERNAL_BINARY_FIELDS + ErlDrvBinary binary; + } driver; +} ErtsBinary; + +/* + * 'Binary' alignment: + * Address of orig_bytes[0] of a Binary should always be 8-byte aligned. + * It is assumed that the flags, refc, and orig_size fields are 4 bytes on + * 32-bits architectures and 8 bytes on 64-bits architectures. + */ + +#define ERTS_MAGIC_BIN_REFN(BP) \ + ((ErtsBinary *) (BP))->magic_binary.refn +#define ERTS_MAGIC_BIN_ATYPE(BP) \ + ((ErtsBinary *) (BP))->magic_binary.alloc_type +#define ERTS_MAGIC_DATA_OFFSET \ + (offsetof(ErtsMagicBinary,u.aligned.data) - offsetof(Binary,orig_bytes)) +#define ERTS_MAGIC_BIN_DESTRUCTOR(BP) \ + ((ErtsBinary *) (BP))->magic_binary.destructor +#define ERTS_MAGIC_BIN_DATA(BP) \ + ((void *) ((ErtsBinary *) (BP))->magic_binary.u.aligned.data) +#define ERTS_MAGIC_BIN_DATA_SIZE(BP) \ + ((BP)->orig_size - ERTS_MAGIC_DATA_OFFSET) +#define ERTS_MAGIC_DATA_OFFSET \ + (offsetof(ErtsMagicBinary,u.aligned.data) - offsetof(Binary,orig_bytes)) +#define ERTS_MAGIC_BIN_ORIG_SIZE(Sz) \ + (ERTS_MAGIC_DATA_OFFSET + (Sz)) +#define ERTS_MAGIC_BIN_SIZE(Sz) \ + (offsetof(ErtsMagicBinary,u.aligned.data) + (Sz)) +#define ERTS_MAGIC_BIN_FROM_DATA(DATA) \ + ((ErtsBinary*)((char*)(DATA) - offsetof(ErtsMagicBinary,u.aligned.data))) + +/* On 32-bit arch these macro variants will save memory + by not forcing 8-byte alignment for the magic payload. +*/ +#define ERTS_MAGIC_BIN_UNALIGNED_DATA(BP) \ + ((void *) ((ErtsBinary *) (BP))->magic_binary.u.unaligned.data) +#define ERTS_MAGIC_UNALIGNED_DATA_OFFSET \ + (offsetof(ErtsMagicBinary,u.unaligned.data) - offsetof(Binary,orig_bytes)) +#define ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(BP) \ + ((BP)->orig_size - ERTS_MAGIC_UNALIGNED_DATA_OFFSET) +#define ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(Sz) \ + (ERTS_MAGIC_UNALIGNED_DATA_OFFSET + (Sz)) +#define ERTS_MAGIC_BIN_UNALIGNED_SIZE(Sz) \ + (offsetof(ErtsMagicBinary,u.unaligned.data) + (Sz)) +#define ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(DATA) \ + ((ErtsBinary*)((char*)(DATA) - offsetof(ErtsMagicBinary,u.unaligned.data))) + + +#define Binary2ErlDrvBinary(B) (&((ErtsBinary *) (B))->driver.binary) +#define ErlDrvBinary2Binary(D) ((Binary *) \ + (((char *) (D)) \ + - offsetof(ErtsBinary, driver.binary))) + +/* A "magic" binary flag */ +#define BIN_FLAG_MAGIC 1 +#define BIN_FLAG_USR1 2 /* Reserved for use by different modules too mark */ +#define BIN_FLAG_USR2 4 /* certain binaries as special (used by ets) */ +#define BIN_FLAG_DRV 8 + +#endif /* ERL_BINARY_H__TYPES__ */ + +#if !defined(ERL_BINARY_H__) && !defined(ERTS_BINARY_TYPES_ONLY__) +#define ERL_BINARY_H__ #include "erl_threads.h" #include "bif.h" +#include "erl_bif_unique.h" /* * Maximum number of bytes to place in a heap binary. @@ -190,6 +325,7 @@ ERTS_GLB_INLINE Binary *erts_bin_realloc(Binary *bp, Uint size); ERTS_GLB_INLINE void erts_bin_free(Binary *bp); ERTS_GLB_INLINE Binary *erts_create_magic_binary_x(Uint size, void (*destructor)(Binary *), + ErtsAlcType_t alloc_type, int unaligned); ERTS_GLB_INLINE Binary *erts_create_magic_binary(Uint size, void (*destructor)(Binary *)); @@ -320,9 +456,12 @@ erts_bin_realloc(Binary *bp, Uint size) ERTS_GLB_INLINE void erts_bin_free(Binary *bp) { - if (bp->flags & BIN_FLAG_MAGIC) + if (bp->flags & BIN_FLAG_MAGIC) { ERTS_MAGIC_BIN_DESTRUCTOR(bp)(bp); - if (bp->flags & BIN_FLAG_DRV) + erts_magic_ref_remove_bin(ERTS_MAGIC_BIN_REFN(bp)); + erts_free(ERTS_MAGIC_BIN_ATYPE(bp), (void *) bp); + } + else if (bp->flags & BIN_FLAG_DRV) erts_free(ERTS_ALC_T_DRV_BINARY, (void *) bp); else erts_free(ERTS_ALC_T_BINARY, (void *) bp); @@ -330,29 +469,33 @@ erts_bin_free(Binary *bp) ERTS_GLB_INLINE Binary * erts_create_magic_binary_x(Uint size, void (*destructor)(Binary *), + ErtsAlcType_t alloc_type, int unaligned) { Uint bsize = unaligned ? ERTS_MAGIC_BIN_UNALIGNED_SIZE(size) : ERTS_MAGIC_BIN_SIZE(size); - Binary* bptr = erts_alloc_fnf(ERTS_ALC_T_BINARY, bsize); + Binary* bptr = erts_alloc_fnf(alloc_type, bsize); ASSERT(bsize > size); if (!bptr) - erts_alloc_n_enomem(ERTS_ALC_T2N(ERTS_ALC_T_BINARY), bsize); + erts_alloc_n_enomem(ERTS_ALC_T2N(alloc_type), bsize); ERTS_CHK_BIN_ALIGNMENT(bptr); bptr->flags = BIN_FLAG_MAGIC; bptr->orig_size = unaligned ? ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(size) : ERTS_MAGIC_BIN_ORIG_SIZE(size); erts_refc_init(&bptr->refc, 0); ERTS_MAGIC_BIN_DESTRUCTOR(bptr) = destructor; + ERTS_MAGIC_BIN_ATYPE(bptr) = alloc_type; + erts_make_magic_ref_in_array(ERTS_MAGIC_BIN_REFN(bptr)); return bptr; } ERTS_GLB_INLINE Binary * erts_create_magic_binary(Uint size, void (*destructor)(Binary *)) { - return erts_create_magic_binary_x(size, destructor, 0); + return erts_create_magic_binary_x(size, destructor, + ERTS_ALC_T_BINARY, 0); } #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ -#endif /* !__ERL_BINARY_H */ +#endif /* !ERL_BINARY_H__ */ diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 5537935802..0ab42394ce 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -5289,24 +5289,35 @@ void db_match_dis(Binary *bp) case matchEqRef: ++t; { - RefThing *rt = (RefThing *) t; + Uint32 *num; int ri; - n = thing_arityval(rt->header); - erts_printf("EqRef\t(%d) {", (int) n); + + if (is_ordinary_ref_thing(t)) { + ErtsORefThing *rt = (ErtsORefThing *) t; + num = rt->num; + t += TermWords(ERTS_REF_THING_SIZE); + } + else { + ErtsMRefThing *mrt = (ErtsMRefThing *) t; + ASSERT(is_magic_ref_thing(t)); + num = mrt->mb->refn; + t += TermWords(ERTS_MAGIC_REF_THING_SIZE); + } + + erts_printf("EqRef\t(%d) {", (int) ERTS_REF_NUMBERS); first = 1; - for (ri = 0; ri < n; ++ri) { + for (ri = 0; ri < ERTS_REF_NUMBERS; ++ri) { if (first) first = 0; else erts_printf(", "); #if defined(ARCH_64) - erts_printf("0x%016bex", rt->data.ui[ri]); + erts_printf("0x%016bex", num[ri]); #else - erts_printf("0x%08bex", rt->data.ui[ri]); + erts_printf("0x%08bex", num[ri]); #endif } } - t += TermWords(REF_THING_SIZE); erts_printf("}\n"); break; case matchEqBig: diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c index 3526bb684d..517dad9cbb 100644 --- a/erts/emulator/beam/erl_debug.c +++ b/erts/emulator/beam/erl_debug.c @@ -130,7 +130,7 @@ pdisplay1(fmtfn_t to, void *to_arg, Process* p, Eterm obj) Uint32 *ref_num; erts_print(to, to_arg, "#Ref<%lu", ref_channel_no(obj)); ref_num = ref_numbers(obj); - for (i = ref_no_of_numbers(obj)-1; i >= 0; i--) + for (i = ref_no_numbers(obj)-1; i >= 0; i--) erts_print(to, to_arg, ",%lu", ref_num[i]); erts_print(to, to_arg, ">"); break; diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 897dbbe82b..807d9d15e4 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -178,7 +178,7 @@ Uint erts_test_long_gc_sleep; /* Only used for testing... */ typedef struct { Process *proc; Eterm ref; - Eterm ref_heap[REF_THING_SIZE]; + Eterm ref_heap[ERTS_REF_THING_SIZE]; Uint req_sched; erts_smp_atomic32_t refc; } ErtsGCInfoReq; @@ -2354,6 +2354,9 @@ copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap, switch (val & _HEADER_SUBTAG_MASK) { case ARITYVAL_SUBTAG: break; + case REF_SUBTAG: + if (is_ordinary_ref_thing(fhp - 1)) + goto the_default; case REFC_BINARY_SUBTAG: case FUN_SUBTAG: case EXTERNAL_PID_SUBTAG: @@ -2363,6 +2366,7 @@ copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap, cpy_sz = thing_arityval(val); goto cpy_words; default: + the_default: cpy_sz = header_arity(val); cpy_words: @@ -2862,6 +2866,15 @@ sweep_off_heap(Process *p, int fullsweep) } break; } + case REF_SUBTAG: + { + ErtsMagicBinary *bptr; + ASSERT(is_magic_ref_thing(ptr)); + bptr = ((ErtsMRefThing *) ptr)->mb; + if (erts_refc_dectest(&bptr->refc, 0) == 0) + erts_bin_free((Binary *) bptr); + break; + } default: ASSERT(is_external_header(ptr->thing_word)); erts_deref_node_entry(((ExternalThing*)ptr)->node); @@ -2883,7 +2896,8 @@ sweep_off_heap(Process *p, int fullsweep) } else { ASSERT(is_fun_header(ptr->thing_word) || - is_external_header(ptr->thing_word)); + is_external_header(ptr->thing_word) + || is_magic_ref_thing(ptr)); prev = &ptr->next; ptr = ptr->next; } @@ -2978,6 +2992,9 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) } tari = thing_arityval(val); switch (thing_subtag(val)) { + case REF_SUBTAG: + if (is_ordinary_ref_thing(hp)) + break; case REFC_BINARY_SUBTAG: case FUN_SUBTAG: case EXTERNAL_PID_SUBTAG: @@ -3175,7 +3192,7 @@ reply_gc_info(void *vgcirp) if (hpp) ref_copy = STORE_NC(hpp, ohp, gcirp->ref); else - *szp += REF_THING_SIZE; + *szp += ERTS_REF_THING_SIZE; msg = erts_bld_tuple(hpp, szp, 3, make_small(esdp->no), @@ -3526,6 +3543,10 @@ erts_check_off_heap2(Process *p, Eterm *htop) case EXTERNAL_REF_SUBTAG: refc = erts_smp_refc_read(&u.ext->node->refc, 1); break; + case REF_SUBTAG: + ASSERT(is_magic_ref_thing(u.hdr)); + refc = erts_refc_read(&u.mref->mb->refc, 1); + break; default: ASSERT(!"erts_check_off_heap2: Invalid thing_word"); } diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index 647fa26811..26be8c7edf 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -1771,7 +1771,7 @@ setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos, esdp = erts_proc_sched_data(c_p); - hp = HAlloc(c_p, REF_THING_SIZE); + hp = HAlloc(c_p, ERTS_REF_THING_SIZE); ref = erts_sched_make_ref_in_buffer(esdp, hp); ASSERT(erts_get_ref_numbers_thr_id( @@ -1939,7 +1939,7 @@ access_sched_local_btm(Process *c_p, Eterm pid, Eterm *hp_end; #endif - hsz = REF_THING_SIZE; + hsz = ERTS_REF_THING_SIZE; if (async) { refn = trefn; /* timer ref */ hsz += 4; /* 3-tuple */ @@ -1973,7 +1973,7 @@ access_sched_local_btm(Process *c_p, Eterm pid, refn[1], refn[2]); ref = make_internal_ref(hp); - hp += REF_THING_SIZE; + hp += ERTS_REF_THING_SIZE; msg = (async ? TUPLE3(hp, (cancel @@ -2087,7 +2087,7 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp, ErtsProcLocks proc_locks = ERTS_PROC_LOCK_MAIN; ErlOffHeap *ohp; - hsz = 4 + REF_THING_SIZE; + hsz = 4 + ERTS_REF_THING_SIZE; if (time_left > (Sint64) MAX_SMALL) hsz += ERTS_SINT64_HEAP_SIZE(time_left); @@ -2103,7 +2103,7 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp, trefn[1], trefn[2]); tref = make_internal_ref(hp); - hp += REF_THING_SIZE; + hp += ERTS_REF_THING_SIZE; if (time_left < 0) res = am_false; @@ -2189,7 +2189,7 @@ access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info) Eterm *hp, rref; Uint32 *rrefn; - hp = HAlloc(c_p, REF_THING_SIZE); + hp = HAlloc(c_p, ERTS_REF_THING_SIZE); rref = erts_sched_make_ref_in_buffer(esdp, hp); rrefn = internal_ref_numbers(rref); diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 0646665b07..b2c307e826 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -111,10 +111,12 @@ const int etp_lock_check = 1; #else const int etp_lock_check = 0; #endif -#ifdef WORDS_BIGENDIAN -const int etp_big_endian = 1; +const int etp_endianness = ERTS_ENDIANNESS; +const Eterm etp_ref_header = ERTS_REF_THING_HEADER; +#ifdef ERTS_MAGIC_REF_THING_HEADER +const Eterm etp_magic_ref_header = ERTS_MAGIC_REF_THING_HEADER; #else -const int etp_big_endian = 0; +const Eterm etp_magic_ref_header = ERTS_REF_THING_HEADER; #endif const Eterm etp_the_non_value = THE_NON_VALUE; #ifdef ERTS_HOLE_MARKER @@ -772,6 +774,8 @@ early_init(int *argc, char **argv) /* H_MAX_SIZE = H_DEFAULT_MAX_SIZE; H_MAX_FLAGS = MAX_HEAP_SIZE_KILL|MAX_HEAP_SIZE_LOG; + erts_term_init(); + erts_initialized = 0; erts_use_sender_punish = 1; @@ -1143,6 +1147,8 @@ early_init(int *argc, char **argv) /* no_schedulers_online = schdlrs_onln; erts_no_schedulers = (Uint) no_schedulers; +#else + erts_no_schedulers = 1; #endif #ifdef ERTS_DIRTY_SCHEDULERS erts_no_dirty_cpu_schedulers = no_dirty_cpu_schedulers = dirty_cpu_scheds; diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 08dcbed91c..c98581a45e 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -156,6 +156,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "tracer_mtx", NULL }, { "port_table", NULL }, #endif + { "magic_ref_table", "address" }, { "mtrace_op", NULL }, { "instr_x", NULL }, { "instr", NULL }, diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 5e160de080..f181c1e3cb 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -176,6 +176,11 @@ erts_cleanup_offheap(ErlOffHeap *offheap) erts_erase_fun_entry(u.fun->fe); } break; + case REF_SUBTAG: + ASSERT(is_magic_ref_thing(u.hdr)); + if (erts_refc_dectest(&u.mref->mb->refc, 0) == 0) + erts_bin_free((Binary *)u.mref->mb); + break; default: ASSERT(is_external_header(u.hdr->thing_word)); erts_deref_node_entry(u.ext->node); diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c index c207dea10f..bdfc7f30d9 100644 --- a/erts/emulator/beam/erl_monitors.c +++ b/erts/emulator/beam/erl_monitors.c @@ -45,6 +45,7 @@ #include "bif.h" #include "big.h" #include "erl_monitors.h" +#include "erl_bif_unique.h" #define STACK_NEED 50 #define MAX_MONITORS 0xFFFFFFFFUL @@ -79,7 +80,24 @@ static ERTS_INLINE int cmp_mon_ref(Eterm ref1, Eterm ref2) b2 = boxed_val(ref2); if (is_ref_thing_header(*b1)) { if (is_ref_thing_header(*b2)) { - return memcmp(b1+1,b2+1,ERTS_REF_WORDS*sizeof(Uint)); + Uint32 *num1, *num2; + if (is_ordinary_ref_thing(b1)) { + ErtsORefThing *rtp = (ErtsORefThing *) b1; + num1 = rtp->num; + } + else { + ErtsMRefThing *mrtp = (ErtsMRefThing *) b1; + num1 = mrtp->mb->refn; + } + if (is_ordinary_ref_thing(b2)) { + ErtsORefThing *rtp = (ErtsORefThing *) b2; + num2 = rtp->num; + } + else { + ErtsMRefThing *mrtp = (ErtsMRefThing *) b2; + num2 = mrtp->mb->refn; + } + return erts_internal_ref_number_cmp(num1, num2); } return -1; } @@ -97,7 +115,8 @@ do { \ Uint i__; \ Uint len__; \ ASSERT((Hp)); \ - ASSERT(is_internal_ref((From)) || is_external((From))); \ + ASSERT(is_internal_ordinary_ref((From)) \ + || is_external((From))); \ (To) = make_boxed((Hp)); \ len__ = thing_arityval(*boxed_val((From))) + 1; \ for(i__ = 0; i__ < len__; i__++) \ @@ -339,6 +358,8 @@ void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, Eterm pid, int state = 0; ErtsMonitor **this = root; Sint c; + + ASSERT(is_internal_ordinary_ref(ref) || is_external_ref(ref)); dstack[0] = DIR_END; for (;;) { diff --git a/erts/emulator/beam/erl_monitors.h b/erts/emulator/beam/erl_monitors.h index 9e2beedea3..bb493bf336 100644 --- a/erts/emulator/beam/erl_monitors.h +++ b/erts/emulator/beam/erl_monitors.h @@ -91,7 +91,7 @@ /* Size of a monitor without heap, in words (fixalloc) */ #define ERTS_MONITOR_SIZE ((sizeof(ErtsMonitor) - sizeof(Uint))/sizeof(Uint)) -#define ERTS_MONITOR_SH_SIZE (ERTS_MONITOR_SIZE + REF_THING_SIZE) +#define ERTS_MONITOR_SH_SIZE (ERTS_MONITOR_SIZE + ERTS_REF_THING_SIZE) #define ERTS_LINK_SIZE ((sizeof(ErtsLink) - sizeof(Uint))/sizeof(Uint)) #define ERTS_LINK_SH_SIZE ERTS_LINK_SIZE /* Size of fix-alloced links */ diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c index 66bb55e6c8..1c3160efaf 100644 --- a/erts/emulator/beam/erl_msacc.c +++ b/erts/emulator/beam/erl_msacc.c @@ -66,9 +66,9 @@ static Uint msacc_unmanaged_count = 0; #endif #if ERTS_MSACC_STATE_COUNT < MAP_SMALL_MAP_LIMIT -#define DEFAULT_MSACC_MSG_SIZE (3 + 1 + ERTS_MSACC_STATE_COUNT * 2 + 3 + REF_THING_SIZE) +#define DEFAULT_MSACC_MSG_SIZE (3 + 1 + ERTS_MSACC_STATE_COUNT * 2 + 3 + ERTS_REF_THING_SIZE) #else -#define DEFAULT_MSACC_MSG_SIZE (3 + ERTS_MSACC_STATE_COUNT * 3 + 3 + REF_THING_SIZE) +#define DEFAULT_MSACC_MSG_SIZE (3 + ERTS_MSACC_STATE_COUNT * 3 + 3 + ERTS_REF_THING_SIZE) #endif /* we have to split initiation as atoms are not inited in early init */ @@ -212,7 +212,7 @@ typedef struct { int action; Process *proc; Eterm ref; - Eterm ref_heap[REF_THING_SIZE]; + Eterm ref_heap[ERTS_REF_THING_SIZE]; Uint req_sched; erts_smp_atomic32_t refc; } ErtsMSAccReq; @@ -245,7 +245,7 @@ static void send_reply(ErtsMsAcc *msacc, ErtsMSAccReq *msaccrp) { if (msacc->unmanaged) erts_mtx_lock(&msacc->mtx); - hp = erts_produce_heap(&factory, REF_THING_SIZE + 3 /* tuple */, 0); + hp = erts_produce_heap(&factory, ERTS_REF_THING_SIZE + 3 /* tuple */, 0); ref_copy = STORE_NC(&hp, &msgp->hfrag.off_heap, msaccrp->ref); msg = erts_msacc_gather_stats(msacc, &factory); msg = TUPLE2(hp, ref_copy, msg); @@ -255,7 +255,7 @@ static void send_reply(ErtsMsAcc *msacc, ErtsMSAccReq *msaccrp) { erts_factory_close(&factory); } else { ErlOffHeap *ohp = NULL; - msgp = erts_alloc_message_heap(rp, &rp_locks, REF_THING_SIZE, &hp, &ohp); + msgp = erts_alloc_message_heap(rp, &rp_locks, ERTS_REF_THING_SIZE, &hp, &ohp); msg = STORE_NC(&hp, &msgp->hfrag.off_heap, msaccrp->ref); } diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 7c7a32f234..20b00bd4ba 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1721,7 +1721,7 @@ ERL_NIF_TERM enif_make_string_len(ErlNifEnv* env, const char* string, ERL_NIF_TERM enif_make_ref(ErlNifEnv* env) { - Eterm* hp = alloc_heap(env, REF_THING_SIZE); + Eterm* hp = alloc_heap(env, ERTS_REF_THING_SIZE); return erts_make_ref_in_buffer(hp); } @@ -1967,7 +1967,6 @@ typedef struct enif_resource_t byte align__[4]; # endif #endif - char data[1]; }ErlNifResource; @@ -2166,6 +2165,7 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t size) { Binary* bin = erts_create_magic_binary_x(SIZEOF_ErlNifResource(size), &nif_resource_dtor, + ERTS_ALC_T_BINARY, 1); /* unaligned */ ErlNifResource* resource = ERTS_MAGIC_BIN_UNALIGNED_DATA(bin); diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h index 0c76c6fe7d..e27f1d121a 100644 --- a/erts/emulator/beam/erl_node_container_utils.h +++ b/erts/emulator/beam/erl_node_container_utils.h @@ -255,19 +255,16 @@ extern ErtsPTab erts_port; * Refs * \* */ +#define internal_ref_no_numbers(x) ERTS_REF_NUMBERS +#define internal_ref_numbers(x) (is_internal_ordinary_ref((x)) \ + ? internal_ordinary_ref_numbers((x)) \ + : (ASSERT(is_internal_magic_ref((x))), \ + internal_magic_ref_numbers((x)))) #if defined(ARCH_64) -#define internal_ref_no_of_numbers(x) \ - (internal_ref_data((x))[0]) -#define internal_thing_ref_no_of_numbers(thing) \ - (internal_thing_ref_data(thing)[0]) -#define internal_ref_numbers(x) \ - (&internal_ref_data((x))[1]) -#define internal_thing_ref_numbers(thing) \ - (&internal_thing_ref_data(thing)[1]) -#define external_ref_no_of_numbers(x) \ +#define external_ref_no_numbers(x) \ (external_ref_data((x))[0]) -#define external_thing_ref_no_of_numbers(thing) \ +#define external_thing_ref_no_numbers(thing) \ (external_thing_ref_data(thing)[0]) #define external_ref_numbers(x) \ (&external_ref_data((x))[1]) @@ -277,12 +274,8 @@ extern ErtsPTab erts_port; #else -#define internal_ref_no_of_numbers(x) (internal_ref_data_words((x))) -#define internal_thing_ref_no_of_numbers(t) (internal_thing_ref_data_words(t)) -#define internal_ref_numbers(x) (internal_ref_data((x))) -#define internal_thing_ref_numbers(t) (internal_thing_ref_data(t)) -#define external_ref_no_of_numbers(x) (external_ref_data_words((x))) -#define external_thing_ref_no_of_numbers(t) (external_thing_ref_data_words((t))) +#define external_ref_no_numbers(x) (external_ref_data_words((x))) +#define external_thing_ref_no_numbers(t) (external_thing_ref_data_words((t))) #define external_ref_numbers(x) (external_ref_data((x))) #define external_thing_ref_numbers(t) (external_thing_ref_data((t))) @@ -299,15 +292,9 @@ extern ErtsPTab erts_port; #define internal_ref_channel_no(x) (internal_channel_no((x))) #define external_ref_channel_no(x) (external_channel_no((x))) -#define ref_data_words(x) (is_internal_ref((x)) \ - ? internal_ref_data_words((x)) \ - : external_ref_data_words((x))) -#define ref_data(x) (is_internal_ref((x)) \ - ? internal_ref_data((x)) \ - : external_ref_data((x))) -#define ref_no_of_numbers(x) (is_internal_ref((x)) \ - ? internal_ref_no_of_numbers((x))\ - : external_ref_no_of_numbers((x))) +#define ref_no_numbers(x) (is_internal_ref((x)) \ + ? internal_ref_no_numbers((x))\ + : external_ref_no_numbers((x))) #define ref_numbers(x) (is_internal_ref((x)) \ ? internal_ref_numbers((x)) \ : external_ref_numbers((x))) diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 467a05f950..060fbd5883 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -1126,8 +1126,8 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) for (u.hdr = oh->first; u.hdr; u.hdr = u.hdr->next) { switch (thing_subtag(u.hdr->thing_word)) { - case REFC_BINARY_SUBTAG: - if(IsMatchProgBinary(u.pb->val)) { + case REF_SUBTAG: + if(IsMatchProgBinary(u.mref->mb)) { InsertedBin *ib; int insert_bin = 1; for (ib = inserted_bins; ib; ib = ib->next) @@ -1152,6 +1152,7 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) } } break; + case REFC_BINARY_SUBTAG: case FUN_SUBTAG: break; /* No need to */ default: diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index b43a1b0190..6b64bbd2f1 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -382,12 +382,15 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) { break; } case REF_DEF: + if (!ERTS_IS_CRASH_DUMPING) + erts_magic_ref_save_bin(obj); + /* fall through... */ case EXTERNAL_REF_DEF: PRINT_STRING(res, fn, arg, "#Ref<"); PRINT_UWORD(res, fn, arg, 'u', 0, 1, (ErlPfUWord) ref_channel_no(wobj)); ref_num = ref_numbers(wobj); - for (i = ref_no_of_numbers(wobj)-1; i >= 0; i--) { + for (i = ref_no_numbers(wobj)-1; i >= 0; i--) { PRINT_CHAR(res, fn, arg, '.'); PRINT_UWORD(res, fn, arg, 'u', 0, 1, (ErlPfUWord) ref_num[i]); } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 3691e31922..87d71fdd22 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1173,7 +1173,7 @@ typedef struct { int enable; Process *proc; Eterm ref; - Eterm ref_heap[REF_THING_SIZE]; + Eterm ref_heap[ERTS_REF_THING_SIZE]; Uint req_sched; erts_smp_atomic32_t refc; #ifdef ERTS_DIRTY_SCHEDULERS @@ -1185,7 +1185,7 @@ typedef struct { typedef struct { Process *proc; Eterm ref; - Eterm ref_heap[REF_THING_SIZE]; + Eterm ref_heap[ERTS_REF_THING_SIZE]; Uint req_sched; erts_smp_atomic32_t refc; } ErtsSystemCheckReq; @@ -1297,7 +1297,7 @@ reply_sched_wall_time(void *vswtrp) if (hpp) ref_copy = STORE_NC(hpp, ohp, swtrp->ref); else - *szp += REF_THING_SIZE; + *szp += ERTS_REF_THING_SIZE; ASSERT(!swtrp->set); @@ -1342,7 +1342,7 @@ reply_sched_wall_time(void *vswtrp) if (hpp) ref_copy = STORE_NC(hpp, ohp, swtrp->ref); else - *szp += REF_THING_SIZE; + *szp += ERTS_REF_THING_SIZE; if (swtrp->set) msg = ref_copy; @@ -1444,7 +1444,7 @@ reply_system_check(void *vscrp) ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); #endif - sz = REF_THING_SIZE; + sz = ERTS_REF_THING_SIZE; mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); hpp = &hp; msg = STORE_NC(hpp, ohp, scrp->ref); @@ -6381,7 +6381,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online erts_tsd_set(sched_data_key, (void *) esdp); #endif } - erts_no_schedulers = 1; erts_no_dirty_cpu_schedulers = 0; erts_no_dirty_io_schedulers = 0; #endif diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c index 7d857ad326..6b25728af7 100644 --- a/erts/emulator/beam/erl_term.c +++ b/erts/emulator/beam/erl_term.c @@ -59,6 +59,42 @@ erts_set_literal_tag(Eterm *term, Eterm *hp_start, Eterm hsz) #endif } +void +erts_term_init(void) +{ +#ifdef ERTS_ORDINARY_REF_MARKER + /* Ordinary and magic references of same size... */ + + ErtsRefThing ref_thing; + + ERTS_CT_ASSERT(ERTS_ORDINARY_REF_MARKER == ~((Uint32)0)); + ref_thing.m.header = ERTS_REF_THING_HEADER; + ref_thing.m.mb = (ErtsMagicBinary *) ~((UWord) 3); + ref_thing.m.next = (struct erl_off_heap_header *) ~((UWord) 3); + if (ref_thing.o.marker == ERTS_ORDINARY_REF_MARKER) + ERTS_INTERNAL_ERROR("Cannot differentiate between magic and ordinary references"); + + ERTS_CT_ASSERT(offsetof(ErtsORefThing,marker) != 0); + ERTS_CT_ASSERT(sizeof(ErtsORefThing) == sizeof(ErtsMRefThing)); +# ifdef ERTS_MAGIC_REF_THING_HEADER +# error Magic ref thing header should not have been defined... +# endif + +#else + /* Ordinary and magic references of different sizes... */ + +# ifndef ERTS_MAGIC_REF_THING_HEADER +# error Magic ref thing header should have been defined... +# endif + ERTS_CT_ASSERT(sizeof(ErtsORefThing) != sizeof(ErtsMRefThing)); + +#endif + + ERTS_CT_ASSERT(ERTS_REF_THING_SIZE*sizeof(Eterm) == sizeof(ErtsORefThing)); + ERTS_CT_ASSERT(ERTS_MAGIC_REF_THING_SIZE*sizeof(Eterm) == sizeof(ErtsMRefThing)); + +} + /* * XXX: define NUMBER_CODE() here when new representation is used */ @@ -95,8 +131,8 @@ ET_DEFINE_CHECKED(Eterm*,tuple_val,Wterm,is_tuple); ET_DEFINE_CHECKED(struct erl_node_*,internal_pid_node,Eterm,is_internal_pid); ET_DEFINE_CHECKED(struct erl_node_*,internal_port_node,Eterm,is_internal_port); ET_DEFINE_CHECKED(Eterm*,internal_ref_val,Wterm,is_internal_ref); -ET_DEFINE_CHECKED(Uint,internal_ref_data_words,Wterm,is_internal_ref); -ET_DEFINE_CHECKED(Uint32*,internal_ref_data,Wterm,is_internal_ref); +ET_DEFINE_CHECKED(Uint32*,internal_magic_ref_numbers,Wterm,is_internal_magic_ref); +ET_DEFINE_CHECKED(Uint32*,internal_ordinary_ref_numbers,Wterm,is_internal_ordinary_ref); ET_DEFINE_CHECKED(struct erl_node_*,internal_ref_node,Eterm,is_internal_ref); ET_DEFINE_CHECKED(Eterm*,external_val,Wterm,is_external); ET_DEFINE_CHECKED(Uint,external_data_words,Wterm,is_external); diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index c3234ee349..a602a8f7c6 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -23,6 +23,8 @@ #include "erl_mmap.h" +void erts_term_init(void); + typedef UWord Wterm; /* Full word terms */ struct erl_node_; /* Declared in erl_node_tables.h */ @@ -718,73 +720,224 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_port_node,Eterm) #define ERTS_MAX_REF_NUMBERS 3 #define ERTS_REF_NUMBERS ERTS_MAX_REF_NUMBERS -#if defined(ARCH_64) -# define ERTS_REF_WORDS (ERTS_REF_NUMBERS/2 + 1) -# define ERTS_REF_32BIT_WORDS (ERTS_REF_NUMBERS+1) -#else -# define ERTS_REF_WORDS ERTS_REF_NUMBERS -# define ERTS_REF_32BIT_WORDS ERTS_REF_NUMBERS +#ifndef ERTS_ENDIANNESS +# error ERTS_ENDIANNESS not defined... #endif -typedef struct { - Eterm header; - union { - Uint32 ui32[ERTS_REF_32BIT_WORDS]; - Uint ui[ERTS_REF_WORDS]; - } data; -} RefThing; - -#define REF_THING_SIZE (sizeof(RefThing)/sizeof(Uint)) -#define REF_THING_HEAD_SIZE (sizeof(Eterm)/sizeof(Uint)) +#if ERTS_REF_NUMBERS != 3 +# error "A new reference layout for 64-bit needs to be implemented..." +#endif -#define make_ref_thing_header(DW) \ - _make_header((DW)+REF_THING_HEAD_SIZE-1,_TAG_HEADER_REF) +struct magic_binary; #if defined(ARCH_64) +# define ERTS_ORDINARY_REF_MARKER (~((Uint32) 0)) + +typedef struct { + Eterm header; +#if ERTS_ENDIANNESS <= 0 + Uint32 marker; +#endif + Uint32 num[ERTS_REF_NUMBERS]; +#if ERTS_ENDIANNESS > 0 + Uint32 marker; +#endif +} ErtsORefThing; + +typedef struct { + Eterm header; + struct magic_binary *mb; + struct erl_off_heap_header* next; +#if !ERTS_ENDIANNESS + Uint32 num[ERTS_REF_NUMBERS]; + Uint32 marker; +#endif +} ErtsMRefThing; + /* - * Ref layout on a 64-bit little endian machine: + * Ordinary ref layout on a 64-bit little endian machine: * * 63 31 0 * +--------------+--------------+ * | Thing word | * +--------------+--------------+ - * | Data 0 | 32-bit arity | + * | Data 0 | 0xffffffff | * +--------------+--------------+ * | Data 2 | Data 1 | * +--------------+--------------+ * - * Data is stored as an Uint32 array with 32-bit arity as first number. + * Ordinary ref layout on a 64-bit big endian machine: + * + * 63 31 0 + * +--------------+--------------+ + * | Thing word | + * +--------------+--------------+ + * | Data 0 | Data 1 | + * +--------------+--------------+ + * | Data 2 | 0xffffffff | + * +--------------+--------------+ + * + * Magic Ref layout on a 64-bit machine: + * + * 63 31 0 + * +--------------+--------------+ + * | Thing word | + * +--------------+--------------+ + * | Magic Binary Pointer | + * +--------------+--------------+ + * | Next Off Heap Pointer | + * +--------------+--------------+ + * + * Both pointers in the magic ref are 64-bit aligned. That is, + * least significant bits are zero. The marker 32-bit word is + * placed over the least significant bits of one of the pointers. + * That is, we can distinguish between magic and ordinary ref + * by looking at the marker field. + * */ #define write_ref_thing(Hp, R0, R1, R2) \ do { \ - ((RefThing *) (Hp))->header = make_ref_thing_header(ERTS_REF_WORDS); \ - ((RefThing *) (Hp))->data.ui32[0] = ERTS_REF_NUMBERS; \ - ((RefThing *) (Hp))->data.ui32[1] = (R0); \ - ((RefThing *) (Hp))->data.ui32[2] = (R1); \ - ((RefThing *) (Hp))->data.ui32[3] = (R2); \ + ((ErtsORefThing *) (Hp))->header = ERTS_REF_THING_HEADER; \ + ((ErtsORefThing *) (Hp))->marker = ERTS_ORDINARY_REF_MARKER; \ + ((ErtsORefThing *) (Hp))->num[0] = (R0); \ + ((ErtsORefThing *) (Hp))->num[1] = (R1); \ + ((ErtsORefThing *) (Hp))->num[2] = (R2); \ } while (0) -#else +#if ERTS_ENDIANNESS +/* Known big or little endian */ + +#define write_magic_ref_thing(Hp, Ohp, Binp) \ +do { \ + ((ErtsMRefThing *) (Hp))->header = ERTS_REF_THING_HEADER; \ + ((ErtsMRefThing *) (Hp))->mb = (Binp); \ + ((ErtsMRefThing *) (Hp))->next = (Ohp)->first; \ + (Ohp)->first = (struct erl_off_heap_header*) (Hp); \ + ASSERT(erts_is_ref_numbers_magic((Binp)->refn)); \ +} while (0) + +#else /* !ERTS_ENDIANNESS */ + +#define write_magic_ref_thing(Hp, Ohp, Binp) \ +do { \ + ((ErtsMRefThing *) (Hp))->header = ERTS_MAGIC_REF_THING_HEADER; \ + ((ErtsMRefThing *) (Hp))->mb = (Binp); \ + ((ErtsMRefThing *) (Hp))->next = (Ohp)->first; \ + (Ohp)->first = (struct erl_off_heap_header*) (Hp); \ + ((ErtsMRefThing *) (Hp))->marker = 0; \ + ((ErtsMRefThing *) (Hp))->num[0] = (Binp)->refn[0]; \ + ((ErtsMRefThing *) (Hp))->num[1] = (Binp)->refn[1]; \ + ((ErtsMRefThing *) (Hp))->num[2] = (Binp)->refn[2]; \ + ASSERT(erts_is_ref_numbers_magic((Binp)->refn)); \ +} while (0) + +#endif /* !ERTS_ENDIANNESS */ + +#else /* ARCH_32 */ + +typedef struct { + Eterm header; + Uint32 num[ERTS_REF_NUMBERS]; +} ErtsORefThing; + +typedef struct { + Eterm header; + struct magic_binary *mb; + struct erl_off_heap_header* next; +} ErtsMRefThing; + #define write_ref_thing(Hp, R0, R1, R2) \ do { \ - ((RefThing *) (Hp))->header = make_ref_thing_header(ERTS_REF_WORDS); \ - ((RefThing *) (Hp))->data.ui32[0] = (R0); \ - ((RefThing *) (Hp))->data.ui32[1] = (R1); \ - ((RefThing *) (Hp))->data.ui32[2] = (R2); \ + ((ErtsORefThing *) (Hp))->header = ERTS_REF_THING_HEADER; \ + ((ErtsORefThing *) (Hp))->num[0] = (R0); \ + ((ErtsORefThing *) (Hp))->num[1] = (R1); \ + ((ErtsORefThing *) (Hp))->num[2] = (R2); \ } while (0) +#define write_magic_ref_thing(Hp, Ohp, Binp) \ +do { \ + ((ErtsMRefThing *) (Hp))->header = ERTS_MAGIC_REF_THING_HEADER; \ + ((ErtsMRefThing *) (Hp))->mb = (Binp); \ + ((ErtsMRefThing *) (Hp))->next = (Ohp)->first; \ + (Ohp)->first = (struct erl_off_heap_header*) (Hp); \ + ASSERT(erts_is_ref_numbers_magic(&(Binp)->refn)); \ +} while (0) + +#endif /* ARCH_32 */ + +typedef union { + ErtsMRefThing m; + ErtsORefThing o; +} ErtsRefThing; + +#define ERTS_REF_THING_SIZE (sizeof(ErtsORefThing)/sizeof(Uint)) +#define ERTS_MAGIC_REF_THING_SIZE (sizeof(ErtsMRefThing)/sizeof(Uint)) +#define ERTS_MAX_INTERNAL_REF_SIZE (sizeof(ErtsRefThing)/sizeof(Uint)) + +#define make_ref_thing_header(Words) \ + _make_header((Words)-1,_TAG_HEADER_REF) + +#define ERTS_REF_THING_HEADER _make_header(ERTS_REF_THING_SIZE-1,_TAG_HEADER_REF) + +#if defined(ARCH_64) && ERTS_ENDIANNESS /* All internal refs of same size... */ + +# undef ERTS_MAGIC_REF_THING_HEADER + +# define is_ref_thing_header(x) ((x) == ERTS_REF_THING_HEADER) + +#define is_ordinary_ref_thing(x) \ + (ASSERT(is_ref_thing_header(*((Eterm *)(x)))), \ + ((ErtsRefThing *) (x))->o.marker == ERTS_ORDINARY_REF_MARKER) + +#define is_magic_ref_thing(x) \ + (!is_ordinary_ref_thing((x))) + +#define is_internal_magic_ref(x) \ + ((_unchecked_is_boxed((x)) && *boxed_val((x)) == ERTS_REF_THING_HEADER) \ + && is_magic_ref_thing(boxed_val((x)))) + +#define is_internal_ordinary_ref(x) \ + ((_unchecked_is_boxed((x)) && *boxed_val((x)) == ERTS_REF_THING_HEADER) \ + && is_ordinary_ref_thing(boxed_val((x)))) + +#else /* Ordinary and magic references of different sizes... */ + +# define ERTS_MAGIC_REF_THING_HEADER \ + _make_header(ERTS_MAGIC_REF_THING_SIZE-1,_TAG_HEADER_REF) + +# define is_ref_thing_header(x) \ + (((x) & _TAG_HEADER_MASK) == _TAG_HEADER_REF) + +#define is_ordinary_ref_thing(x) \ + (ASSERT(is_ref_thing_header(*((Eterm *)(x)))), \ + *((Eterm *)(x)) == ERTS_REF_THING_HEADER) + +#define is_magic_ref_thing(x) \ + (ASSERT(is_ref_thing_header(*((Eterm *)(x)))), \ + *((Eterm *)(x)) == ERTS_MAGIC_REF_THING_HEADER) + +#define is_internal_magic_ref(x) \ + (_unchecked_is_boxed((x)) && *boxed_val((x)) == ERTS_MAGIC_REF_THING_HEADER) + +#define is_internal_ordinary_ref(x) \ + (_unchecked_is_boxed((x)) && *boxed_val((x)) == ERTS_REF_THING_HEADER) + #endif -#define is_ref_thing_header(x) (((x) & _TAG_HEADER_MASK) == _TAG_HEADER_REF) #define make_internal_ref(x) make_boxed((Eterm*)(x)) -#define _unchecked_ref_thing_ptr(x) \ - ((RefThing*) _unchecked_internal_ref_val(x)) -#define ref_thing_ptr(x) \ - ((RefThing*) internal_ref_val(x)) +#define _unchecked_ordinary_ref_thing_ptr(x) \ + ((ErtsORefThing*) _unchecked_internal_ref_val(x)) +#define ordinary_ref_thing_ptr(x) \ + ((ErtsORefThing*) internal_ref_val(x)) + +#define _unchecked_magic_ref_thing_ptr(x) \ + ((ErtsMRefThing*) _unchecked_internal_ref_val(x)) +#define magic_ref_thing_ptr(x) \ + ((ErtsMRefThing*) internal_ref_val(x)) #define is_internal_ref(x) \ (_unchecked_is_boxed((x)) && is_ref_thing_header(*boxed_val((x)))) @@ -796,16 +949,21 @@ do { \ _ET_DECLARE_CHECKED(Eterm*,internal_ref_val,Wterm) #define internal_ref_val(x) _ET_APPLY(internal_ref_val,(x)) -#define internal_thing_ref_data_words(t) (thing_arityval(*(Eterm*)(t))) -#define _unchecked_internal_ref_data_words(x) \ - (_unchecked_thing_arityval(*_unchecked_internal_ref_val(x))) -_ET_DECLARE_CHECKED(Uint,internal_ref_data_words,Wterm) -#define internal_ref_data_words(x) _ET_APPLY(internal_ref_data_words,(x)) +#define internal_ordinary_thing_ref_numbers(ort) (((ErtsORefThing *)(ort))->num) +#define _unchecked_internal_ordinary_ref_numbers(x) (internal_ordinary_thing_ref_numbers(_unchecked_ordinary_ref_thing_ptr(x))) +_ET_DECLARE_CHECKED(Uint32*,internal_ordinary_ref_numbers,Wterm) +#define internal_ordinary_ref_numbers(x) _ET_APPLY(internal_ordinary_ref_numbers,(x)) + +#if defined(ARCH_64) && !ERTS_ENDIANNESS +#define internal_magic_thing_ref_numbers(mrt) (((ErtsMRefThing *)(mrt))->num) +#else +#define internal_magic_thing_ref_numbers(mrt) (((ErtsMRefThing *)(mrt))->mb->refn) +#endif + +#define _unchecked_internal_magic_ref_numbers(x) (internal_magic_thing_ref_numbers(_unchecked_magic_ref_thing_ptr(x))) +_ET_DECLARE_CHECKED(Uint32*,internal_magic_ref_numbers,Wterm) +#define internal_magic_ref_numbers(x) _ET_APPLY(internal_magic_ref_numbers,(x)) -#define internal_thing_ref_data(thing) ((thing)->data.ui32) -#define _unchecked_internal_ref_data(x) (internal_thing_ref_data(_unchecked_ref_thing_ptr(x))) -_ET_DECLARE_CHECKED(Uint32*,internal_ref_data,Wterm) -#define internal_ref_data(x) _ET_APPLY(internal_ref_data,(x)) #define _unchecked_internal_ref_node(x) erts_this_node _ET_DECLARE_CHECKED(struct erl_node_*,internal_ref_node,Eterm) diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index 6aa2a7500f..1d747e5fab 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -1831,7 +1831,10 @@ erts_demonitor_time_offset(Eterm ref) ErtsMonitor *mon; ASSERT(is_internal_ref(ref)); erts_smp_mtx_lock(&erts_get_time_mtx); - mon = erts_remove_monitor(&time_offset_monitors, ref); + if (is_internal_ordinary_ref(ref)) + mon = erts_remove_monitor(&time_offset_monitors, ref); + else + mon = NULL; if (!mon) res = 0; else { @@ -1848,7 +1851,7 @@ erts_demonitor_time_offset(Eterm ref) typedef struct { Eterm pid; Eterm ref; - Eterm heap[REF_THING_SIZE]; + Eterm heap[ERTS_REF_THING_SIZE]; } ErtsTimeOffsetMonitorInfo; typedef struct { @@ -1869,11 +1872,11 @@ save_time_offset_monitor(ErtsMonitor *mon, void *vcntxt) cntxt->to_mon_info[mix].pid = mon->pid; to_hp = &cntxt->to_mon_info[mix].heap[0]; - ASSERT(is_internal_ref(mon->ref)); + ASSERT(is_internal_ordinary_ref(mon->ref)); from_hp = internal_ref_val(mon->ref); - ASSERT(thing_arityval(*from_hp) + 1 == REF_THING_SIZE); + ASSERT(thing_arityval(*from_hp) + 1 == ERTS_REF_THING_SIZE); - for (hix = 0; hix < REF_THING_SIZE; hix++) + for (hix = 0; hix < ERTS_REF_THING_SIZE; hix++) to_hp[hix] = from_hp[hix]; cntxt->to_mon_info[mix].ref @@ -1933,7 +1936,7 @@ send_time_offset_changed_notifications(void *new_offsetp) hp = (Eterm *) (tmp + no_monitors*sizeof(ErtsTimeOffsetMonitorInfo)); hsz = 6; /* 5-tuple */ - hsz += REF_THING_SIZE; + hsz += ERTS_REF_THING_SIZE; hsz += ERTS_SINT64_HEAP_SIZE(new_offset); if (IS_SSMALL(new_offset)) diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 6bcddb9e69..64b17a3f57 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2579,7 +2579,9 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, ASSERT(dflags & DFLAG_EXTENDED_REFERENCES); - i = ref_no_of_numbers(obj); + erts_magic_ref_save_bin(obj); + + i = ref_no_numbers(obj); put_int16(i, ep); ep += 2; ep = enc_atom(acmp, sysname, ep, dflags); @@ -3435,26 +3437,35 @@ dec_term_atom_common: r0 = get_int32(ep); /* allow full word */ ep += 4; - ref_ext_common: + ref_ext_common: { + ErtsORefThing *rtp; + if (ref_words > ERTS_MAX_REF_NUMBERS) goto error; node = dec_get_node(sysname, cre); if(node == erts_this_node) { - RefThing *rtp = (RefThing *) hp; - ref_num = (Uint32 *) (hp + REF_THING_HEAD_SIZE); + + rtp = (ErtsORefThing *) hp; + ref_num = &rtp->num[0]; + if (ref_words != ERTS_REF_NUMBERS) { + int i; + if (ref_words > ERTS_REF_NUMBERS) + goto error; /* Not a ref that we created... */ + for (i = ref_words; i < ERTS_REF_NUMBERS; i++) + ref_num[i] = 0; + } -#if defined(ARCH_64) - hp += REF_THING_HEAD_SIZE + ref_words/2 + 1; - rtp->header = make_ref_thing_header(ref_words/2 + 1); -#else - hp += REF_THING_HEAD_SIZE + ref_words; - rtp->header = make_ref_thing_header(ref_words); +#ifdef ERTS_ORDINARY_REF_MARKER + rtp->marker = ERTS_ORDINARY_REF_MARKER; #endif + hp += ERTS_REF_THING_SIZE; + rtp->header = ERTS_REF_THING_HEADER; *objp = make_internal_ref(rtp); } else { ExternalThing *etp = (ExternalThing *) hp; + rtp = NULL; #if defined(ARCH_64) hp += EXTERNAL_THING_HEAD_SIZE + ref_words/2 + 1; #else @@ -3472,12 +3483,13 @@ dec_term_atom_common: factory->off_heap->first = (struct erl_off_heap_header*)etp; *objp = make_external_ref(etp); ref_num = &(etp->data.ui32[0]); - } - #if defined(ARCH_64) - *(ref_num++) = ref_words /* 32-bit arity */; + *(ref_num++) = ref_words /* 32-bit arity */; #endif + } + ref_num[0] = r0; + for(i = 1; i < ref_words; i++) { ref_num[i] = get_int32(ep); ep += 4; @@ -3486,8 +3498,24 @@ dec_term_atom_common: if ((1 + ref_words) % 2) ref_num[ref_words] = 0; #endif + if (node == erts_this_node) { + /* Check if it was a magic reference... */ + ErtsMagicBinary *mb = erts_magic_ref_lookup_bin(ref_num); + if (mb) { + /* + * Was a magic ref; adjust it... + * + * Refc on binary was increased by lookup above... + */ + ASSERT(rtp); + hp = (Eterm *) rtp; + write_magic_ref_thing(hp, factory->off_heap, mb); + hp += ERTS_MAGIC_REF_THING_SIZE; + } + } break; } + } case BINARY_EXT: { n = get_int32(ep); @@ -4109,7 +4137,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, /*fall through*/ case REF_DEF: ASSERT(dflags & DFLAG_EXTENDED_REFERENCES); - i = ref_no_of_numbers(obj); + i = ref_no_numbers(obj); result += (1 + 2 + encode_size_struct2(acmp, ref_node_name(obj), dflags) + 1 + 4*i); break; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 713fb81c77..187a948a7b 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -42,6 +42,9 @@ #include "erl_utils.h" #include "erl_port.h" #include "erl_gc.h" +#define ERTS_BINARY_TYPES_ONLY__ +#include "erl_binary.h" +#undef ERTS_BINARY_TYPES_ONLY__ struct enif_func_t; @@ -126,7 +129,7 @@ typedef struct de_proc_entry { PROC_AWAIT_LOAD == Wants to be notified when we reloaded the driver (old was locked) */ Uint flags; /* ERL_FL_DE_DEREFERENCED when reload in progress */ - Eterm heap[REF_THING_SIZE]; /* "ref heap" */ + Eterm heap[ERTS_REF_THING_SIZE]; /* "ref heap" */ struct de_proc_entry *next; } DE_ProcEntry; @@ -213,118 +216,6 @@ extern Eterm erts_ddll_monitor_driver(Process *p, Eterm description, ErtsProcLocks plocks); -/* -** Just like the driver binary but with initial flags -** Note that the two structures Binary and ErlDrvBinary HAVE to -** be equal except for extra fields in the beginning of the struct. -** ErlDrvBinary is defined in erl_driver.h. -** When driver_alloc_binary is called, a Binary is allocated, but -** the pointer returned is to the address of the first element that -** also occurs in the ErlDrvBinary struct (driver.*binary takes care if this). -** The driver need never know about additions to the internal Binary of the -** emulator. One should however NEVER be sloppy when mixing ErlDrvBinary -** and Binary, the macros below can convert one type to the other, as they both -** in reality are equal. -*/ - -#ifdef ARCH_32 - /* *DO NOT USE* only for alignment. */ -#define ERTS_BINARY_STRUCT_ALIGNMENT Uint32 align__; -#else -#define ERTS_BINARY_STRUCT_ALIGNMENT -#endif - -/* Add fields in ERTS_INTERNAL_BINARY_FIELDS, otherwise the drivers crash */ -#define ERTS_INTERNAL_BINARY_FIELDS \ - UWord flags; \ - erts_refc_t refc; \ - ERTS_BINARY_STRUCT_ALIGNMENT - -typedef struct binary { - ERTS_INTERNAL_BINARY_FIELDS - SWord orig_size; - char orig_bytes[1]; /* to be continued */ -} Binary; - -#define ERTS_SIZEOF_Binary(Sz) \ - (offsetof(Binary,orig_bytes) + (Sz)) - -typedef struct { - ERTS_INTERNAL_BINARY_FIELDS - SWord orig_size; - void (*destructor)(Binary *); - union { - struct { - ERTS_BINARY_STRUCT_ALIGNMENT - char data[1]; - } aligned; - struct { - char data[1]; - } unaligned; - } u; -} ErtsMagicBinary; - -#ifdef ARCH_32 -#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 4 -#else -#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 0 -#endif - -typedef union { - Binary binary; - ErtsMagicBinary magic_binary; - struct { - ERTS_INTERNAL_BINARY_FIELDS - ErlDrvBinary binary; - } driver; -} ErtsBinary; - -/* - * 'Binary' alignment: - * Address of orig_bytes[0] of a Binary should always be 8-byte aligned. - * It is assumed that the flags, refc, and orig_size fields are 4 bytes on - * 32-bits architectures and 8 bytes on 64-bits architectures. - */ - -#define ERTS_MAGIC_BIN_DESTRUCTOR(BP) \ - ((ErtsBinary *) (BP))->magic_binary.destructor -#define ERTS_MAGIC_BIN_DATA(BP) \ - ((void *) ((ErtsBinary *) (BP))->magic_binary.u.aligned.data) -#define ERTS_MAGIC_DATA_OFFSET \ - (offsetof(ErtsMagicBinary,u.aligned.data) - offsetof(Binary,orig_bytes)) -#define ERTS_MAGIC_BIN_ORIG_SIZE(Sz) \ - (ERTS_MAGIC_DATA_OFFSET + (Sz)) -#define ERTS_MAGIC_BIN_SIZE(Sz) \ - (offsetof(ErtsMagicBinary,u.aligned.data) + (Sz)) - -/* On 32-bit arch these macro variants will save memory - by not forcing 8-byte alignment for the magic payload. -*/ -#define ERTS_MAGIC_BIN_UNALIGNED_DATA(BP) \ - ((void *) ((ErtsBinary *) (BP))->magic_binary.u.unaligned.data) -#define ERTS_MAGIC_UNALIGNED_DATA_OFFSET \ - (offsetof(ErtsMagicBinary,u.unaligned.data) - offsetof(Binary,orig_bytes)) -#define ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(BP) \ - ((BP)->orig_size - ERTS_MAGIC_UNALIGNED_DATA_OFFSET) -#define ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(Sz) \ - (ERTS_MAGIC_UNALIGNED_DATA_OFFSET + (Sz)) -#define ERTS_MAGIC_BIN_UNALIGNED_SIZE(Sz) \ - (offsetof(ErtsMagicBinary,u.unaligned.data) + (Sz)) -#define ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(DATA) \ - ((ErtsBinary*)((char*)(DATA) - offsetof(ErtsMagicBinary,u.unaligned.data))) - - -#define Binary2ErlDrvBinary(B) (&((ErtsBinary *) (B))->driver.binary) -#define ErlDrvBinary2Binary(D) ((Binary *) \ - (((char *) (D)) \ - - offsetof(ErtsBinary, driver.binary))) - -/* A "magic" binary flag */ -#define BIN_FLAG_MAGIC 1 -#define BIN_FLAG_USR1 2 /* Reserved for use by different modules too mark */ -#define BIN_FLAG_USR2 4 /* certain binaries as special (used by ets) */ -#define BIN_FLAG_DRV 8 - /* * This structure represents one type of a binary in a process. */ @@ -386,6 +277,7 @@ union erl_off_heap_ptr { ProcBin *pb; struct erl_fun_thing* fun; struct external_thing_* ext; + ErtsMRefThing *mref; Eterm* ep; void* voidp; }; @@ -978,21 +870,6 @@ void erts_bif_info_init(void); /* bif.c */ -ERTS_GLB_INLINE Eterm -erts_proc_store_ref(Process *c_p, Uint32 ref[ERTS_MAX_REF_NUMBERS]); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -ERTS_GLB_INLINE Eterm -erts_proc_store_ref(Process *c_p, Uint32 ref[ERTS_MAX_REF_NUMBERS]) -{ - Eterm *hp = HAlloc(c_p, REF_THING_SIZE); - write_ref_thing(hp, ref[0], ref[1], ref[2]); - return make_internal_ref(hp); -} - -#endif - void erts_queue_monitor_message(Process *, ErtsProcLocks*, Eterm, diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 5446ebaab0..dad24d7ef6 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -1445,7 +1445,7 @@ finalize_force_imm_drv_call(ErtsTryImmDrvCallState *sp) erts_unblock_fpe(sp->fpe_was_unmasked); } -#define ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE (REF_THING_SIZE + 3) +#define ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE (ERTS_REF_THING_SIZE + 3) static ERTS_INLINE void queue_port_sched_op_reply(Process *rp, @@ -1460,7 +1460,7 @@ queue_port_sched_op_reply(Process *rp, ref= make_internal_ref(hp); write_ref_thing(hp, ref_num[0], ref_num[1], ref_num[2]); - hp += REF_THING_SIZE; + hp += ERTS_REF_THING_SIZE; msg = TUPLE2(hp, ref, msg); @@ -3099,7 +3099,7 @@ port_monitor(Port *prt, erts_aint32_t state, Eterm origin, ASSERT(is_pid(origin)); ASSERT(is_atom(name) || is_port(name) || name == NIL); - ASSERT(is_internal_ref(ref)); + ASSERT(is_internal_ordinary_ref(ref)); if (!(state & ERTS_PORT_SFLGS_INVALID_LOOKUP)) { ErtsProcLocks p_locks = ERTS_PROC_LOCK_LINK; @@ -3124,7 +3124,7 @@ static int port_sig_monitor(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp) { - Eterm hp[REF_THING_SIZE]; + Eterm hp[ERTS_REF_THING_SIZE]; Eterm ref = make_internal_ref(&hp); write_ref_thing(hp, sigdp->ref[0], sigdp->ref[1], sigdp->ref[2]); @@ -3245,7 +3245,7 @@ static int port_sig_demonitor(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp) { - Eterm hp[REF_THING_SIZE]; + Eterm hp[ERTS_REF_THING_SIZE]; Eterm ref = make_internal_ref(&hp); write_ref_thing(hp, sigdp->u.demonitor.ref[0], sigdp->u.demonitor.ref[1], @@ -3302,10 +3302,10 @@ ErtsPortOpResult erts_port_demonitor(Process *origin, ErtsDemonitorMode mode, sigdp->u.demonitor.origin = origin->common.id; sigdp->u.demonitor.name = target->common.id; { - RefThing *reft = ref_thing_ptr(ref); + Uint32 *nums = internal_ref_numbers(ref); /* Start from 1 skip ref arity */ sys_memcpy(sigdp->u.demonitor.ref, - internal_thing_ref_numbers(reft), + nums, sizeof(sigdp->u.demonitor.ref)); } @@ -5413,7 +5413,7 @@ reply_io_bytes(void *vreq) rp_locks = ERTS_PROC_LOCK_MAIN; } - hsz = 5 /* 4-tuple */ + REF_THING_SIZE; + hsz = 5 /* 4-tuple */ + ERTS_REF_THING_SIZE; erts_bld_uint64(NULL, &hsz, in); erts_bld_uint64(NULL, &hsz, out); @@ -5422,7 +5422,7 @@ reply_io_bytes(void *vreq) ref = make_internal_ref(hp); write_ref_thing(hp, req->refn[0], req->refn[1], req->refn[2]); - hp += REF_THING_SIZE; + hp += ERTS_REF_THING_SIZE; ein = erts_bld_uint64(&hp, NULL, in); eout = erts_bld_uint64(&hp, NULL, out); @@ -5451,7 +5451,7 @@ erts_request_io_bytes(Process *c_p) ErtsIOBytesReq *req = erts_alloc(ERTS_ALC_T_IOB_REQ, sizeof(ErtsIOBytesReq)); - hp = HAlloc(c_p, REF_THING_SIZE); + hp = HAlloc(c_p, ERTS_REF_THING_SIZE); ref = erts_sched_make_ref_in_buffer(esdp, hp); refn = internal_ref_numbers(ref); @@ -7609,20 +7609,16 @@ erl_drv_convert_time_unit(ErlDrvTime val, static void ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon) { - RefThing *refp; - ASSERT(is_internal_ref(ref)); - ERTS_CT_ASSERT(sizeof(RefThing) <= sizeof(ErlDrvMonitor)); - refp = ref_thing_ptr(ref); - memset(mon,0,sizeof(ErlDrvMonitor)); - memcpy(mon,refp,sizeof(RefThing)); + ERTS_CT_ASSERT(sizeof(ErtsOIRefStorage) <= sizeof(ErlDrvMonitor)); + erts_oiref_storage_save((ErtsOIRefStorage *) mon, ref); } static int do_driver_monitor_process(Port *prt, - Eterm *buf, ErlDrvTermData process, ErlDrvMonitor *monitor) { + Eterm buf[ERTS_REF_THING_SIZE]; Process *rp; Eterm ref; @@ -7665,26 +7661,22 @@ int driver_monitor_process(ErlDrvPort drvport, /* Now (in SMP) we should have either the port lock (if we have a scheduler) or the port data lock (if we're a driver thread) */ ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock)); - { - DeclareTmpHeapNoproc(buf,REF_THING_SIZE); - UseTmpHeapNoproc(REF_THING_SIZE); - ret = do_driver_monitor_process(prt,buf,process,monitor); - UnUseTmpHeapNoproc(REF_THING_SIZE); - } + ret = do_driver_monitor_process(prt,process,monitor); DRV_MONITOR_UNLOCK_PDL(prt); return ret; } -static int do_driver_demonitor_process(Port *prt, Eterm *buf, - const ErlDrvMonitor *monitor) +static int do_driver_demonitor_process(Port *prt, const ErlDrvMonitor *monitor) { + Eterm heap[ERTS_REF_THING_SIZE]; + Eterm *hp = &heap[0]; Process *rp; Eterm ref; ErtsMonitor *mon; Eterm to; - memcpy(buf,monitor,sizeof(Eterm)*REF_THING_SIZE); - ref = make_internal_ref(buf); + ref = erts_oiref_storage_make_ref((ErtsOIRefStorage *) monitor, &hp), + mon = erts_lookup_monitor(ERTS_P_MONITORS(prt), ref); if (mon == NULL) { return 1; @@ -7728,25 +7720,21 @@ int driver_demonitor_process(ErlDrvPort drvport, /* Now we should have either the port lock (if we have a scheduler) or the port data lock (if we're a driver thread) */ ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock)); - { - DeclareTmpHeapNoproc(buf,REF_THING_SIZE); - UseTmpHeapNoproc(REF_THING_SIZE); - ret = do_driver_demonitor_process(prt,buf,monitor); - UnUseTmpHeapNoproc(REF_THING_SIZE); - } + ret = do_driver_demonitor_process(prt,monitor); DRV_MONITOR_UNLOCK_PDL(prt); return ret; } -static ErlDrvTermData do_driver_get_monitored_process(Port *prt, Eterm *buf, - const ErlDrvMonitor *monitor) +static ErlDrvTermData do_driver_get_monitored_process(Port *prt,const ErlDrvMonitor *monitor) { Eterm ref; ErtsMonitor *mon; Eterm to; + Eterm heap[ERTS_REF_THING_SIZE]; + Eterm *hp = &heap[0]; + + ref = erts_oiref_storage_make_ref((ErtsOIRefStorage *) monitor, &hp), - memcpy(buf,monitor,sizeof(Eterm)*REF_THING_SIZE); - ref = make_internal_ref(buf); mon = erts_lookup_monitor(ERTS_P_MONITORS(prt), ref); if (mon == NULL) { return driver_term_nil; @@ -7774,12 +7762,7 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort drvport, /* Now we should have either the port lock (if we have a scheduler) or the port data lock (if we're a driver thread) */ ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock)); - { - DeclareTmpHeapNoproc(buf,REF_THING_SIZE); - UseTmpHeapNoproc(REF_THING_SIZE); - ret = do_driver_get_monitored_process(prt,buf,monitor); - UnUseTmpHeapNoproc(REF_THING_SIZE); - } + ret = do_driver_get_monitored_process(prt,monitor); DRV_MONITOR_UNLOCK_PDL(prt); return ret; } @@ -7788,7 +7771,8 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort drvport, int driver_compare_monitors(const ErlDrvMonitor *monitor1, const ErlDrvMonitor *monitor2) { - return memcmp(monitor1,monitor2,sizeof(ErlDrvMonitor)); + return erts_oiref_storage_cmp((ErtsOIRefStorage *) monitor1, + (ErtsOIRefStorage *) monitor2); } void erts_fire_port_monitor(Port *prt, Eterm ref) diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 25940edab7..fc459e17ad 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1838,7 +1838,7 @@ make_internal_hash(Eterm term) break; case REF_SUBTAG: UINT32_HASH(internal_ref_numbers(term)[0], HCONST_7); - ASSERT(internal_ref_no_of_numbers(term) == 3); + ASSERT(internal_ref_no_numbers(term) == 3); UINT32_HASH_2(internal_ref_numbers(term)[1], internal_ref_numbers(term)[2], HCONST_8); goto pop_next; @@ -1847,7 +1847,7 @@ make_internal_hash(Eterm term) { ExternalThing* thing = external_thing_ptr(term); - ASSERT(external_thing_ref_no_of_numbers(thing) == 3); + ASSERT(external_thing_ref_no_numbers(thing) == 3); /* See limitation #2 */ #ifdef ARCH_64 POINTER_HASH(thing->node, HCONST_7); @@ -2486,22 +2486,20 @@ tailrecur_ne: anum = external_thing_ref_numbers(athing); bnum = external_thing_ref_numbers(bthing); - alen = external_thing_ref_no_of_numbers(athing); - blen = external_thing_ref_no_of_numbers(bthing); + alen = external_thing_ref_no_numbers(athing); + blen = external_thing_ref_no_numbers(bthing); goto ref_common; + case REF_SUBTAG: - if (!is_internal_ref(b)) - goto not_equal; - { - RefThing* athing = ref_thing_ptr(a); - RefThing* bthing = ref_thing_ptr(b); - alen = internal_thing_ref_no_of_numbers(athing); - blen = internal_thing_ref_no_of_numbers(bthing); - anum = internal_thing_ref_numbers(athing); - bnum = internal_thing_ref_numbers(bthing); - } + if (!is_internal_ref(b)) + goto not_equal; + + alen = internal_ref_no_numbers(a); + anum = internal_ref_numbers(a); + blen = internal_ref_no_numbers(b); + bnum = internal_ref_numbers(b); ref_common: ASSERT(alen > 0 && blen > 0); @@ -3133,25 +3131,21 @@ tailrecur_ne: */ if (is_internal_ref(b)) { - RefThing* bthing = ref_thing_ptr(b); bnode = erts_this_node; - bnum = internal_thing_ref_numbers(bthing); - blen = internal_thing_ref_no_of_numbers(bthing); + blen = internal_ref_no_numbers(b); + bnum = internal_ref_numbers(b); } else if(is_external_ref(b)) { ExternalThing* bthing = external_thing_ptr(b); bnode = bthing->node; bnum = external_thing_ref_numbers(bthing); - blen = external_thing_ref_no_of_numbers(bthing); + blen = external_thing_ref_no_numbers(bthing); } else { a_tag = REF_DEF; goto mixed_types; } - { - RefThing* athing = ref_thing_ptr(a); - anode = erts_this_node; - anum = internal_thing_ref_numbers(athing); - alen = internal_thing_ref_no_of_numbers(athing); - } + anode = erts_this_node; + alen = internal_ref_no_numbers(a); + anum = internal_ref_numbers(a); ref_common: CMP_NODES(anode, bnode); @@ -3181,15 +3175,14 @@ tailrecur_ne: goto pop_next; case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE): if (is_internal_ref(b)) { - RefThing* bthing = ref_thing_ptr(b); bnode = erts_this_node; - bnum = internal_thing_ref_numbers(bthing); - blen = internal_thing_ref_no_of_numbers(bthing); + blen = internal_ref_no_numbers(b); + bnum = internal_ref_numbers(b); } else if (is_external_ref(b)) { ExternalThing* bthing = external_thing_ptr(b); bnode = bthing->node; bnum = external_thing_ref_numbers(bthing); - blen = external_thing_ref_no_of_numbers(bthing); + blen = external_thing_ref_no_numbers(bthing); } else { a_tag = EXTERNAL_REF_DEF; goto mixed_types; @@ -3198,7 +3191,7 @@ tailrecur_ne: ExternalThing* athing = external_thing_ptr(a); anode = athing->node; anum = external_thing_ref_numbers(athing); - alen = external_thing_ref_no_of_numbers(athing); + alen = external_thing_ref_no_numbers(athing); } goto ref_common; default: @@ -3575,40 +3568,39 @@ not_equal: Eterm store_external_or_ref_(Uint **hpp, ErlOffHeap* oh, Eterm ns) { + struct erl_off_heap_header *ohhp; Uint i; Uint size; - Uint *from_hp; - Uint *to_hp = *hpp; + Eterm *from_hp; + Eterm *to_hp = *hpp; ASSERT(is_external(ns) || is_internal_ref(ns)); - if(is_external(ns)) { - from_hp = external_val(ns); - size = thing_arityval(*from_hp) + 1; - *hpp += size; - - for(i = 0; i < size; i++) - to_hp[i] = from_hp[i]; - - erts_smp_refc_inc(&((ExternalThing *) to_hp)->node->refc, 2); - - ((struct erl_off_heap_header*) to_hp)->next = oh->first; - oh->first = (struct erl_off_heap_header*) to_hp; - - return make_external(to_hp); - } - - /* Internal ref */ - from_hp = internal_ref_val(ns); - + from_hp = boxed_val(ns); size = thing_arityval(*from_hp) + 1; - *hpp += size; for(i = 0; i < size; i++) to_hp[i] = from_hp[i]; - return make_internal_ref(to_hp); + if (is_external_header(*from_hp)) { + ExternalThing *etp = (ExternalThing *) from_hp; + ASSERT(is_external(ns)); + erts_smp_refc_inc(&etp->node->refc, 2); + } + else if (is_ordinary_ref_thing(from_hp)) + return make_internal_ref(to_hp); + else { + ErtsMRefThing *mreft = (ErtsMRefThing *) from_hp; + ASSERT(is_magic_ref_thing(from_hp)); + erts_refc_inc(&mreft->mb->refc, 2); + } + + ohhp = (struct erl_off_heap_header*) to_hp; + ohhp->next = oh->first; + oh->first = ohhp; + + return make_boxed(to_hp); } Eterm -- cgit v1.2.3 From 7fee9c57d89be05c980d1684329b26ac991bb26a Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 23 Jan 2017 17:16:28 +0100 Subject: Use magic refs for NIF resources --- erts/emulator/beam/erl_nif.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 20b00bd4ba..1de56ce0a1 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2209,34 +2209,40 @@ ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); - Eterm* hp = alloc_heap(env,PROC_BIN_SIZE); - return erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary); + Eterm* hp = alloc_heap(env, ERTS_MAGIC_REF_THING_SIZE); + return erts_mk_magic_ref(&hp, &MSO(env->proc), &bin->binary); } ERL_NIF_TERM enif_make_resource_binary(ErlNifEnv* env, void* obj, const void* data, size_t size) { - Eterm bin = enif_make_resource(env, obj); - ProcBin* pb = (ProcBin*) binary_val(bin); + ErlNifResource* resource = DATA_TO_RESOURCE(obj); + ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); + Eterm* hp = alloc_heap(env,PROC_BIN_SIZE); + Eterm ebin = erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary); + ProcBin* pb = (ProcBin*) binary_val(ebin); pb->bytes = (byte*) data; pb->size = size; - return bin; + return ebin; } int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* type, void** objp) { - ProcBin* pb; Binary* mbin; ErlNifResource* resource; - if (!ERTS_TERM_IS_MAGIC_BINARY(term)) { - return 0; + if (is_internal_magic_ref(term)) + mbin = erts_magic_ref2bin(term); + else { + ProcBin* pb; + if (!ERTS_TERM_IS_MAGIC_BINARY(term)) + return 0; + pb = (ProcBin*) binary_val(term); + /*if (pb->size != 0) { + return 0; / * Or should we allow "resource binaries" as handles? * / + }*/ + mbin = pb->val; } - pb = (ProcBin*) binary_val(term); - /*if (pb->size != 0) { - return 0; / * Or should we allow "resource binaries" as handles? * / - }*/ - mbin = pb->val; resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(mbin); if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor || resource->type != type) { -- cgit v1.2.3 From 7c70239985c4591ef2770a59a1bf62b51f5108cc Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 23 Jan 2017 17:06:42 +0100 Subject: Use magic refs in trapping processes()/ports() BIFs --- erts/emulator/beam/erl_ptab.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c index 22830a19c4..a132b1d334 100644 --- a/erts/emulator/beam/erl_ptab.c +++ b/erts/emulator/beam/erl_ptab.c @@ -771,18 +771,18 @@ erts_ptab_list(Process *c_p, ErtsPTab *ptab) } else { Eterm *hp; - Eterm magic_bin; + Eterm magic_ref; ERTS_PTAB_LIST_DBG_CHK_RESLIST(res_acc); - hp = HAlloc(c_p, PROC_BIN_SIZE); - ERTS_PTAB_LIST_DBG_SAVE_HEAP_ALLOC(ptlbdp, hp, PROC_BIN_SIZE); - magic_bin = erts_mk_magic_binary_term(&hp, &MSO(c_p), mbp); + hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE); + ERTS_PTAB_LIST_DBG_SAVE_HEAP_ALLOC(ptlbdp, hp, ERTS_MAGIC_REF_THING_SIZE); + magic_ref = erts_mk_magic_ref(&hp, &MSO(c_p), mbp); ERTS_PTAB_LIST_DBG_VERIFY_HEAP_ALLOC_USED(ptlbdp, hp); ERTS_PTAB_LIST_DBG_TRACE(c_p->common.id, trap); ERTS_BIF_PREP_YIELD2(ret_val, &ptab_list_continue_export, c_p, res_acc, - magic_bin); + magic_ref); } return ret_val; } @@ -1287,9 +1287,7 @@ static BIF_RETTYPE ptab_list_continue(BIF_ALIST_2) res_acc = BIF_ARG_1; - ERTS_PTAB_LIST_ASSERT(ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_2)); - - mbp = ((ProcBin *) binary_val(BIF_ARG_2))->val; + mbp = erts_magic_ref2bin(BIF_ARG_2); ERTS_PTAB_LIST_ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == cleanup_ptab_list_bif_data); -- cgit v1.2.3 From 5a97997217e5c3f901e8fefbd7bbf6c64652c9a8 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 23 Jan 2017 17:07:23 +0100 Subject: Use magic refs for code loading state --- erts/emulator/beam/beam_bif_load.c | 26 ++++++++++++-------------- erts/emulator/beam/beam_load.c | 8 ++++---- 2 files changed, 16 insertions(+), 18 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index a5891a0ec2..4ba8c2a669 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -156,11 +156,13 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) Module* modp; Eterm res, mod; - if (!ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_1) || - is_not_atom(mod = erts_module_for_prepared_code - (((ProcBin*)binary_val(BIF_ARG_1))->val))) { + if (!is_internal_magic_ref(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); + + mod = erts_module_for_prepared_code(erts_magic_ref2bin(BIF_ARG_1)); + + if (is_not_atom(mod)) BIF_ERROR(BIF_P, BADARG); - } if (!erts_try_seize_code_write_permission(BIF_P)) { ERTS_BIF_YIELD3(bif_export[BIF_code_make_stub_module_3], @@ -229,8 +231,8 @@ prepare_loading_2(BIF_ALIST_2) res = TUPLE2(hp, am_error, reason); BIF_RET(res); } - hp = HAlloc(BIF_P, PROC_BIN_SIZE); - res = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), magic); + hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE); + res = erts_mk_magic_ref(&hp, &MSO(BIF_P), magic); erts_refc_dec(&magic->refc, 1); BIF_RET(res); } @@ -239,15 +241,13 @@ BIF_RETTYPE has_prepared_code_on_load_1(BIF_ALIST_1) { Eterm res; - ProcBin* pb; - if (!ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_1)) { + if (!is_internal_magic_ref(BIF_ARG_1)) { error: BIF_ERROR(BIF_P, BADARG); } - pb = (ProcBin*) binary_val(BIF_ARG_1); - res = erts_has_code_on_load(pb->val); + res = erts_has_code_on_load(erts_magic_ref2bin(BIF_ARG_1)); if (res == NIL) { goto error; } @@ -333,13 +333,11 @@ finish_loading_1(BIF_ALIST_1) for (i = 0; i < n; i++) { Eterm* cons = list_val(BIF_ARG_1); Eterm term = CAR(cons); - ProcBin* pb; - if (!ERTS_TERM_IS_MAGIC_BINARY(term)) { + if (!is_internal_magic_ref(term)) { goto badarg; } - pb = (ProcBin*) binary_val(term); - p[i].code = pb->val; + p[i].code = erts_magic_ref2bin(term); p[i].module = erts_module_for_prepared_code(p[i].code); if (p[i].module == NIL) { goto badarg; diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 9f38dd4c3a..e75e7afd54 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -6322,8 +6322,8 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) stp = ERTS_MAGIC_BIN_DATA(magic); hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code)); - if (!ERTS_TERM_IS_MAGIC_BINARY(hipe_magic_bin) || - !(hipe_magic = ((ProcBin*)binary_val(hipe_magic_bin))->val, + if (!is_internal_magic_ref(hipe_magic_bin) || + !(hipe_magic = erts_magic_ref2bin(hipe_magic_bin), hipe_stp = hipe_get_loader_state(hipe_magic)) || hipe_stp->module == NIL || hipe_stp->text_segment == 0) { goto error; @@ -6568,8 +6568,8 @@ int erts_commit_hipe_patch_load(Eterm hipe_magic_bin) HipeModule *hipe_code; Module* modp; - if (!ERTS_TERM_IS_MAGIC_BINARY(hipe_magic_bin) || - !(hipe_magic = ((ProcBin*)binary_val(hipe_magic_bin))->val, + if (!is_internal_magic_ref(hipe_magic_bin) || + !(hipe_magic = erts_magic_ref2bin(hipe_magic_bin), hipe_stp = hipe_get_loader_state(hipe_magic)) || hipe_stp->module == NIL || hipe_stp->text_segment == 0) { return 0; -- cgit v1.2.3 From 8d4dd97bcbd0988b08f8f8141ec7cfb17a16aa4a Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 23 Jan 2017 18:44:39 +0100 Subject: Use magic refs for re:run() static NIFs trap --- erts/emulator/beam/erl_bif_re.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index ff7746ce1d..ba183f24e8 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -1319,17 +1319,17 @@ handle_iolist: Binary *mbp = erts_create_magic_binary(sizeof(RestartContext), cleanup_restart_context_bin); RestartContext *restartp = ERTS_MAGIC_BIN_DATA(mbp); - Eterm magic_bin; + Eterm magic_ref; Eterm *hp; memcpy(restartp,&restart,sizeof(RestartContext)); BUMP_ALL_REDS(p); - hp = HAlloc(p, PROC_BIN_SIZE); - magic_bin = erts_mk_magic_binary_term(&hp, &MSO(p), mbp); + hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); + magic_ref = erts_mk_magic_ref(&hp, &MSO(p), mbp); BIF_TRAP3(&re_exec_trap_export, p, arg1, arg2 /* To avoid GC of precompiled code, XXX: not utilized yet */, - magic_bin); + magic_ref); } res = build_exec_return(p, rc, &restart, arg1); @@ -1366,9 +1366,7 @@ static BIF_RETTYPE re_exec_trap(BIF_ALIST_3) Uint loop_limit_tmp; Eterm res; - ASSERT(ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_3)); - - mbp = ((ProcBin *) binary_val(BIF_ARG_3))->val; + mbp = erts_magic_ref2bin(BIF_ARG_3); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == cleanup_restart_context_bin); -- cgit v1.2.3 From 2a78349342b9f72651c016b650321bb317098a3c Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 23 Jan 2017 21:26:22 +0100 Subject: Use magic refs for compiled match specs --- erts/emulator/beam/erl_db.c | 18 +++++---------- erts/emulator/beam/erl_db_hash.c | 32 +++++++++++++-------------- erts/emulator/beam/erl_db_tree.c | 40 +++++++++++++++------------------ erts/emulator/beam/erl_db_util.c | 15 ------------- erts/emulator/beam/erl_db_util.h | 43 ++++++++++++++++++++++++++++++++---- erts/emulator/beam/erl_node_tables.c | 8 +++---- 6 files changed, 82 insertions(+), 74 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 6a915a88a1..9f077dd407 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -2779,7 +2779,7 @@ BIF_RETTYPE ets_info_2(BIF_ALIST_2) BIF_RETTYPE ets_is_compiled_ms_1(BIF_ALIST_1) { - if (erts_db_is_compiled_ms(BIF_ARG_1)) { + if (erts_db_get_match_prog_binary(BIF_ARG_1)) { BIF_RET(am_true); } else { BIF_RET(am_false); @@ -2794,9 +2794,9 @@ BIF_RETTYPE ets_match_spec_compile_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } - hp = HAlloc(BIF_P, PROC_BIN_SIZE); + hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE); - BIF_RET(erts_mk_magic_binary_term(&hp, &MSO(BIF_P), mp)); + BIF_RET(erts_db_make_match_prog_ref(BIF_P, mp, &hp)); } BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3) @@ -2805,24 +2805,18 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3) int i = 0; Eterm *hp; Eterm lst; - ProcBin *bp; Binary *mp; Eterm res; Uint32 dummy; - if (!(is_list(BIF_ARG_1) || BIF_ARG_1 == NIL) || !is_binary(BIF_ARG_2)) { + if (!(is_list(BIF_ARG_1) || BIF_ARG_1 == NIL)) { error: BIF_ERROR(BIF_P, BADARG); } - bp = (ProcBin*) binary_val(BIF_ARG_2); - if (thing_subtag(bp->thing_word) != REFC_BINARY_SUBTAG) { + mp = erts_db_get_match_prog_binary(BIF_ARG_2); + if (!mp) goto error; - } - mp = bp->val; - if (!IsMatchProgBinary(mp)) { - goto error; - } if (BIF_ARG_1 == NIL) { BIF_RET(BIF_ARG_3); diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index b2a231467b..bb29d56aa7 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -1287,15 +1287,13 @@ static int db_select_continue_hash(Process *p, if (arityval(*tptr) != 6) RET_TO_BIF(NIL,DB_ERROR_BADPARAM); - if (!is_small(tptr[2]) || !is_small(tptr[3]) || !is_binary(tptr[4]) || + if (!is_small(tptr[2]) || !is_small(tptr[3]) || !(is_list(tptr[5]) || tptr[5] == NIL) || !is_small(tptr[6])) RET_TO_BIF(NIL,DB_ERROR_BADPARAM); if ((chunk_size = signed_val(tptr[3])) < 0) RET_TO_BIF(NIL,DB_ERROR_BADPARAM); - if (!(thing_subtag(*binary_val(tptr[4])) == REFC_BINARY_SUBTAG)) - RET_TO_BIF(NIL,DB_ERROR_BADPARAM); - mp = ((ProcBin *) binary_val(tptr[4]))->val; - if (!IsMatchProgBinary(mp)) + mp = erts_db_get_match_prog_binary(tptr[4]); + if (!mp) RET_TO_BIF(NIL,DB_ERROR_BADPARAM); all_objects = mp->flags & BIN_FLAG_ALL_OBJECTS; match_list = tptr[5]; @@ -1554,8 +1552,8 @@ done: been in 'user space' */ } if (rest != NIL || slot_ix >= 0) { /* Need more calls */ - hp = HAlloc(p,3+7+PROC_BIN_SIZE); - mpb =db_make_mp_binary(p,(mpi.mp),&hp); + hp = HAlloc(p,3+7+ERTS_MAGIC_REF_THING_SIZE); + mpb = erts_db_make_match_prog_ref(p,(mpi.mp),&hp); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; continuation = TUPLE6(hp, tb->common.id,make_small(slot_ix), @@ -1580,8 +1578,8 @@ trap: BUMP_ALL_REDS(p); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; - hp = HAlloc(p,7+PROC_BIN_SIZE); - mpb =db_make_mp_binary(p,(mpi.mp),&hp); + hp = HAlloc(p,7+ERTS_MAGIC_REF_THING_SIZE); + mpb = erts_db_make_match_prog_ref(p,(mpi.mp),&hp); continuation = TUPLE6(hp, tb->common.id, make_small(slot_ix), make_small(chunk_size), mpb, match_list, @@ -1693,15 +1691,15 @@ done: trap: BUMP_ALL_REDS(p); if (IS_USMALL(0, got)) { - hp = HAlloc(p, PROC_BIN_SIZE + 5); + hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE + 5); egot = make_small(got); } else { - hp = HAlloc(p, BIG_UINT_HEAP_SIZE + PROC_BIN_SIZE + 5); + hp = HAlloc(p, BIG_UINT_HEAP_SIZE + ERTS_MAGIC_REF_THING_SIZE + 5); egot = uint_to_big(got, hp); hp += BIG_UINT_HEAP_SIZE; } - mpb = db_make_mp_binary(p,mpi.mp,&hp); + mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp); continuation = TUPLE4(hp, tb->common.id, make_small(slot_ix), mpb, egot); @@ -1838,15 +1836,15 @@ done: trap: BUMP_ALL_REDS(p); if (IS_USMALL(0, got)) { - hp = HAlloc(p, PROC_BIN_SIZE + 5); + hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE + 5); egot = make_small(got); } else { - hp = HAlloc(p, BIG_UINT_HEAP_SIZE + PROC_BIN_SIZE + 5); + hp = HAlloc(p, BIG_UINT_HEAP_SIZE + ERTS_MAGIC_REF_THING_SIZE + 5); egot = uint_to_big(got, hp); hp += BIG_UINT_HEAP_SIZE; } - mpb = db_make_mp_binary(p,mpi.mp,&hp); + mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp); continuation = TUPLE4(hp, tb->common.id, make_small(slot_ix), mpb, egot); @@ -1887,7 +1885,7 @@ static int db_select_delete_continue_hash(Process *p, tptr = tuple_val(continuation); slot_ix = unsigned_val(tptr[2]); - mp = ((ProcBin *) binary_val(tptr[3]))->val; + mp = erts_db_get_match_prog_binary_unchecked(tptr[3]); if (is_big(tptr[4])) { got = big_to_uint32(tptr[4]); } else { @@ -1999,7 +1997,7 @@ static int db_select_count_continue_hash(Process *p, tptr = tuple_val(continuation); slot_ix = unsigned_val(tptr[2]); - mp = ((ProcBin *) binary_val(tptr[3]))->val; + mp = erts_db_get_match_prog_binary_unchecked(tptr[3]); if (is_big(tptr[4])) { got = big_to_uint32(tptr[4]); } else { diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index dd9403e132..c4ecd2ba37 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -935,17 +935,15 @@ static int db_select_continue_tree(Process *p, if (arityval(*tptr) != 8) RET_TO_BIF(NIL,DB_ERROR_BADPARAM); - if (!is_small(tptr[4]) || !is_binary(tptr[5]) || + if (!is_small(tptr[4]) || !(is_list(tptr[6]) || tptr[6] == NIL) || !is_small(tptr[7]) || !is_small(tptr[8])) RET_TO_BIF(NIL,DB_ERROR_BADPARAM); lastkey = tptr[2]; end_condition = tptr[3]; - if (!(thing_subtag(*binary_val(tptr[5])) == REFC_BINARY_SUBTAG)) - RET_TO_BIF(NIL,DB_ERROR_BADPARAM); - mp = ((ProcBin *) binary_val(tptr[5]))->val; - if (!IsMatchProgBinary(mp)) + mp = erts_db_get_match_prog_binary(tptr[5]); + if (!mp) RET_TO_BIF(NIL,DB_ERROR_BADPARAM); chunk_size = signed_val(tptr[4]); @@ -1145,11 +1143,11 @@ static int db_select_tree(Process *p, DbTable *tbl, key = GETKEY(tb, sc.lastobj); sz = size_object(key); - hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE); + hp = HAlloc(p, 9 + sz + ERTS_MAGIC_REF_THING_SIZE); key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; - mpb=db_make_mp_binary(p,mpi.mp,&hp); + mpb= erts_db_make_match_prog_ref(p,mpi.mp,&hp); continuation = TUPLE8 (hp, @@ -1208,10 +1206,8 @@ static int db_select_count_continue_tree(Process *p, lastkey = tptr[2]; end_condition = tptr[3]; - if (!(thing_subtag(*binary_val(tptr[4])) == REFC_BINARY_SUBTAG)) - RET_TO_BIF(NIL,DB_ERROR_BADPARAM); - mp = ((ProcBin *) binary_val(tptr[4]))->val; - if (!IsMatchProgBinary(mp)) + mp = erts_db_get_match_prog_binary(tptr[4]); + if (!mp) RET_TO_BIF(NIL,DB_ERROR_BADPARAM); sc.p = p; @@ -1338,18 +1334,18 @@ static int db_select_count_tree(Process *p, DbTable *tbl, key = GETKEY(tb, sc.lastobj); sz = size_object(key); if (IS_USMALL(0, sc.got)) { - hp = HAlloc(p, sz + PROC_BIN_SIZE + 6); + hp = HAlloc(p, sz + ERTS_MAGIC_REF_THING_SIZE + 6); egot = make_small(sc.got); } else { - hp = HAlloc(p, BIG_UINT_HEAP_SIZE + sz + PROC_BIN_SIZE + 6); + hp = HAlloc(p, BIG_UINT_HEAP_SIZE + sz + ERTS_MAGIC_REF_THING_SIZE + 6); egot = uint_to_big(sc.got, hp); hp += BIG_UINT_HEAP_SIZE; } key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; - mpb = db_make_mp_binary(p,mpi.mp,&hp); + mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp); continuation = TUPLE5 (hp, @@ -1470,11 +1466,11 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, key = GETKEY(tb, sc.lastobj); sz = size_object(key); - hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE); + hp = HAlloc(p, 9 + sz + ERTS_MAGIC_REF_THING_SIZE); key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; - mpb = db_make_mp_binary(p,mpi.mp,&hp); + mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp); continuation = TUPLE8 (hp, @@ -1495,12 +1491,12 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, key = GETKEY(tb, sc.lastobj); sz = size_object(key); - hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE); + hp = HAlloc(p, 9 + sz + ERTS_MAGIC_REF_THING_SIZE); key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; - mpb = db_make_mp_binary(p,mpi.mp,&hp); + mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp); continuation = TUPLE8 (hp, tb->common.id, @@ -1558,7 +1554,7 @@ static int db_select_delete_continue_tree(Process *p, sc.erase_lastterm = 0; /* Before first RET_TO_BIF */ sc.lastterm = NULL; - mp = ((ProcBin *) binary_val(tptr[4]))->val; + mp = erts_db_get_match_prog_binary_unchecked(tptr[4]); sc.p = p; sc.tb = tb; if (is_big(tptr[5])) { @@ -1682,16 +1678,16 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, key = GETKEY(tb, (sc.lastterm)->dbterm.tpl); sz = size_object(key); if (IS_USMALL(0, sc.accum)) { - hp = HAlloc(p, sz + PROC_BIN_SIZE + 6); + hp = HAlloc(p, sz + ERTS_MAGIC_REF_THING_SIZE + 6); eaccsum = make_small(sc.accum); } else { - hp = HAlloc(p, BIG_UINT_HEAP_SIZE + sz + PROC_BIN_SIZE + 6); + hp = HAlloc(p, BIG_UINT_HEAP_SIZE + sz + ERTS_MAGIC_REF_THING_SIZE + 6); eaccsum = uint_to_big(sc.accum, hp); hp += BIG_UINT_HEAP_SIZE; } key = copy_struct(key, sz, &hp, &MSO(p)); - mpb = db_make_mp_binary(p,mpi.mp,&hp); + mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp); continuation = TUPLE5 (hp, diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 0ab42394ce..070e29578f 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -2545,14 +2545,6 @@ success: } -/* - * Convert a match program to a "magic" binary to return up to erlang - */ -Eterm db_make_mp_binary(Process *p, Binary *mp, Eterm **hpp) -{ - return erts_mk_magic_binary_term(hpp, &MSO(p), mp); -} - DMCErrInfo *db_new_dmc_err_info(void) { DMCErrInfo *ret = erts_alloc(ERTS_ALC_T_DB_DMC_ERR_INFO, @@ -3266,13 +3258,6 @@ int db_has_variable(Eterm node) { return 0; } -int erts_db_is_compiled_ms(Eterm term) -{ - return (is_binary(term) - && (thing_subtag(*binary_val(term)) == REFC_BINARY_SUBTAG) - && IsMatchProgBinary((((ProcBin *) binary_val(term))->val))); -} - /* ** Local (static) utilities. */ diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 3fb063797e..b2a3bb6c20 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -23,6 +23,7 @@ #include "global.h" #include "erl_message.h" +#include "erl_bif_unique.h" /*#define HARDDEBUG 1*/ @@ -456,10 +457,44 @@ Eterm db_format_dmc_err_info(Process *p, DMCErrInfo *ei); void db_free_dmc_err_info(DMCErrInfo *ei); /* Completely free's an error info structure, including all recorded errors */ -Eterm db_make_mp_binary(Process *p, Binary *mp, Eterm **hpp); -/* Convert a match program to a erlang "magic" binary to be returned to userspace, - increments the reference counter. */ -int erts_db_is_compiled_ms(Eterm term); + +ERTS_GLB_INLINE Eterm erts_db_make_match_prog_ref(Process *p, Binary *mp, Eterm **hpp); +ERTS_GLB_INLINE Binary *erts_db_get_match_prog_binary(Eterm term); +ERTS_GLB_INLINE Binary *erts_db_get_match_prog_binary_unchecked(Eterm term); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +/* + * Convert a match program to a "magic" ref to return up to erlang + */ +ERTS_GLB_INLINE Eterm erts_db_make_match_prog_ref(Process *p, Binary *mp, Eterm **hpp) +{ + return erts_mk_magic_ref(hpp, &MSO(p), mp); +} + +ERTS_GLB_INLINE Binary * +erts_db_get_match_prog_binary_unchecked(Eterm term) +{ + Binary *bp = erts_magic_ref2bin(term); + ASSERT(bp->flags & BIN_FLAG_MAGIC); + ASSERT((ERTS_MAGIC_BIN_DESTRUCTOR(bp) == erts_db_match_prog_destructor)); + return bp; +} + +ERTS_GLB_INLINE Binary * +erts_db_get_match_prog_binary(Eterm term) +{ + Binary *bp; + if (!is_internal_magic_ref(term)) + return NULL; + bp = erts_magic_ref2bin(term); + ASSERT(bp->flags & BIN_FLAG_MAGIC); + if (ERTS_MAGIC_BIN_DESTRUCTOR(bp) != erts_db_match_prog_destructor) + return NULL; + return bp; +} + +#endif /* ** Convenience when compiling into Binary structures diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 060fbd5883..5e54e5aef2 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -1131,7 +1131,7 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) InsertedBin *ib; int insert_bin = 1; for (ib = inserted_bins; ib; ib = ib->next) - if(ib->bin_val == u.pb->val) { + if(ib->bin_val == (Binary *) u.mref->mb) { insert_bin = 0; break; } @@ -1140,12 +1140,12 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) Uint *hp = &id_heap[0]; InsertedBin *nib; UseTmpHeapNoproc(BIG_UINT_HEAP_SIZE); - a.id = erts_bld_uint(&hp, NULL, (Uint) u.pb->val); - erts_match_prog_foreach_offheap(u.pb->val, + a.id = erts_bld_uint(&hp, NULL, (Uint) u.mref->mb); + erts_match_prog_foreach_offheap((Binary *) u.mref->mb, insert_offheap2, (void *) &a); nib = erts_alloc(ERTS_ALC_T_NC_TMP, sizeof(InsertedBin)); - nib->bin_val = u.pb->val; + nib->bin_val = (Binary *) u.mref->mb; nib->next = inserted_bins; inserted_bins = nib; UnUseTmpHeapNoproc(BIG_UINT_HEAP_SIZE); -- cgit v1.2.3 From 94df64424e41952d340c0082c70f79e6b6dbabd3 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 24 Jan 2017 15:07:09 +0100 Subject: Use magic refs for list_to_binary/binary_to_list traps --- erts/emulator/beam/binary.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index a09950ec40..d38097fb12 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -440,18 +440,17 @@ binary_to_list(Process *c_p, Eterm *hp, Eterm tail, byte *bytes, sp->size = size; sp->bitoffs = bitoffs; - hp = HAlloc(c_p, PROC_BIN_SIZE); - mb = erts_mk_magic_binary_term(&hp, &MSO(c_p), mbp); + hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE); + mb = erts_mk_magic_ref(&hp, &MSO(c_p), mbp); return binary_to_list_chunk(c_p, mb, sp, reds_left, 0); } } static BIF_RETTYPE binary_to_list_continue(BIF_ALIST_1) { - Binary *mbp = ((ProcBin *) binary_val(BIF_ARG_1))->val; + Binary *mbp = erts_magic_ref2bin(BIF_ARG_1); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == b2l_state_destructor); - ASSERT(BIF_P->flags & F_DISABLE_GC); return binary_to_list_chunk(BIF_P, @@ -787,8 +786,8 @@ list_to_binary_chunk(Eterm mb_eterm, ERTS_L2B_STATE_MOVE(new_sp, sp); sp = new_sp; - hp = HAlloc(c_p, PROC_BIN_SIZE); - mb_eterm = erts_mk_magic_binary_term(&hp, &MSO(c_p), mbp); + hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE); + mb_eterm = erts_mk_magic_ref(&hp, &MSO(c_p), mbp); ASSERT(is_value(mb_eterm)); @@ -834,9 +833,9 @@ list_to_binary_chunk(Eterm mb_eterm, static BIF_RETTYPE list_to_binary_continue(BIF_ALIST_1) { - Binary *mbp = ((ProcBin *) binary_val(BIF_ARG_1))->val; - ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == l2b_state_destructor); + Binary *mbp = erts_magic_ref2bin(BIF_ARG_1); + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == l2b_state_destructor); ASSERT(BIF_P->flags & F_DISABLE_GC); return list_to_binary_chunk(BIF_ARG_1, -- cgit v1.2.3 From 3bdb334a95e9dd49c1ada55b6f442099fdf7f72b Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 24 Jan 2017 15:10:55 +0100 Subject: Use magic refs for binary compile patterns --- erts/emulator/beam/erl_bif_binary.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index 6e10980b6b..e57cd06cec 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -1010,8 +1010,8 @@ BIF_RETTYPE binary_compile_pattern_1(BIF_ALIST_1) if (do_binary_match_compile(BIF_ARG_1,&tag,&bin)) { BIF_ERROR(BIF_P,BADARG); } - hp = HAlloc(BIF_P, PROC_BIN_SIZE+3); - ret = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), bin); + hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE+3); + ret = erts_mk_magic_ref(&hp, &MSO(BIF_P), bin); ret = TUPLE2(hp, tag, ret); BIF_RET(ret); } @@ -1426,11 +1426,11 @@ binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, Uint flags) goto badarg; } if (((tp[1] != am_bm) && (tp[1] != am_ac)) || - !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) { + !is_internal_magic_ref(tp[2])) { goto badarg; } bfs.type = tp[1]; - bin = ((ProcBin *) binary_val(tp[2]))->val; + bin = erts_magic_ref2bin(tp[2]); if (bfs.type == am_bm && ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) { goto badarg; @@ -1448,8 +1448,8 @@ binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, Uint flags) bfs.global_result = &do_match_global_result; runres = do_binary_find(p, arg1, &bfs, bin, NIL, &result); if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) { - Eterm *hp = HAlloc(p, PROC_BIN_SIZE); - bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), bin); + Eterm *hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); + bin_term = erts_mk_magic_ref(&hp, &MSO(p), bin); } else if (bin_term == NIL) { erts_bin_free(bin); } @@ -1512,11 +1512,11 @@ binary_split(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) goto badarg; } if (((tp[1] != am_bm) && (tp[1] != am_ac)) || - !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) { + !is_internal_magic_ref(tp[2])) { goto badarg; } bfs.type = tp[1]; - bin = ((ProcBin *) binary_val(tp[2]))->val; + bin = erts_magic_ref2bin(tp[2]); if (bfs.type == am_bm && ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) { goto badarg; @@ -1534,8 +1534,8 @@ binary_split(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) bfs.global_result = &do_split_global_result; runres = do_binary_find(p, arg1, &bfs, bin, NIL, &result); if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) { - Eterm *hp = HAlloc(p, PROC_BIN_SIZE); - bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), bin); + Eterm *hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); + bin_term = erts_mk_magic_ref(&hp, &MSO(p), bin); } else if (bin_term == NIL) { erts_bin_free(bin); } @@ -1767,7 +1767,8 @@ static BIF_RETTYPE binary_find_trap(BIF_ALIST_3) { int runres; Eterm result; - Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val; + Binary *bin = erts_magic_ref2bin(BIF_ARG_3); + runres = do_binary_find(BIF_P, BIF_ARG_1, NULL, bin, BIF_ARG_2, &result); if (runres == DO_BIN_MATCH_OK) { BIF_RET(result); @@ -2186,8 +2187,8 @@ static BIF_RETTYPE do_longest_common(Process *p, Eterm list, int direction) cd[i].type = CL_TYPE_HEAP; } } - hp = HAlloc(p, PROC_BIN_SIZE); - bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), mb); + hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); + bin_term = erts_mk_magic_ref(&hp, &MSO(p), mb); BUMP_ALL_REDS(p); BIF_TRAP3(trapper, p, bin_term, epos,list); } @@ -2214,8 +2215,7 @@ static BIF_RETTYPE do_longest_common_trap(Process *p, Eterm bin_term, Eterm curr #else term_to_Uint(current_pos, &pos); #endif - ASSERT(ERTS_TERM_IS_MAGIC_BINARY(bin_term)); - bin = ((ProcBin *) binary_val(bin_term))->val; + bin = erts_magic_ref2bin(bin_term); cd = (CommonData *) ERTS_MAGIC_BIN_DATA(bin); if (direction == DIRECTION_PREFIX) { trapper = &binary_longest_prefix_trap_export; @@ -2680,8 +2680,8 @@ static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en) cbs->source_size = size; cbs->result_pos = pos; cbs->times_left = n-i; - hp = HAlloc(p,PROC_BIN_SIZE); - trap_term = erts_mk_magic_binary_term(&hp, &MSO(p), mb); + hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); + trap_term = erts_mk_magic_ref(&hp, &MSO(p), mb); BUMP_ALL_REDS(p); BIF_TRAP2(&binary_copy_trap_export, p, bin, trap_term); } else { @@ -2716,7 +2716,7 @@ BIF_RETTYPE binary_copy_trap(BIF_ALIST_2) Uint reds = get_reds(BIF_P, BINARY_COPY_LOOP_FACTOR); byte *t; Uint pos; - Binary *mb = ((ProcBin *) binary_val(BIF_ARG_2))->val; + Binary *mb = erts_magic_ref2bin(BIF_ARG_2); CopyBinState *cbs = (CopyBinState *) ERTS_MAGIC_BIN_DATA(mb); Uint opos; -- cgit v1.2.3 From 40d55ced08579e9fc0ade28ba0810bf800690394 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 24 Jan 2017 15:11:28 +0100 Subject: Use magic refs for unicode static NIFs traps --- erts/emulator/beam/erl_unicode.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index 8919898181..dd7c589c3d 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -129,13 +129,9 @@ static void cleanup_restart_context_bin(Binary *bp) cleanup_restart_context(rc); } -static RestartContext *get_rc_from_bin(Eterm bin) +static RestartContext *get_rc_from_bin(Eterm mref) { - Binary *mbp; - ASSERT(ERTS_TERM_IS_MAGIC_BINARY(bin)); - - mbp = ((ProcBin *) binary_val(bin))->val; - + Binary *mbp = erts_magic_ref2bin(mref); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == cleanup_restart_context_bin); return (RestartContext *) ERTS_MAGIC_BIN_DATA(mbp); @@ -148,8 +144,8 @@ static Eterm make_magic_bin_for_restart(Process *p, RestartContext *rc) RestartContext *restartp = ERTS_MAGIC_BIN_DATA(mbp); Eterm *hp; memcpy(restartp,rc,sizeof(RestartContext)); - hp = HAlloc(p, PROC_BIN_SIZE); - return erts_mk_magic_binary_term(&hp, &MSO(p), mbp); + hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); + return erts_mk_magic_ref(&hp, &MSO(p), mbp); } -- cgit v1.2.3 From 13f02f2bb6e1e3ce709abc9ff846b24331ada58d Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 24 Jan 2017 17:29:18 +0100 Subject: Use magic refs for distributed send trap context --- erts/emulator/beam/bif.c | 2 +- erts/emulator/beam/dist.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index bda7b9224c..2190c47262 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -2446,7 +2446,7 @@ BIF_RETTYPE send_2(BIF_ALIST_2) static BIF_RETTYPE dsend_continue_trap_1(BIF_ALIST_1) { - Binary* bin = ((ProcBin*) binary_val(BIF_ARG_1))->val; + Binary* bin = erts_magic_ref2bin(BIF_ARG_1); ErtsSendContext* ctx = (ErtsSendContext*) ERTS_MAGIC_BIN_DATA(bin); Sint initial_reds = (Sint) (ERTS_BIF_REDS_LEFT(BIF_P) * TERM_TO_BINARY_LOOP_FACTOR); int result; diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 89d91065c2..390f2a0db3 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -741,7 +741,7 @@ Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx) Binary* ctx_bin = erts_create_magic_binary(sizeof(struct exported_ctx), erts_dsend_context_dtor); struct exported_ctx* dst = ERTS_MAGIC_BIN_DATA(ctx_bin); - Eterm* hp = HAlloc(p, PROC_BIN_SIZE); + Eterm* hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); sys_memcpy(&dst->ctx, ctx, sizeof(ErtsSendContext)); ASSERT(ctx->dss.ctl == make_tuple(ctx->ctl_heap)); @@ -750,7 +750,7 @@ Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx) sys_memcpy(&dst->acm, ctx->dss.acmp, sizeof(ErtsAtomCacheMap)); dst->ctx.dss.acmp = &dst->acm; } - return erts_mk_magic_binary_term(&hp, &MSO(p), ctx_bin); + return erts_mk_magic_ref(&hp, &MSO(p), ctx_bin); } -- cgit v1.2.3 From cf0e60eb5b1a6a2fec388bc650235b9eded335ec Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 24 Jan 2017 17:30:03 +0100 Subject: Use magic refs binary_to_term/term_to_binary trap context --- erts/emulator/beam/external.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 64b17a3f57..60df7b8a08 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1079,7 +1079,7 @@ static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1) Eterm *tp = tuple_val(BIF_ARG_1); Eterm Term = tp[1]; Eterm bt = tp[2]; - Binary *bin = ((ProcBin *) binary_val(bt))->val; + Binary *bin = erts_magic_ref2bin(bt); Eterm res = erts_term_to_binary_int(BIF_P, Term, 0, 0,bin); if (is_tuple(res)) { ASSERT(BIF_P->flags & F_DISABLE_GC); @@ -1408,7 +1408,7 @@ static void b2t_context_destructor(Binary *context_bin) static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1) { - Binary *context_bin = ((ProcBin *) binary_val(BIF_ARG_1))->val; + Binary *context_bin = erts_magic_ref2bin(BIF_ARG_1); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor); return binary_to_term_int(BIF_P, 0, THE_NON_VALUE, context_bin, NULL, @@ -1442,8 +1442,8 @@ static B2TContext* b2t_export_context(Process* p, B2TContext* src) if (ctx->state >= B2TDecode && ctx->u.dc.next == &src->u.dc.res) { ctx->u.dc.next = &ctx->u.dc.res; } - hp = HAlloc(p, PROC_BIN_SIZE); - ctx->trap_bin = erts_mk_magic_binary_term(&hp, &MSO(p), context_b); + hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); + ctx->trap_bin = erts_mk_magic_ref(&hp, &MSO(p), context_b); return ctx; } @@ -1871,8 +1871,8 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla #define RETURN_STATE() \ do { \ - hp = HAlloc(p, PROC_BIN_SIZE+3); \ - c_term = erts_mk_magic_binary_term(&hp, &MSO(p), context_b); \ + hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE+3); \ + c_term = erts_mk_magic_ref(&hp, &MSO(p), context_b); \ res = TUPLE2(hp, Term, c_term); \ BUMP_ALL_REDS(p); \ return res; \ -- cgit v1.2.3 From 3a34a18c2f318a4a9ea475ee80fe05bd64f05070 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 24 Jan 2017 17:31:01 +0100 Subject: Use magic refs for maps merge trap context --- erts/emulator/beam/erl_map.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index 979a0040b0..990be8efb2 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -1197,7 +1197,7 @@ static void hashmap_merge_ctx_destructor(Binary* ctx_bin) } BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1) { - Binary* ctx_bin = ((ProcBin *) binary_val(BIF_ARG_1))->val; + Binary* ctx_bin = erts_magic_ref2bin(BIF_ARG_1); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(ctx_bin) == hashmap_merge_ctx_destructor); @@ -1453,9 +1453,9 @@ trap: /* Yield */ hashmap_merge_ctx_destructor); ctx = ERTS_MAGIC_BIN_DATA(ctx_b); sys_memcpy(ctx, &local_ctx, sizeof(HashmapMergeContext)); - hp = HAlloc(p, PROC_BIN_SIZE); + hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); ASSERT(ctx->trap_bin == THE_NON_VALUE); - ctx->trap_bin = erts_mk_magic_binary_term(&hp, &MSO(p), ctx_b); + ctx->trap_bin = erts_mk_magic_ref(&hp, &MSO(p), ctx_b); erts_set_gc_state(p, 0); } -- cgit v1.2.3 From 28735b9c2c6390df593b05f300151addbd01e367 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 24 Jan 2017 17:31:53 +0100 Subject: Adjust the only usage of exposed magic binaries --- erts/emulator/beam/erl_nif.c | 40 ++++++++++++++++++++++++++++------------ erts/emulator/beam/global.h | 35 ----------------------------------- 2 files changed, 28 insertions(+), 47 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 1de56ce0a1..3674a29bf8 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2218,12 +2218,22 @@ ERL_NIF_TERM enif_make_resource_binary(ErlNifEnv* env, void* obj, { ErlNifResource* resource = DATA_TO_RESOURCE(obj); ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); + ErlOffHeap *ohp = &MSO(env->proc); Eterm* hp = alloc_heap(env,PROC_BIN_SIZE); - Eterm ebin = erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary); - ProcBin* pb = (ProcBin*) binary_val(ebin); - pb->bytes = (byte*) data; + ProcBin* pb = (ProcBin *) hp; + + pb->thing_word = HEADER_PROC_BIN; pb->size = size; - return ebin; + pb->next = ohp->first; + ohp->first = (struct erl_off_heap_header*) pb; + pb->val = &bin->binary; + pb->bytes = (byte*) data; + pb->flags = 0; + + OH_OVERHEAD(ohp, size / sizeof(Eterm)); + erts_refc_inc(&bin->binary.refc, 1); + + return make_binary(hp); } int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* type, @@ -2234,14 +2244,20 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ if (is_internal_magic_ref(term)) mbin = erts_magic_ref2bin(term); else { - ProcBin* pb; - if (!ERTS_TERM_IS_MAGIC_BINARY(term)) - return 0; - pb = (ProcBin*) binary_val(term); - /*if (pb->size != 0) { - return 0; / * Or should we allow "resource binaries" as handles? * / - }*/ - mbin = pb->val; + Eterm *hp; + if (!is_binary(term)) + return 0; + hp = binary_val(term); + if (thing_subtag(*hp) != REFC_BINARY_SUBTAG) + return 0; + /* + if (((ProcBin *) hp)->size != 0) { + return 0; / * Or should we allow "resource binaries" as handles? * / + } + */ + mbin = ((ProcBin *) hp)->val; + if (!(mbin->flags & BIN_FLAG_MAGIC)) + return 0; } resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(mbin); if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 187a948a7b..fff22fe9c1 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -237,41 +237,6 @@ typedef struct proc_bin { */ #define PROC_BIN_SIZE (sizeof(ProcBin)/sizeof(Eterm)) -ERTS_GLB_INLINE Eterm erts_mk_magic_binary_term(Eterm **hpp, - ErlOffHeap *ohp, - Binary *mbp); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -ERTS_GLB_INLINE Eterm -erts_mk_magic_binary_term(Eterm **hpp, ErlOffHeap *ohp, Binary *mbp) -{ - ProcBin *pb = (ProcBin *) *hpp; - *hpp += PROC_BIN_SIZE; - - ASSERT(mbp->flags & BIN_FLAG_MAGIC); - - pb->thing_word = HEADER_PROC_BIN; - pb->size = 0; - pb->next = ohp->first; - ohp->first = (struct erl_off_heap_header*) pb; - pb->val = mbp; - pb->bytes = (byte *) mbp->orig_bytes; - pb->flags = 0; - - erts_refc_inc(&mbp->refc, 1); - - return make_binary(pb); -} - -#endif - -#define ERTS_TERM_IS_MAGIC_BINARY(T) \ - (is_binary((T)) \ - && (thing_subtag(*binary_val((T))) == REFC_BINARY_SUBTAG) \ - && (((ProcBin *) binary_val((T)))->val->flags & BIN_FLAG_MAGIC)) - - union erl_off_heap_ptr { struct erl_off_heap_header* hdr; ProcBin *pb; -- cgit v1.2.3 From 8a72a253e2eda59e510d953f0b6eb21b50e06d0e Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 24 Jan 2017 20:43:08 +0100 Subject: Add binary overhead for magic ref/binaries --- erts/emulator/beam/erl_bif_unique.h | 2 ++ erts/emulator/beam/erl_gc.c | 59 +++++++++++++++++++++++++++++-------- erts/emulator/beam/external.c | 2 ++ erts/emulator/beam/utils.c | 4 ++- 4 files changed, 53 insertions(+), 14 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h index 6cd6cc3e09..27c2a15a5e 100644 --- a/erts/emulator/beam/erl_bif_unique.h +++ b/erts/emulator/beam/erl_bif_unique.h @@ -182,6 +182,7 @@ erts_mk_magic_ref(Eterm **hpp, ErlOffHeap *ohp, Binary *bp) write_magic_ref_thing(hp, ohp, (ErtsMagicBinary *) bp); *hpp += ERTS_MAGIC_REF_THING_SIZE; erts_refc_inc(&bp->refc, 1); + OH_OVERHEAD(ohp, bp->orig_size / sizeof(Eterm)); return make_internal_ref(hp); } @@ -329,6 +330,7 @@ erts_iref_storage_make_ref(ErtsIRefStorage *iref, } else { write_magic_ref_thing(hp, ohp, iref->u.mb); + OH_OVERHEAD(ohp, iref->u.mb->orig_size / sizeof(Eterm)); *hpp += ERTS_MAGIC_REF_THING_SIZE; /* * If we clean storage, the term inherits the diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 807d9d15e4..cb3a189e8f 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -2800,6 +2800,16 @@ link_live_proc_bin(struct shrink_cand_data *shrink, *prevppp = &pbp->next; } +#ifdef ERTS_MAGIC_REF_THING_HEADER +/* + * ERTS_MAGIC_REF_THING_HEADER only defined when there + * is a size difference between magic and ordinary references... + */ +# define ERTS_USED_MAGIC_REF_THING_HEADER__ ERTS_MAGIC_REF_THING_HEADER +#else +# define ERTS_USED_MAGIC_REF_THING_HEADER__ ERTS_REF_THING_HEADER +#endif + static void sweep_off_heap(Process *p, int fullsweep) @@ -2832,7 +2842,8 @@ sweep_off_heap(Process *p, int fullsweep) ASSERT(!ErtsInArea(ptr, oheap, oheap_sz)); *prev = ptr = (struct erl_off_heap_header*) boxed_val(ptr->thing_word); ASSERT(!IS_MOVED_BOXED(ptr->thing_word)); - if (ptr->thing_word == HEADER_PROC_BIN) { + switch (ptr->thing_word) { + case HEADER_PROC_BIN: { int to_new_heap = !ErtsInArea(ptr, oheap, oheap_sz); ASSERT(to_new_heap == !seen_mature || (!to_new_heap && (seen_mature=1))); if (to_new_heap) { @@ -2841,13 +2852,28 @@ sweep_off_heap(Process *p, int fullsweep) BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/ } link_live_proc_bin(&shrink, &prev, &ptr, to_new_heap); - } - else { + break; + } + case ERTS_USED_MAGIC_REF_THING_HEADER__: { + Uint size; + int to_new_heap = !ErtsInArea(ptr, oheap, oheap_sz); + ASSERT(is_magic_ref_thing(ptr)); + ASSERT(to_new_heap == !seen_mature || (!to_new_heap && (seen_mature=1))); + size = (Uint) ((ErtsMRefThing *) ptr)->mb->orig_size; + if (to_new_heap) + bin_vheap += size / sizeof(Eterm); + else + BIN_OLD_VHEAP(p) += size / sizeof(Eterm); /* for binary gc (words)*/ + /* fall through... */ + } + default: prev = &ptr->next; ptr = ptr->next; } } - else if (!ErtsInArea(ptr, oheap, oheap_sz)) { + else if (ErtsInArea(ptr, oheap, oheap_sz)) + break; /* and let old-heap loop continue */ + else { /* garbage */ switch (thing_subtag(ptr->thing_word)) { case REFC_BINARY_SUBTAG: @@ -2881,7 +2907,6 @@ sweep_off_heap(Process *p, int fullsweep) } *prev = ptr = ptr->next; } - else break; /* and let old-heap loop continue */ } /* The rest of the list resides on old-heap, and we just did a @@ -2890,16 +2915,24 @@ sweep_off_heap(Process *p, int fullsweep) while (ptr) { ASSERT(ErtsInArea(ptr, oheap, oheap_sz)); ASSERT(!IS_MOVED_BOXED(ptr->thing_word)); - if (ptr->thing_word == HEADER_PROC_BIN) { + switch (ptr->thing_word) { + case HEADER_PROC_BIN: BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/ link_live_proc_bin(&shrink, &prev, &ptr, 0); - } - else { - ASSERT(is_fun_header(ptr->thing_word) || - is_external_header(ptr->thing_word) - || is_magic_ref_thing(ptr)); - prev = &ptr->next; - ptr = ptr->next; + break; + case ERTS_USED_MAGIC_REF_THING_HEADER__: + ASSERT(is_magic_ref_thing(ptr)); + BIN_OLD_VHEAP(p) += + (((Uint) ((ErtsMRefThing *) ptr)->mb->orig_size) + / sizeof(Eterm)); /* for binary gc (words)*/ + /* fall through... */ + default: + ASSERT(is_fun_header(ptr->thing_word) || + is_external_header(ptr->thing_word) + || is_magic_ref_thing(ptr)); + prev = &ptr->next; + ptr = ptr->next; + break; } } diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 60df7b8a08..f9cba0aec0 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -3510,6 +3510,8 @@ dec_term_atom_common: ASSERT(rtp); hp = (Eterm *) rtp; write_magic_ref_thing(hp, factory->off_heap, mb); + OH_OVERHEAD(factory->off_heap, + mb->orig_size / sizeof(Eterm)); hp += ERTS_MAGIC_REF_THING_SIZE; } } diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index fc459e17ad..092a5320ba 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -3592,8 +3592,10 @@ store_external_or_ref_(Uint **hpp, ErlOffHeap* oh, Eterm ns) return make_internal_ref(to_hp); else { ErtsMRefThing *mreft = (ErtsMRefThing *) from_hp; + ErtsMagicBinary *mb = mreft->mb; ASSERT(is_magic_ref_thing(from_hp)); - erts_refc_inc(&mreft->mb->refc, 2); + erts_refc_inc(&mb->refc, 2); + OH_OVERHEAD(oh, mb->orig_size / sizeof(Eterm)); } ohhp = (struct erl_off_heap_header*) to_hp; -- cgit v1.2.3 From 75fdce43ef567668bb89508b9b8ce0df7efaa569 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 6 Feb 2017 17:15:52 +0100 Subject: erts: Add enif_monitor_process and enif_demonitor_process --- erts/emulator/beam/bif.c | 4 +- erts/emulator/beam/break.c | 24 +- erts/emulator/beam/dist.c | 25 +- erts/emulator/beam/erl_bif_info.c | 90 +++++--- erts/emulator/beam/erl_bits.c | 3 - erts/emulator/beam/erl_driver.h | 7 - erts/emulator/beam/erl_drv_nif.h | 8 + erts/emulator/beam/erl_lock_check.c | 3 + erts/emulator/beam/erl_monitors.c | 47 ++-- erts/emulator/beam/erl_monitors.h | 19 +- erts/emulator/beam/erl_nif.c | 402 +++++++++++++++++++++++++++++++-- erts/emulator/beam/erl_nif.h | 22 +- erts/emulator/beam/erl_nif_api_funcs.h | 4 + erts/emulator/beam/erl_node_tables.c | 4 +- erts/emulator/beam/erl_process.c | 50 ++-- erts/emulator/beam/erl_time_sup.c | 2 +- erts/emulator/beam/global.h | 34 ++- erts/emulator/beam/io.c | 20 +- 18 files changed, 607 insertions(+), 161 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 65c370c55b..51057c3ebb 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -361,7 +361,7 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to) c_p->common.id, (mon->name != NIL ? mon->name - : mon->pid), + : mon->u.pid), ref, 0); res = (code == ERTS_DSIG_SEND_YIELD ? am_yield : am_true); @@ -498,7 +498,7 @@ BIF_RETTYPE demonitor(Process *c_p, Eterm ref, Eterm *multip) res = am_true; break; case MON_ORIGIN: - to = mon->pid; + to = mon->u.pid; *multip = am_false; if (is_atom(to)) { /* Monitoring a name at node to */ diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 6e1e94b95b..5509c50a8e 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -178,19 +178,29 @@ static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext) prefix = ""; } - if (mon->type == MON_ORIGIN) { - if (is_atom(mon->pid)) { /* dist by name */ - ASSERT(is_node_name_atom(mon->pid)); + switch (mon->type) { + case MON_ORIGIN: + if (is_atom(mon->u.pid)) { /* dist by name */ + ASSERT(is_node_name_atom(mon->u.pid)); erts_print(to, to_arg, "%s{to,{%T,%T},%T}", prefix, mon->name, - mon->pid, mon->ref); + mon->u.pid, mon->ref); } else if (is_atom(mon->name)){ /* local by name */ erts_print(to, to_arg, "%s{to,{%T,%T},%T}", prefix, mon->name, erts_this_dist_entry->sysname, mon->ref); } else { /* local and distributed by pid */ - erts_print(to, to_arg, "%s{to,%T,%T}", prefix, mon->pid, mon->ref); + erts_print(to, to_arg, "%s{to,%T,%T}", prefix, mon->u.pid, mon->ref); } - } else { /* MON_TARGET */ - erts_print(to, to_arg, "%s{from,%T,%T}", prefix, mon->pid, mon->ref); + break; + case MON_TARGET: + erts_print(to, to_arg, "%s{from,%T,%T}", prefix, mon->u.pid, mon->ref); + break; + case MON_NIF_TARGET: { + ErtsResource* rsrc = mon->u.resource; + /*SVERK: Print resource-ref? */ + erts_print(to, to_arg, "%s{from,{%T,%T},%T}", prefix, rsrc->type->module, + rsrc->type->name, mon->ref); + break; + } } } diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 9e8f853279..016abaf717 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -259,7 +259,7 @@ static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp) DistEntry *dep = ((NetExitsContext *) vnecp)->dep; ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK; - rp = erts_pid2proc(NULL, 0, mon->pid, rp_locks); + rp = erts_pid2proc(NULL, 0, mon->u.pid, rp_locks); if (!rp) goto done; @@ -278,10 +278,11 @@ static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp) rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref); /* ASSERT(rmon != NULL); can happen during process exit */ if (rmon != NULL) { + ASSERT(rmon->type == MON_ORIGIN); ASSERT(is_atom(rmon->name) || is_nil(rmon->name)); watched = (is_atom(rmon->name) ? TUPLE2(lhp, rmon->name, dep->sysname) - : rmon->pid); + : rmon->u.pid); #ifdef ERTS_SMP rp_locks |= ERTS_PROC_LOCKS_MSG_SEND; erts_smp_proc_lock(rp, ERTS_PROC_LOCKS_MSG_SEND); @@ -1391,7 +1392,7 @@ int erts_net_message(Port *prt, if (mon == NULL) { break; } - watched = mon->pid; + watched = mon->u.pid; erts_destroy_monitor(mon); rp = erts_pid2proc_opt(NULL, 0, watched, ERTS_PROC_LOCK_LINK, @@ -1549,7 +1550,7 @@ int erts_net_message(Port *prt, if (mon == NULL) { break; } - rp = erts_pid2proc(NULL, 0, mon->pid, rp_locks); + rp = erts_pid2proc(NULL, 0, mon->u.pid, rp_locks); erts_destroy_monitor(mon); if (rp == NULL) { @@ -1566,7 +1567,7 @@ int erts_net_message(Port *prt, watched = (is_not_nil(mon->name) ? TUPLE2(&lhp[0], mon->name, sysname) - : mon->pid); + : mon->u.pid); erts_queue_monitor_message(rp, &rp_locks, ref, am_process, watched, reason); @@ -2415,21 +2416,21 @@ static void doit_print_monitor_info(ErtsMonitor *mon, void *vptdp) void *arg = ((struct print_to_data *) vptdp)->arg; Process *rp; ErtsMonitor *rmon; - rp = erts_proc_lookup(mon->pid); + rp = erts_proc_lookup(mon->u.pid); if (!rp || (rmon = erts_lookup_monitor(ERTS_P_MONITORS(rp), mon->ref)) == NULL) { - erts_print(to, arg, "Warning, stray monitor for: %T\n", mon->pid); + erts_print(to, arg, "Warning, stray monitor for: %T\n", mon->u.pid); } else if (mon->type == MON_ORIGIN) { /* Local pid is being monitored */ erts_print(to, arg, "Remotely monitored by: %T %T\n", - mon->pid, rmon->pid); + mon->u.pid, rmon->u.pid); } else { - erts_print(to, arg, "Remote monitoring: %T ", mon->pid); - if (is_not_atom(rmon->pid)) - erts_print(to, arg, "%T\n", rmon->pid); + erts_print(to, arg, "Remote monitoring: %T ", mon->u.pid); + if (is_not_atom(rmon->u.pid)) + erts_print(to, arg, "%T\n", rmon->u.pid); else erts_print(to, arg, "{%T, %T}\n", rmon->name, - rmon->pid); /* which in this case is the + rmon->u.pid); /* which in this case is the remote system name... */ } } diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 5ee19aead8..9f32e0e7bd 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -202,8 +202,10 @@ bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh) static void do_calc_mon_size(ErtsMonitor *mon, void *vpsz) { Uint *psz = vpsz; - *psz += IS_CONST(mon->ref) ? 0 : NC_HEAP_SIZE(mon->ref); - *psz += IS_CONST(mon->pid) ? 0 : NC_HEAP_SIZE(mon->pid); + *psz += NC_HEAP_SIZE(mon->ref); + *psz += (mon->type == MON_NIF_TARGET ? + erts_resource_ref_size(mon->u.resource) : + (is_immed(mon->u.pid) ? 0 : NC_HEAP_SIZE(mon->u.pid))); *psz += 8; /* CONS + 5-tuple */ } @@ -218,12 +220,11 @@ static void do_make_one_mon_element(ErtsMonitor *mon, void * vpmlc) { MonListContext *pmlc = vpmlc; Eterm tup; - Eterm r = (IS_CONST(mon->ref) - ? mon->ref - : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->ref)); - Eterm p = (IS_CONST(mon->pid) - ? mon->pid - : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->pid)); + Eterm r = STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->ref); + Eterm p = (mon->type == MON_NIF_TARGET ? + erts_bld_resource_ref(&(pmlc->hp), &MSO(pmlc->p), mon->u.resource) + : (is_immed(mon->u.pid) ? mon->u.pid + : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->u.pid))); tup = TUPLE5(pmlc->hp, pmlc->tag, make_small(mon->type), r, p, mon->name); pmlc->hp += 6; pmlc->res = CONS(pmlc->hp, tup, pmlc->res); @@ -262,7 +263,7 @@ make_monitor_list(Process *p, ErtsMonitor *root) static void do_calc_lnk_size(ErtsLink *lnk, void *vpsz) { Uint *psz = vpsz; - *psz += IS_CONST(lnk->pid) ? 0 : NC_HEAP_SIZE(lnk->pid); + *psz += is_immed(lnk->pid) ? 0 : NC_HEAP_SIZE(lnk->pid); if (lnk->type != LINK_NODE && ERTS_LINK_ROOT(lnk) != NULL) { /* Node links use this pointer as ref counter... */ erts_doforall_links(ERTS_LINK_ROOT(lnk),&do_calc_lnk_size,vpsz); @@ -282,7 +283,7 @@ static void do_make_one_lnk_element(ErtsLink *lnk, void * vpllc) LnkListContext *pllc = vpllc; Eterm tup; Eterm old_res, targets = NIL; - Eterm p = (IS_CONST(lnk->pid) + Eterm p = (is_immed(lnk->pid) ? lnk->pid : STORE_NC(&(pllc->hp), &MSO(pllc->p), lnk->pid)); if (lnk->type == LINK_NODE) { @@ -366,8 +367,12 @@ erts_print_system_version(fmtfn_t to, void *arg, Process *c_p) typedef struct { /* {Entity,Node} = {monitor.Name,monitor.Pid} for external by name * {Entity,Node} = {monitor.Pid,NIL} for external/external by pid - * {Entity,Node} = {monitor.Name,erlang:node()} for internal by name */ - Eterm entity; + * {Entity,Node} = {monitor.Name,erlang:node()} for internal by name + * {Entity,Node} = {monitor.resource,MON_NIF_TARGET}*/ + union { + Eterm term; + ErtsResource* resource; + }entity; Eterm node; /* pid is actual target being monitored, no matter pid/port or name */ Eterm pid; @@ -413,7 +418,7 @@ static void collect_one_link(ErtsLink *lnk, void *vmicp) if (!(lnk->type == LINK_PID)) { return; } - micp->mi[micp->mi_i].entity = lnk->pid; + micp->mi[micp->mi_i].entity.term = lnk->pid; micp->sz += 2 + NC_HEAP_SIZE(lnk->pid); micp->mi_i++; } @@ -426,20 +431,20 @@ static void collect_one_origin_monitor(ErtsMonitor *mon, void *vmicp) return; } EXTEND_MONITOR_INFOS(micp); - if (is_atom(mon->pid)) { /* external by name */ - micp->mi[micp->mi_i].entity = mon->name; - micp->mi[micp->mi_i].node = mon->pid; + if (is_atom(mon->u.pid)) { /* external by name */ + micp->mi[micp->mi_i].entity.term = mon->name; + micp->mi[micp->mi_i].node = mon->u.pid; micp->sz += 3; /* need one 2-tuple */ - } else if (is_external_pid(mon->pid)) { /* external by pid */ - micp->mi[micp->mi_i].entity = mon->pid; + } else if (is_external_pid(mon->u.pid)) { /* external by pid */ + micp->mi[micp->mi_i].entity.term = mon->u.pid; micp->mi[micp->mi_i].node = NIL; - micp->sz += NC_HEAP_SIZE(mon->pid); + micp->sz += NC_HEAP_SIZE(mon->u.pid); } else if (!is_nil(mon->name)) { /* internal by name */ - micp->mi[micp->mi_i].entity = mon->name; + micp->mi[micp->mi_i].entity.term = mon->name; micp->mi[micp->mi_i].node = erts_this_dist_entry->sysname; micp->sz += 3; /* need one 2-tuple */ } else { /* internal by pid */ - micp->mi[micp->mi_i].entity = mon->pid; + micp->mi[micp->mi_i].entity.term = mon->u.pid; micp->mi[micp->mi_i].node = NIL; /* no additional heap space needed */ } @@ -447,7 +452,7 @@ static void collect_one_origin_monitor(ErtsMonitor *mon, void *vmicp) /* have always pid at hand, to assist with figuring out if its a port or * a process, when we monitored by name and process_info is requested. * See: erl_bif_info.c:process_info_aux section for am_monitors */ - micp->mi[micp->mi_i].pid = mon->pid; + micp->mi[micp->mi_i].pid = mon->u.pid; micp->mi_i++; micp->sz += 2 + 3; /* For a cons cell and a 2-tuple */ @@ -457,15 +462,24 @@ static void collect_one_target_monitor(ErtsMonitor *mon, void *vmicp) { MonitorInfoCollection *micp = vmicp; - if (mon->type != MON_TARGET) { - return; + if (mon->type != MON_TARGET && mon->type != MON_NIF_TARGET) { + return; } EXTEND_MONITOR_INFOS(micp); - micp->mi[micp->mi_i].node = NIL; - micp->mi[micp->mi_i].entity = mon->pid; - micp->sz += (NC_HEAP_SIZE(mon->pid) + 2 /* cons */); + + if (mon->type == MON_NIF_TARGET) { + micp->mi[micp->mi_i].entity.resource = mon->u.resource; + micp->mi[micp->mi_i].node = make_small(MON_NIF_TARGET); + micp->sz += erts_resource_ref_size(mon->u.resource); + } + else { + micp->mi[micp->mi_i].entity.term = mon->u.pid; + micp->mi[micp->mi_i].node = NIL; + micp->sz += NC_HEAP_SIZE(mon->u.pid); + } + micp->sz += 2; /* cons */; micp->mi_i++; } @@ -1194,7 +1208,7 @@ process_info_aux(Process *BIF_P, hp = HAlloc(BIF_P, 3 + mic.sz); res = NIL; for (i = 0; i < mic.mi_i; i++) { - item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); + item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity.term); res = CONS(hp, item, res); hp += 2; } @@ -1212,7 +1226,7 @@ process_info_aux(Process *BIF_P, hp = HAlloc(BIF_P, 3 + mic.sz); res = NIL; for (i = 0; i < mic.mi_i; i++) { - if (is_atom(mic.mi[i].entity)) { + if (is_atom(mic.mi[i].entity.term)) { /* Monitor by name. * Build {process|port, {Name, Node}} and cons it. */ @@ -1224,7 +1238,7 @@ process_info_aux(Process *BIF_P, || is_port(mic.mi[i].pid) || is_atom(mic.mi[i].pid)); - t1 = TUPLE2(hp, mic.mi[i].entity, mic.mi[i].node); + t1 = TUPLE2(hp, mic.mi[i].entity.term, mic.mi[i].node); hp += 3; t2 = TUPLE2(hp, m_type, t1); hp += 3; @@ -1234,7 +1248,7 @@ process_info_aux(Process *BIF_P, else { /* Monitor by pid. Build {process|port, Pid} and cons it. */ Eterm t; - Eterm pid = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); + Eterm pid = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity.term); Eterm m_type = is_port(mic.mi[i].pid) ? am_port : am_process; ASSERT(is_pid(mic.mi[i].pid) @@ -1261,7 +1275,12 @@ process_info_aux(Process *BIF_P, res = NIL; for (i = 0; i < mic.mi_i; ++i) { - item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); + if (mic.mi[i].node == make_small(MON_NIF_TARGET)) { + item = erts_bld_resource_ref(&hp, &MSO(BIF_P), mic.mi[i].entity.resource); + } + else { + item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity.term); + } res = CONS(hp, item, res); hp += 2; } @@ -2950,7 +2969,7 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt, if (hpp) { res = NIL; for (i = 0; i < mic.mi_i; i++) { - item = STORE_NC(hpp, ohp, mic.mi[i].entity); + item = STORE_NC(hpp, ohp, mic.mi[i].entity.term); res = CONS(*hpp, item, res); *hpp += 2; } @@ -2981,7 +3000,7 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt, Eterm t; Eterm m_type; - item = STORE_NC(hpp, ohp, mic.mi[i].entity); + item = STORE_NC(hpp, ohp, mic.mi[i].entity.term); m_type = is_port(item) ? am_port : am_process; t = TUPLE2(*hpp, m_type, item); *hpp += 3; @@ -3010,7 +3029,8 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt, if (hpp) { res = NIL; for (i = 0; i < mic.mi_i; ++i) { - item = STORE_NC(hpp, ohp, mic.mi[i].entity); + ASSERT(mic.mi[i].node == NIL); + item = STORE_NC(hpp, ohp, mic.mi[i].entity.term); res = CONS(*hpp, item, res); *hpp += 2; } diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index 6bf52fb303..885e955332 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -110,9 +110,6 @@ erts_init_bits(void) { ERTS_CT_ASSERT(offsetof(Binary,orig_bytes) % 8 == 0); ERTS_CT_ASSERT(offsetof(ErtsMagicBinary,u.aligned.data) % 8 == 0); - ERTS_CT_ASSERT(ERTS_MAGIC_BIN_BYTES_TO_ALIGN == - (offsetof(ErtsMagicBinary,u.aligned.data) - - offsetof(ErtsMagicBinary,u.unaligned.data))); ERTS_CT_ASSERT(offsetof(ErtsBinary,driver.binary.orig_bytes) == offsetof(Binary,orig_bytes)); diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 5bea92e198..b386b68cff 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -175,13 +175,6 @@ struct erl_drv_event_data { #endif typedef struct erl_drv_event_data *ErlDrvEventData; /* Event data */ -/* - * A driver monitor - */ -typedef struct { - unsigned char data[sizeof(void *)*4]; -} ErlDrvMonitor; - typedef struct { unsigned long megasecs; unsigned long secs; diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h index 46bb06d642..8de4a7855d 100644 --- a/erts/emulator/beam/erl_drv_nif.h +++ b/erts/emulator/beam/erl_drv_nif.h @@ -64,6 +64,14 @@ enum ErlNifSelectReturn { ERL_NIF_SELECT_FAILED = (1 << 4) }; +/* + * A driver monitor + */ +typedef struct { + unsigned char data[sizeof(void *)*4]; +} ErlDrvMonitor; + + #ifdef SIZEOF_CHAR # define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR # undef SIZEOF_CHAR diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 08dcbed91c..0798faf53a 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -89,6 +89,9 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "hipe_mfait_lock", NULL }, #endif { "nodes_monitors", NULL }, +#ifdef ERTS_SMP + { "resource_monitors", "address" }, +#endif { "driver_list", NULL }, { "proc_link", "pid" }, { "proc_msgq", "pid" }, diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c index 910598690d..e5be66be79 100644 --- a/erts/emulator/beam/erl_monitors.c +++ b/erts/emulator/beam/erl_monitors.c @@ -91,7 +91,7 @@ static ERTS_INLINE int cmp_mon_ref(Eterm ref1, Eterm ref2) #define CP_LINK_VAL(To, Hp, From) \ do { \ - if (IS_CONST(From)) \ + if (is_immed(From)) \ (To) = (From); \ else { \ Uint i__; \ @@ -109,15 +109,15 @@ do { \ } \ } while (0) -static ErtsMonitor *create_monitor(Uint type, Eterm ref, Eterm pid, Eterm name) +static ErtsMonitor *create_monitor(Uint type, Eterm ref, UWord entity, Eterm name) { Uint mon_size = ERTS_MONITOR_SIZE; ErtsMonitor *n; Eterm *hp; mon_size += NC_HEAP_SIZE(ref); - if (!IS_CONST(pid)) { - mon_size += NC_HEAP_SIZE(pid); + if (type != MON_NIF_TARGET && is_not_immed(entity)) { + mon_size += NC_HEAP_SIZE(entity); } if (mon_size <= ERTS_MONITOR_SH_SIZE) { @@ -136,7 +136,10 @@ static ErtsMonitor *create_monitor(Uint type, Eterm ref, Eterm pid, Eterm name) n->balance = 0; /* Always the same initial value */ n->name = name; /* atom() or [] */ CP_LINK_VAL(n->ref, hp, ref); /*XXX Unneccesary check, never immediate*/ - CP_LINK_VAL(n->pid, hp, pid); + if (type == MON_NIF_TARGET) + n->u.resource = (ErtsResource*)entity; + else + CP_LINK_VAL(n->u.pid, hp, (Eterm)entity); return n; } @@ -147,7 +150,7 @@ static ErtsLink *create_link(Uint type, Eterm pid) ErtsLink *n; Eterm *hp; - if (!IS_CONST(pid)) { + if (is_not_immed(pid)) { lnk_size += NC_HEAP_SIZE(pid); } @@ -206,16 +209,16 @@ void erts_destroy_monitor(ErtsMonitor *mon) Uint mon_size = ERTS_MONITOR_SIZE; ErlNode *node; - ASSERT(!IS_CONST(mon->ref)); + ASSERT(is_not_immed(mon->ref)); mon_size += NC_HEAP_SIZE(mon->ref); if (is_external(mon->ref)) { node = external_thing_ptr(mon->ref)->node; erts_deref_node_entry(node); } - if (!IS_CONST(mon->pid)) { - mon_size += NC_HEAP_SIZE(mon->pid); - if (is_external(mon->pid)) { - node = external_thing_ptr(mon->pid)->node; + if (mon->type != MON_NIF_TARGET && is_not_immed(mon->u.pid)) { + mon_size += NC_HEAP_SIZE(mon->u.pid); + if (is_external(mon->u.pid)) { + node = external_thing_ptr(mon->u.pid)->node; erts_deref_node_entry(node); } } @@ -234,7 +237,7 @@ void erts_destroy_link(ErtsLink *lnk) ASSERT(lnk->type == LINK_NODE || ERTS_LINK_ROOT(lnk) == NULL); - if (!IS_CONST(lnk->pid)) { + if (is_not_immed(lnk->pid)) { lnk_size += NC_HEAP_SIZE(lnk->pid); if (is_external(lnk->pid)) { node = external_thing_ptr(lnk->pid)->node; @@ -329,7 +332,7 @@ static void insertion_rotation(int dstack[], int dpos, } } -void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, Eterm pid, +void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, UWord entity, Eterm name) { void *tstack[STACK_NEED]; @@ -344,7 +347,7 @@ void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, Eterm pid, for (;;) { if (!*this) { /* Found our place */ state = 1; - *this = create_monitor(type,ref,pid,name); + *this = create_monitor(type,ref,entity,name); break; } else if ((c = CMP_MON_REF(ref,(*this)->ref)) < 0) { /* go left */ @@ -914,8 +917,12 @@ static void erts_dump_monitors(ErtsMonitor *root, int indent) if (root == NULL) return; erts_dump_monitors(root->right,indent+2); - erts_printf("%*s[%b16d:%b16u:%T:%T:%T]\n", indent, "", root->balance, - root->type, root->ref, root->pid, root->name); + erts_printf("%*s[%b16d:%b16u:%T:%T", indent, "", root->balance, + root->type, root->ref, root->name); + if (root->type == MON_NIF_TARGET) + erts_printf(":%p]\n", root->u.resource); + else + erts_printf(":%T]\n", root->u.pid); erts_dump_monitors(root->left,indent+2); } @@ -1030,7 +1037,7 @@ void erts_one_link_size(ErtsLink *lnk, void *vpu) { Uint *pu = vpu; *pu += ERTS_LINK_SIZE*sizeof(Uint); - if(!IS_CONST(lnk->pid)) + if(is_not_immed(lnk->pid)) *pu += NC_HEAP_SIZE(lnk->pid)*sizeof(Uint); if (lnk->type != LINK_NODE && ERTS_LINK_ROOT(lnk) != NULL) { erts_doforall_links(ERTS_LINK_ROOT(lnk),&erts_one_link_size,vpu); @@ -1040,8 +1047,8 @@ void erts_one_mon_size(ErtsMonitor *mon, void *vpu) { Uint *pu = vpu; *pu += ERTS_MONITOR_SIZE*sizeof(Uint); - if(!IS_CONST(mon->pid)) - *pu += NC_HEAP_SIZE(mon->pid)*sizeof(Uint); - if(!IS_CONST(mon->ref)) + if(mon->type != MON_NIF_TARGET && is_not_immed(mon->u.pid)) + *pu += NC_HEAP_SIZE(mon->u.pid)*sizeof(Uint); + if(is_not_immed(mon->ref)) *pu += NC_HEAP_SIZE(mon->ref)*sizeof(Uint); } diff --git a/erts/emulator/beam/erl_monitors.h b/erts/emulator/beam/erl_monitors.h index 9e2beedea3..f90ac930e5 100644 --- a/erts/emulator/beam/erl_monitors.h +++ b/erts/emulator/beam/erl_monitors.h @@ -82,8 +82,9 @@ /* Type tags for monitors */ #define MON_ORIGIN 1 -#define MON_TARGET 3 -#define MON_TIME_OFFSET 7 +#define MON_TARGET 2 +#define MON_NIF_TARGET 3 +#define MON_TIME_OFFSET 4 /* Type tags for links */ #define LINK_PID 1 /* ...Or port */ @@ -105,11 +106,15 @@ typedef struct erts_monitor_or_link { typedef struct erts_monitor { struct erts_monitor *left, *right; Sint16 balance; - Uint16 type; /* MON_ORIGIN | MON_TARGET | MON_TIME_OFFSET */ + Uint16 type; /* MON_ORIGIN | MON_TARGET | MON_NIF_TARGET | MON_TIME_OFFSET */ Eterm ref; - Eterm pid; /* In case of distributed named monitor, this is the - nodename atom in MON_ORIGIN process, otherwise a pid or - , in case of a MON_TARGET, a port */ + union { + Eterm pid; /* In case of distributed named monitor, this is the + * nodename atom in MON_ORIGIN process, otherwise a pid or, + * in case of a MON_TARGET, a port + */ + struct ErtsResource_* resource; /* MON_NIF_TARGET */ + }u; Eterm name; /* When monitoring a named process: atom() else [] */ Uint heap[1]; /* Larger in reality */ } ErtsMonitor; @@ -144,7 +149,7 @@ Uint erts_tot_link_lh_size(void); /* Prototypes */ void erts_destroy_monitor(ErtsMonitor *mon); -void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, Eterm pid, +void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, UWord entity, Eterm name); ErtsMonitor *erts_remove_monitor(ErtsMonitor **root, Eterm ref); ErtsMonitor *erts_lookup_monitor(ErtsMonitor *root, Eterm ref); diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index b7425b9e45..62191c4abf 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1913,8 +1913,6 @@ int enif_snprintf(char *buffer, size_t size, const char* format, ...) /* dummy node in circular list */ struct enif_resource_type_t resource_type_list; -#define SIZEOF_ErlNifResource(SIZE) (offsetof(ErtsResource,data) + (SIZE)) - static ErlNifResourceType* find_resource_type(Eterm module, Eterm name) { ErlNifResourceType* type; @@ -2083,6 +2081,7 @@ static void commit_opened_resource_types(struct erl_module_nif* lib) type->owner = lib; type->dtor = ort->new_callbacks.dtor; type->stop = ort->new_callbacks.stop; + type->down = ort->new_callbacks.down; if (type->dtor != NULL) { erts_refc_inc(&lib->rt_dtor_cnt, 1); @@ -2108,12 +2107,148 @@ static void rollback_opened_resource_types(void) } } +struct destroy_monitor_ctx +{ + Binary* resource_bin; + int exiting_procs; + int scheduler; +}; + +static void destroy_one_monitor(ErtsMonitor* mon, void* context) +{ + struct destroy_monitor_ctx* ctx = (struct destroy_monitor_ctx*) context; + Process* rp; + ErtsMonitor *rmon = NULL; + int is_exiting; + + ASSERT(mon->type == MON_ORIGIN); + ASSERT(is_internal_pid(mon->u.pid)); + ASSERT(is_internal_ref(mon->ref)); + + if (ctx->scheduler > 0) { /* Normal scheduler */ + rp = erts_proc_lookup(mon->u.pid); + } + else { +#ifdef ERTS_SMP + rp = erts_proc_lookup_inc_refc(mon->u.pid); +#else + ASSERT(!"nif monitor destruction in non-scheduler thread"); + rp = NULL; +#endif + } + + if (!rp) { + is_exiting = 1; + } + if (rp) { + erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK); + if (ERTS_PROC_IS_EXITING(rp)) { + is_exiting = 1; + } else { + rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref); + ASSERT(rmon); + is_exiting = 0; + } + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); +#ifdef ERTS_SMP + if (ctx->scheduler <= 0) + erts_proc_dec_refc(rp); +#endif + } + if (is_exiting) { + ctx->exiting_procs++; + /* +1 for exiting process */ + erts_refc_inc(&ctx->resource_bin->refc, ctx->exiting_procs); + } + + /* ToDo: Delay destruction after monitor_locks */ + if (rmon) { + ASSERT(rmon->type == MON_NIF_TARGET); + ASSERT(&ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rmon->u.pid)->binary == ctx->resource_bin); + erts_destroy_monitor(rmon); + } + erts_destroy_monitor(mon); +} + +static int destroy_all_monitors(ErtsMonitor* monitors, Binary* bin) +{ + struct destroy_monitor_ctx ctx; + + execution_state(NULL, NULL, &ctx.scheduler); + + ctx.resource_bin = bin; + ctx.exiting_procs = 0; + erts_sweep_monitors(monitors, &destroy_one_monitor, &ctx); + return ctx.exiting_procs == 0; +} + + +#ifdef ERTS_SMP +# define NIF_RESOURCE_DTOR &nif_resource_dtor +#else +# define NIF_RESOURCE_DTOR &nosmp_nif_resource_dtor_prologue + +/* + * NO-SMP: Always run resource destructor on scheduler thread + * as we may have to remove process monitors. + */ +static int nif_resource_dtor(Binary*); + +static void nosmp_nif_resource_dtor_scheduled(void* vbin) +{ + erts_bin_free((Binary*)vbin); +} + +static int nosmp_nif_resource_dtor_prologue(Binary* bin) +{ + if (is_scheduler()) { + return nif_resource_dtor(bin); + } + else { + erts_schedule_misc_aux_work(1, nosmp_nif_resource_dtor_scheduled, bin); + return 0; /* do not free */ + } +} + +#endif /* !ERTS_SMP */ static int nif_resource_dtor(Binary* bin) { ErtsResource* resource = (ErtsResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(bin); ErlNifResourceType* type = resource->type; - ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == NIF_RESOURCE_DTOR); + + if (resource->monitors) { + ErtsResourceMonitors* rm = resource->monitors; + ErtsMonitor* root; + + ASSERT(type->down); + erts_smp_mtx_lock(&rm->lock); + ASSERT(erts_refc_read(&bin->refc, 0) == 0); + root = rm->root; + if (root) { + rm->root = NULL; + if (!destroy_all_monitors(root, bin)) { + /* + * Resource death struggle prolonged to serve exiting process(es). + * Destructor will be called again when last exiting process + * tries to fire its MON_NIF_TARGET monitor (and fails). + * + * This resource is doomed. It has no "real" references and + * should get not get called upon to do anything except the + * final destructor call. + */ + ASSERT(erts_refc_read(&bin->refc, 1)); +#ifdef DEBUG + resource->dbg_is_dying = 1; +#endif + erts_smp_mtx_unlock(&rm->lock); + return 0; + } + } + erts_smp_mtx_unlock(&rm->lock); + erts_smp_mtx_destroy(&rm->lock); + } if (type->dtor != NULL) { struct enif_msg_environment_t msg_env; @@ -2141,20 +2276,99 @@ void erts_resource_stop(ErtsResource* resource, ErlNifEvent e, post_nif_noproc(&msg_env); } -void* enif_alloc_resource(ErlNifResourceType* type, size_t size) +/* SVERK SVERK: Move to ?.h */ +void erts_ref_to_driver_monitor(Eterm ref, ErlDrvMonitor*); + +void erts_fire_nif_monitor(ErtsResource* resource, Eterm pid, Eterm ref) { - Binary* bin = erts_create_magic_binary_x(SIZEOF_ErlNifResource(size), - &nif_resource_dtor, - 1); /* unaligned */ - ErtsResource* resource = ERTS_MAGIC_BIN_UNALIGNED_DATA(bin); + ErtsMonitor* rmon; + ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); + struct enif_msg_environment_t msg_env; + ErlNifPid nif_pid; + ErlNifMonitor nif_monitor; + ErtsResourceMonitors* rmp = resource->monitors; + + ASSERT(rmp); + ASSERT(resource->type->down); + + erts_smp_mtx_lock(&rmp->lock); + rmon = erts_remove_monitor(&rmp->root, ref); + if (!rmon) { + erts_smp_mtx_unlock(&rmp->lock); + /* -1 for exiting process */ + if (erts_refc_dectest(&bin->binary.refc, 0) == 0) { + erts_bin_free(&bin->binary); + } + return; + } + ASSERT(!resource->dbg_is_dying); + if (erts_refc_inctest(&bin->binary.refc, 1) < 2) { + /* + * Racing resource destruction. + * To avoid a more complex refc-dance with destructing thread + * we avoid calling 'down' and just silently remove the monitor. + * There are no real references left to this resource and we have + * monitors_lock, so it's safe to reset refc back to zero. + * This can happen even for non smp as destructor calls may be scheduled. + */ + erts_refc_init(&bin->binary.refc, 0); + erts_smp_mtx_unlock(&rmp->lock); + } + else { + erts_smp_mtx_unlock(&rmp->lock); + + ASSERT(rmon->u.pid == pid); + erts_ref_to_driver_monitor(ref, &nif_monitor); + nif_pid.pid = pid; + pre_nif_noproc(&msg_env, resource->type->owner, NULL); + resource->type->down(&msg_env.env, resource->data, &nif_pid, &nif_monitor); + post_nif_noproc(&msg_env); + + if (erts_refc_dectest(&bin->binary.refc, 0) == 0) { + erts_bin_free(&bin->binary); + } + } + erts_destroy_monitor(rmon); +} + +void* enif_alloc_resource(ErlNifResourceType* type, size_t data_sz) +{ + size_t magic_sz = offsetof(ErtsResource,data); + Binary* bin; + ErtsResource* resource; + size_t monitors_offs; + + if (type->down) { + /* Put ErtsResourceMonitors after user data and properly aligned */ + monitors_offs = ((data_sz + ERTS_ALLOC_ALIGN_BYTES - 1) + & ~((size_t)ERTS_ALLOC_ALIGN_BYTES - 1)); + magic_sz += monitors_offs + sizeof(ErtsResourceMonitors); + } + else { + ERTS_UNDEF(monitors_offs, 0); + magic_sz += data_sz; + } + bin = erts_create_magic_binary_x(magic_sz, NIF_RESOURCE_DTOR, + 1); /* unaligned */ + resource = ERTS_MAGIC_BIN_UNALIGNED_DATA(bin); ASSERT(type->owner && type->next && type->prev); /* not allowed in load/upgrade */ resource->type = type; erts_refc_inc(&bin->refc, 1); #ifdef DEBUG erts_refc_init(&resource->nif_refc, 1); + resource->dbg_is_dying = 0; #endif erts_refc_inc(&resource->type->refc, 2); + if (type->down) { + resource->monitors = (ErtsResourceMonitors*) (resource->data + monitors_offs); + erts_smp_mtx_init(&resource->monitors->lock, "resource_monitors"); + resource->monitors->root = NULL; + resource->monitors->user_data_sz = data_sz; + } + else { + resource->monitors = NULL; + } return resource->data; } @@ -2163,7 +2377,8 @@ void enif_release_resource(void* obj) ErtsResource* resource = DATA_TO_RESOURCE(obj); ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); - ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == NIF_RESOURCE_DTOR); + ASSERT(!resource->dbg_is_dying); #ifdef DEBUG erts_refc_dec(&resource->nif_refc, 0); #endif @@ -2177,18 +2392,27 @@ void enif_keep_resource(void* obj) ErtsResource* resource = DATA_TO_RESOURCE(obj); ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); - ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == NIF_RESOURCE_DTOR); + ASSERT(!resource->dbg_is_dying); #ifdef DEBUG erts_refc_inc(&resource->nif_refc, 1); #endif erts_refc_inc(&bin->binary.refc, 2); } +Eterm erts_bld_resource_ref(Eterm** hpp, ErlOffHeap* oh, ErtsResource* resource) +{ + ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); + ASSERT(!resource->dbg_is_dying); + return erts_mk_magic_binary_term(hpp, oh, &bin->binary); +} + ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj) { ErtsResource* resource = DATA_TO_RESOURCE(obj); ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); Eterm* hp = alloc_heap(env,PROC_BIN_SIZE); + ASSERT(!resource->dbg_is_dying); return erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary); } @@ -2217,7 +2441,7 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ }*/ mbin = pb->val; resource = (ErtsResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(mbin); - if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor + if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != NIF_RESOURCE_DTOR || resource->type != type) { return 0; } @@ -2228,8 +2452,13 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ size_t enif_sizeof_resource(void* obj) { ErtsResource* resource = DATA_TO_RESOURCE(obj); - Binary* bin = &ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource)->binary; - return ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(bin) - offsetof(ErtsResource,data); + if (resource->monitors) { + return resource->monitors->user_data_sz; + } + else { + Binary* bin = &ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource)->binary; + return ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(bin) - offsetof(ErtsResource,data); + } } @@ -2830,6 +3059,150 @@ int enif_map_iterator_get_pair(ErlNifEnv *env, return 0; } +int enif_monitor_process(ErlNifEnv* env, void* obj, const ErlNifPid* target_pid, + ErlNifMonitor* monitor) +{ + int scheduler; + ErtsResource* rsrc = DATA_TO_RESOURCE(obj); + Process *rp; + Eterm tmp[REF_THING_SIZE]; + Eterm ref; + int retval; + + ASSERT(ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rsrc)->magic_binary.destructor + == NIF_RESOURCE_DTOR); + ASSERT(!rsrc->dbg_is_dying); + ASSERT(!rsrc->monitors == !rsrc->type->down); + + + if (!rsrc->monitors) { + ASSERT(!rsrc->type->down); + return -1; + } + ASSERT(rsrc->type->down); + + execution_state(env, NULL, &scheduler); + +#ifdef ERTS_SMP + if (scheduler > 0) /* Normal scheduler */ + rp = erts_proc_lookup_raw(target_pid->pid); + else + rp = erts_proc_lookup_raw_inc_refc(target_pid->pid); +#else + if (scheduler <= 0) { + erts_exit(ERTS_ABORT_EXIT, "enif_monitor_process: called from " + "non-scheduler thread on non-SMP VM"); + } + rp = erts_proc_lookup(target_pid->pid); +#endif + + if (!rp) + return 1; + + ref = erts_make_ref_in_buffer(tmp); + + erts_smp_mtx_lock(&rsrc->monitors->lock); + erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK); + if (ERTS_PSFLG_FREE & erts_smp_atomic32_read_nob(&rp->state)) { + retval = 1; + } + else { + erts_add_monitor(&rsrc->monitors->root, MON_ORIGIN, ref, rp->common.id, NIL); + erts_add_monitor(&ERTS_P_MONITORS(rp), MON_NIF_TARGET, ref, (UWord)rsrc, NIL); + retval = 0; + } + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_smp_mtx_unlock(&rsrc->monitors->lock); + +#ifdef ERTS_SMP + if (scheduler <= 0) + erts_proc_dec_refc(rp); +#endif + if (monitor) + erts_ref_to_driver_monitor(ref,monitor); + + return retval; +} + +int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monitor) +{ + int scheduler; + ErtsResource* rsrc = DATA_TO_RESOURCE(obj); + ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rsrc); + Process *rp; + ErtsMonitor *mon; + ErtsMonitor *rmon = NULL; + Eterm ref_heap[REF_THING_SIZE]; + Eterm ref; + int is_exiting; + + ASSERT(bin->magic_binary.destructor == NIF_RESOURCE_DTOR); + ASSERT(!rsrc->dbg_is_dying); + + execution_state(env, NULL, &scheduler); + + memcpy(ref_heap, monitor, sizeof(Eterm)*REF_THING_SIZE); + ref = make_internal_ref(ref_heap); + erts_smp_mtx_lock(&rsrc->monitors->lock); + mon = erts_remove_monitor(&rsrc->monitors->root, ref); + + if (mon == NULL) { + erts_smp_mtx_unlock(&rsrc->monitors->lock); + return 1; + } + + ASSERT(mon->type == MON_ORIGIN); + ASSERT(is_internal_pid(mon->u.pid)); + +#ifdef ERTS_SMP + if (scheduler > 0) /* Normal scheduler */ + rp = erts_proc_lookup(mon->u.pid); + else + rp = erts_proc_lookup_inc_refc(mon->u.pid); +#else + if (scheduler <= 0) { + erts_exit(ERTS_ABORT_EXIT, "enif_demonitor_process: called from " + "non-scheduler thread on non-SMP VM"); + } + rp = erts_proc_lookup(mon->u.pid); +#endif + + if (!rp) { + is_exiting = 1; + } + else { + erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK); + if (ERTS_PROC_IS_EXITING(rp)) { + is_exiting = 1; + } else { + rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref); + ASSERT(rmon); + is_exiting = 0; + } + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + +#ifdef ERTS_SMP + if (scheduler <= 0) + erts_proc_dec_refc(rp); +#endif + } + if (is_exiting) { + /* +1 for exiting process */ + erts_refc_inc(&bin->binary.refc, 2); + } + erts_smp_mtx_unlock(&rsrc->monitors->lock); + + if (rmon) { + ASSERT(rmon->type == MON_NIF_TARGET); + ASSERT(rmon->u.resource == rsrc); + erts_destroy_monitor(rmon); + } + erts_destroy_monitor(mon); + + return 0; +} + + /*************************************************************************** ** load_nif/2 ** ***************************************************************************/ @@ -3303,7 +3676,8 @@ erts_unload_nif(struct erl_module_nif* lib) void erl_nif_init() { - ERTS_CT_ASSERT((offsetof(ErtsResource,data) % 8) == ERTS_MAGIC_BIN_BYTES_TO_ALIGN); + ERTS_CT_ASSERT((offsetof(ErtsResource,data) % 8) + == ERTS_MAGIC_BIN_BYTES_TO_ALIGN); resource_type_list.next = &resource_type_list; resource_type_list.prev = &resource_type_list; diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 78e0fa1864..5248f287ee 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -142,15 +142,6 @@ typedef struct typedef int ErlNifEvent; /* An event to be selected on. */ //#endif -typedef struct enif_resource_type_t ErlNifResourceType; -typedef void ErlNifResourceDtor(ErlNifEnv*, void*); -typedef void ErlNifResourceStop(ErlNifEnv*, void*, ErlNifEvent, int is_direct_call); - -typedef struct { - ErlNifResourceDtor* dtor; - ErlNifResourceStop* stop; /* at ERL_NIF_SELECT_STOP event */ -} ErlNifResourceTypeInit; - typedef enum { ERL_NIF_RT_CREATE = 1, @@ -172,6 +163,19 @@ typedef struct ERL_NIF_TERM port_id; /* internal, may change */ }ErlNifPort; +typedef ErlDrvMonitor ErlNifMonitor; + +typedef struct enif_resource_type_t ErlNifResourceType; +typedef void ErlNifResourceDtor(ErlNifEnv*, void*); +typedef void ErlNifResourceStop(ErlNifEnv*, void*, ErlNifEvent, int is_direct_call); +typedef void ErlNifResourceDown(ErlNifEnv*, void*, ErlNifPid*, ErlNifMonitor*); + +typedef struct { + ErlNifResourceDtor* dtor; + ErlNifResourceStop* stop; /* at ERL_NIF_SELECT_STOP event */ + ErlNifResourceDown* down; /* enif_monitor_process */ +} ErlNifResourceTypeInit; + typedef ErlDrvSysInfo ErlNifSysInfo; typedef struct ErlDrvTid_ *ErlNifTid; diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index 9163ce25eb..a9ed246962 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -177,6 +177,8 @@ ERL_NIF_API_FUNC_DECL(int,enif_thread_type,(void)); ERL_NIF_API_FUNC_DECL(int,enif_snprintf,(char * buffer, size_t size, const char *format, ...)); ERL_NIF_API_FUNC_DECL(int,enif_select,(ErlNifEnv* env, ErlNifEvent e, enum ErlNifSelectFlags flags, void* obj, ERL_NIF_TERM ref)); ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type_x,(ErlNifEnv*, const char* name_str, const ErlNifResourceTypeInit*, ErlNifResourceFlags flags, ErlNifResourceFlags* tried)); +ERL_NIF_API_FUNC_DECL(int, enif_monitor_process,(ErlNifEnv*,void* obj,const ErlNifPid*,ErlDrvMonitor *monitor)); +ERL_NIF_API_FUNC_DECL(int, enif_demonitor_process,(ErlNifEnv*,void* obj,const ErlDrvMonitor *monitor)); /* ** ADD NEW ENTRIES HERE (before this comment) !!! @@ -336,6 +338,8 @@ ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type_x,(ErlNifEnv*, # define enif_snprintf ERL_NIF_API_FUNC_MACRO(enif_snprintf) # define enif_select ERL_NIF_API_FUNC_MACRO(enif_select) # define enif_open_resource_type_x ERL_NIF_API_FUNC_MACRO(enif_open_resource_type_x) +# define enif_monitor_process ERL_NIF_API_FUNC_MACRO(enif_monitor_process) +# define enif_demonitor_process ERL_NIF_API_FUNC_MACRO(enif_demonitor_process) /* ** ADD NEW ENTRIES HERE (before this comment) diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 70500ed6e1..f463f7fdf4 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -1165,8 +1165,8 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) static void doit_insert_monitor(ErtsMonitor *monitor, void *p) { Eterm *idp = p; - if(is_external(monitor->pid)) - insert_node(external_thing_ptr(monitor->pid)->node, MONITOR_REF, *idp); + if(monitor->type != MON_NIF_TARGET && is_external(monitor->u.pid)) + insert_node(external_thing_ptr(monitor->u.pid)->node, MONITOR_REF, *idp); if(is_external(monitor->ref)) insert_node(external_thing_ptr(monitor->ref)->node, MONITOR_REF, *idp); } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index f80ebdf31b..9db77585f2 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -13320,9 +13320,9 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) switch (mon->type) { case MON_ORIGIN: /* We are monitoring someone else, we need to demonitor that one.. */ - if (is_atom(mon->pid)) { /* remote by name */ - ASSERT(is_node_name_atom(mon->pid)); - dep = erts_sysname_to_connected_dist_entry(mon->pid); + if (is_atom(mon->u.pid)) { /* remote by name */ + ASSERT(is_node_name_atom(mon->u.pid)); + dep = erts_sysname_to_connected_dist_entry(mon->u.pid); if (dep) { erts_smp_de_links_lock(dep); rmon = erts_remove_monitor(&(dep->monitors), mon->ref); @@ -13333,7 +13333,7 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) ERTS_DSP_NO_LOCK, 0); if (code == ERTS_DSIG_PREP_CONNECTED) { code = erts_dsig_send_demonitor(&dsd, - rmon->pid, + rmon->u.pid, mon->name, mon->ref, 1); @@ -13344,10 +13344,10 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) erts_deref_dist_entry(dep); } } else { - ASSERT(is_pid(mon->pid) || is_port(mon->pid)); + ASSERT(is_pid(mon->u.pid) || is_port(mon->u.pid)); /* if is local by pid or name */ - if (is_internal_pid(mon->pid)) { - Process *rp = erts_pid2proc(NULL, 0, mon->pid, ERTS_PROC_LOCK_LINK); + if (is_internal_pid(mon->u.pid)) { + Process *rp = erts_pid2proc(NULL, 0, mon->u.pid, ERTS_PROC_LOCK_LINK); if (!rp) { goto done; } @@ -13357,9 +13357,9 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) goto done; } erts_destroy_monitor(rmon); - } else if (is_internal_port(mon->pid)) { + } else if (is_internal_port(mon->u.pid)) { /* Is a local port */ - Port *prt = erts_port_lookup_raw(mon->pid); + Port *prt = erts_port_lookup_raw(mon->u.pid); if (!prt) { goto done; } @@ -13367,8 +13367,8 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) ERTS_PORT_DEMONITOR_ORIGIN_ON_DEATHBED, prt, mon->ref, NULL); } else { /* remote by pid */ - ASSERT(is_external_pid(mon->pid)); - dep = external_pid_dist_entry(mon->pid); + ASSERT(is_external_pid(mon->u.pid)); + dep = external_pid_dist_entry(mon->u.pid); ASSERT(dep != NULL); if (dep) { erts_smp_de_links_lock(dep); @@ -13380,8 +13380,8 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) ERTS_DSP_NO_LOCK, 0); if (code == ERTS_DSIG_PREP_CONNECTED) { code = erts_dsig_send_demonitor(&dsd, - rmon->pid, - mon->pid, + rmon->u.pid, + mon->u.pid, mon->ref, 1); ASSERT(code == ERTS_DSIG_SEND_OK); @@ -13393,22 +13393,21 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) } break; case MON_TARGET: - ASSERT(mon->type == MON_TARGET); - ASSERT(is_pid(mon->pid) || is_internal_port(mon->pid)); - if (is_internal_port(mon->pid)) { - Port *prt = erts_id2port(mon->pid); + ASSERT(is_pid(mon->u.pid) || is_internal_port(mon->u.pid)); + if (is_internal_port(mon->u.pid)) { + Port *prt = erts_id2port(mon->u.pid); if (prt == NULL) { goto done; } erts_fire_port_monitor(prt, mon->ref); erts_port_release(prt); - } else if (is_internal_pid(mon->pid)) {/* local by name or pid */ + } else if (is_internal_pid(mon->u.pid)) {/* local by name or pid */ Eterm watched; Process *rp; DeclareTmpHeapNoproc(lhp,3); ErtsProcLocks rp_locks = (ERTS_PROC_LOCK_LINK | ERTS_PROC_LOCKS_MSG_SEND); - rp = erts_pid2proc(NULL, 0, mon->pid, rp_locks); + rp = erts_pid2proc(NULL, 0, mon->u.pid, rp_locks); if (rp == NULL) { goto done; } @@ -13427,8 +13426,8 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) /* else: demonitor while we exited, i.e. do nothing... */ erts_smp_proc_unlock(rp, rp_locks); } else { /* external by pid or name */ - ASSERT(is_external_pid(mon->pid)); - dep = external_pid_dist_entry(mon->pid); + ASSERT(is_external_pid(mon->u.pid)); + dep = external_pid_dist_entry(mon->u.pid); ASSERT(dep != NULL); if (dep) { erts_smp_de_links_lock(dep); @@ -13440,10 +13439,10 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) ERTS_DSP_NO_LOCK, 0); if (code == ERTS_DSIG_PREP_CONNECTED) { code = erts_dsig_send_m_exit(&dsd, - mon->pid, + mon->u.pid, (rmon->name != NIL ? rmon->name - : rmon->pid), + : rmon->u.pid), mon->ref, pcontext->reason); ASSERT(code == ERTS_DSIG_SEND_OK); @@ -13453,6 +13452,11 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) } } break; + case MON_NIF_TARGET: + erts_fire_nif_monitor(mon->u.resource, + pcontext->p->common.id, + mon->ref); + break; case MON_TIME_OFFSET: erts_demonitor_time_offset(mon->ref); break; diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index 6aa2a7500f..c01cc26246 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -1866,7 +1866,7 @@ save_time_offset_monitor(ErtsMonitor *mon, void *vcntxt) cntxt = (ErtsTimeOffsetMonitorContext *) vcntxt; mix = (cntxt->ix)++; - cntxt->to_mon_info[mix].pid = mon->pid; + cntxt->to_mon_info[mix].pid = mon->u.pid; to_hp = &cntxt->to_mon_info[mix].heap[0]; ASSERT(is_internal_ref(mon->ref)); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 86c38f8e8c..511e357d14 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -66,30 +66,47 @@ struct enif_resource_type_t struct erl_module_nif* owner; /* that created this type and thus implements the destructor*/ ErlNifResourceDtor* dtor; /* user destructor function */ ErlNifResourceStop* stop; + ErlNifResourceDown* down; erts_refc_t refc; /* num of resources of this type (HOTSPOT warning) +1 for active erl_module_nif */ Eterm module; Eterm name; }; + typedef struct +{ + erts_smp_mtx_t lock; + ErtsMonitor* root; + size_t user_data_sz; +} ErtsResourceMonitors; + +typedef struct ErtsResource_ { struct enif_resource_type_t* type; + ErtsResourceMonitors* monitors; +#ifdef ARCH_32 + byte align__[4]; +#endif #ifdef DEBUG erts_refc_t nif_refc; -# ifdef ARCH_32 - byte align__[4]; + int dbg_is_dying; +# ifdef ARCH_64 + byte dbg_align__[4]; # endif #endif - char data[1]; }ErtsResource; -#define DATA_TO_RESOURCE(PTR) ((ErtsResource*)((char*)(PTR) - offsetof(ErtsResource,data))) +#define DATA_TO_RESOURCE(PTR) ErtsContainerStruct(PTR, ErtsResource, data) +#define erts_resource_ref_size(P) PROC_BIN_SIZE + +extern Eterm erts_bld_resource_ref(Eterm** hp, ErlOffHeap*, ErtsResource*); extern void erts_pre_nif(struct enif_environment_t*, Process*, struct erl_module_nif*, Process* tracee); extern void erts_post_nif(struct enif_environment_t* env); extern void erts_resource_stop(ErtsResource*, ErlNifEvent, int is_direct_call); +void erts_fire_nif_monitor(ErtsResource*, Eterm pid, Eterm ref); extern Eterm erts_nif_taints(Process* p); extern void erts_print_nif_taints(fmtfn_t to, void* to_arg); void erts_unload_nif(struct erl_module_nif* nif); @@ -286,11 +303,9 @@ typedef struct { } u; } ErtsMagicBinary; -#ifdef ARCH_32 -#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 4 -#else -#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 0 -#endif +#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN \ + (offsetof(ErtsMagicBinary,u.aligned.data) - \ + offsetof(ErtsMagicBinary,u.unaligned.data)) typedef union { Binary binary; @@ -1306,6 +1321,7 @@ void erts_stale_drv_select(Eterm, ErlDrvPort, ErlDrvEvent, int, int); Port *erts_get_heart_port(void); void erts_emergency_close_ports(void); +void erts_ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon); #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT) void erts_lcnt_enable_io_lock_count(int enable); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 33b74f30b7..7fa0280396 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -4192,8 +4192,8 @@ static void sweep_one_monitor(ErtsMonitor *mon, void *vpsc) ErtsMonitor *rmon; Process *rp; - ASSERT(is_internal_pid(mon->pid)); - rp = erts_pid2proc(NULL, 0, mon->pid, ERTS_PROC_LOCK_LINK); + ASSERT(is_internal_pid(mon->u.pid)); + rp = erts_pid2proc(NULL, 0, mon->u.pid, ERTS_PROC_LOCK_LINK); if (!rp) { goto done; } @@ -4294,7 +4294,7 @@ port_fire_one_monitor(ErtsMonitor *mon, void *ctx0) Process *origin; ErtsProcLocks origin_locks; - if (mon->type != MON_TARGET || ! is_pid(mon->pid)) { + if (mon->type != MON_TARGET || ! is_pid(mon->u.pid)) { return; } /* @@ -4303,7 +4303,7 @@ port_fire_one_monitor(ErtsMonitor *mon, void *ctx0) */ origin_locks = ERTS_PROC_LOCKS_MSG_SEND | ERTS_PROC_LOCK_LINK; - origin = erts_pid2proc(NULL, 0, mon->pid, origin_locks); + origin = erts_pid2proc(NULL, 0, mon->u.pid, origin_locks); if (origin) { DeclareTmpHeapNoproc(lhp,3); SweepContext *ctx = (SweepContext *)ctx0; @@ -5485,7 +5485,7 @@ typedef struct { static void prt_one_monitor(ErtsMonitor *mon, void *vprtd) { prt_one_lnk_data *prtd = (prt_one_lnk_data *) vprtd; - erts_print(prtd->to, prtd->arg, "(%T,%T)", mon->pid,mon->ref); + erts_print(prtd->to, prtd->arg, "(%T,%T)", mon->u.pid, mon->ref); } static void prt_one_lnk(ErtsLink *lnk, void *vprtd) @@ -7630,7 +7630,7 @@ erl_drv_convert_time_unit(ErlDrvTime val, (int) to); } -static void ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon) +void erts_ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon) { RefThing *refp; ASSERT(is_internal_ref(ref)); @@ -7664,7 +7664,7 @@ static int do_driver_monitor_process(Port *prt, erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, ref, prt->common.id, NIL); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - ref_to_driver_monitor(ref,monitor); + erts_ref_to_driver_monitor(ref,monitor); return 0; } @@ -7713,7 +7713,7 @@ static int do_driver_demonitor_process(Port *prt, Eterm *buf, return 1; } ASSERT(mon->type == MON_ORIGIN); - to = mon->pid; + to = mon->u.pid; ASSERT(is_internal_pid(to)); rp = erts_pid2proc_opt(NULL, 0, @@ -7775,7 +7775,7 @@ static ErlDrvTermData do_driver_get_monitored_process(Port *prt, Eterm *buf, return driver_term_nil; } ASSERT(mon->type == MON_ORIGIN); - to = mon->pid; + to = mon->u.pid; ASSERT(is_internal_pid(to)); return (ErlDrvTermData) to; } @@ -7831,7 +7831,7 @@ void erts_fire_port_monitor(Port *prt, Eterm ref) } callback = prt->drv_ptr->process_exit; ASSERT(callback != NULL); - ref_to_driver_monitor(ref,&drv_monitor); + erts_ref_to_driver_monitor(ref,&drv_monitor); ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT); DRV_MONITOR_UNLOCK_PDL(prt); #ifdef USE_VM_PROBES -- cgit v1.2.3 From d85e74e0c0e4bc66c875e2fd5f54d89255df0047 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 9 Feb 2017 15:23:11 +0100 Subject: erts: Add pid argument to enif_select --- erts/emulator/beam/erl_nif_api_funcs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index a9ed246962..1fc6842acc 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -175,7 +175,7 @@ ERL_NIF_API_FUNC_DECL(size_t, enif_binary_to_term, (ErlNifEnv *env, const unsign ERL_NIF_API_FUNC_DECL(int, enif_port_command, (ErlNifEnv *env, const ErlNifPort* to_port, ErlNifEnv *msg_env, ERL_NIF_TERM msg)); ERL_NIF_API_FUNC_DECL(int,enif_thread_type,(void)); ERL_NIF_API_FUNC_DECL(int,enif_snprintf,(char * buffer, size_t size, const char *format, ...)); -ERL_NIF_API_FUNC_DECL(int,enif_select,(ErlNifEnv* env, ErlNifEvent e, enum ErlNifSelectFlags flags, void* obj, ERL_NIF_TERM ref)); +ERL_NIF_API_FUNC_DECL(int,enif_select,(ErlNifEnv* env, ErlNifEvent e, enum ErlNifSelectFlags flags, void* obj, const ErlNifPid* pid, ERL_NIF_TERM ref)); ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type_x,(ErlNifEnv*, const char* name_str, const ErlNifResourceTypeInit*, ErlNifResourceFlags flags, ErlNifResourceFlags* tried)); ERL_NIF_API_FUNC_DECL(int, enif_monitor_process,(ErlNifEnv*,void* obj,const ErlNifPid*,ErlDrvMonitor *monitor)); ERL_NIF_API_FUNC_DECL(int, enif_demonitor_process,(ErlNifEnv*,void* obj,const ErlDrvMonitor *monitor)); -- cgit v1.2.3 From a8d5234c77634df9522727ff200cef4fcab49c22 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 9 Feb 2017 17:30:30 +0100 Subject: erts: Change return value for enif_select to negative int as error and positive as success. --- erts/emulator/beam/erl_drv_nif.h | 8 -------- erts/emulator/beam/erl_nif.h | 6 ++++++ 2 files changed, 6 insertions(+), 8 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h index 8de4a7855d..2d21dac87a 100644 --- a/erts/emulator/beam/erl_drv_nif.h +++ b/erts/emulator/beam/erl_drv_nif.h @@ -56,14 +56,6 @@ enum ErlNifSelectFlags { ERL_NIF_SELECT_STOP = (1 << 2) }; -enum ErlNifSelectReturn { - ERL_NIF_SELECT_ERROR = (1 << 0), - ERL_NIF_SELECT_STOP_CALLED = (1 << 1), - ERL_NIF_SELECT_STOP_SCHEDULED = (1 << 2), - ERL_NIF_SELECT_INVALID_EVENT = (1 << 3), - ERL_NIF_SELECT_FAILED = (1 << 4) -}; - /* * A driver monitor */ diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 5248f287ee..9ff28a30cc 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -142,6 +142,12 @@ typedef struct typedef int ErlNifEvent; /* An event to be selected on. */ //#endif +/* Return bits from enif_select: */ +#define ERL_NIF_SELECT_STOP_CALLED (1 << 0) +#define ERL_NIF_SELECT_STOP_SCHEDULED (1 << 1) +#define ERL_NIF_SELECT_INVALID_EVENT (1 << 2) +#define ERL_NIF_SELECT_FAILED (1 << 3) + typedef enum { ERL_NIF_RT_CREATE = 1, -- cgit v1.2.3 From 7e9f0932f9d06edca0c30d4a2b9d57f34677344c Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 31 Jan 2017 13:30:20 +0100 Subject: Magic indirection --- erts/emulator/beam/erl_alloc.c | 2 ++ erts/emulator/beam/erl_alloc.types | 1 + erts/emulator/beam/erl_binary.h | 44 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index e433de0f35..6708d96d56 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -684,6 +684,8 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) = sizeof(NifExportTrace); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MREF_NSCHED_ENT)] = sizeof(ErtsNSchedMagicRefTableEntry); + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MINDIRECTION)] + = ERTS_MAGIC_BIN_UNALIGNED_SIZE(sizeof(ErtsMagicIndirectionWord)); #ifdef HARD_DEBUG hdbg_init(); diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index c253ea82c8..f88985b7f5 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -284,6 +284,7 @@ type MREF_NSCHED_ENT FIXED_SIZE SYSTEM nsched_magic_ref_entry type MREF_ENT STANDARD SYSTEM magic_ref_entry type MREF_TAB_BKTS STANDARD SYSTEM magic_ref_table_buckets type MREF_TAB LONG_LIVED SYSTEM magic_ref_table +type MINDIRECTION FIXED_SIZE SYSTEM magic_indirection +if threads_no_smp # Need thread safe allocs, but std_alloc and fix_alloc are not; diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index c811a002ce..4d9c4d6eac 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -300,6 +300,17 @@ BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen); +typedef union { + /* + * These two are almost always of + * the same size, but when fallback + * atomics are used they might + * differ in size. + */ + erts_smp_atomic_t smp_atomic_word; + erts_atomic_t atomic_word; +} ErtsMagicIndirectionWord; + #if defined(__i386__) || !defined(__GNUC__) /* * Doubles aren't required to be 8-byte aligned on intel x86. @@ -329,6 +340,9 @@ ERTS_GLB_INLINE Binary *erts_create_magic_binary_x(Uint size, int unaligned); ERTS_GLB_INLINE Binary *erts_create_magic_binary(Uint size, void (*destructor)(Binary *)); +ERTS_GLB_INLINE Binary *erts_create_magic_indirection(void (*destructor)(Binary *)); +ERTS_GLB_INLINE erts_smp_atomic_t *erts_smp_binary_to_magic_indirection(Binary *bp); +ERTS_GLB_INLINE erts_atomic_t *erts_binary_to_magic_indirection(Binary *bp); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -496,6 +510,36 @@ erts_create_magic_binary(Uint size, void (*destructor)(Binary *)) ERTS_ALC_T_BINARY, 0); } +ERTS_GLB_INLINE Binary * +erts_create_magic_indirection(void (*destructor)(Binary *)) +{ + return erts_create_magic_binary_x(sizeof(ErtsMagicIndirectionWord), + destructor, + ERTS_ALC_T_MINDIRECTION, + 1); /* Not 64-bit aligned, + but word aligned */ +} + +ERTS_GLB_INLINE erts_smp_atomic_t * +erts_smp_binary_to_magic_indirection(Binary *bp) +{ + ErtsMagicIndirectionWord *mip; + ASSERT(bp->flags & BIN_FLAG_MAGIC); + ASSERT(ERTS_MAGIC_BIN_ATYPE(bp) == ERTS_ALC_T_MINDIRECTION); + mip = ERTS_MAGIC_BIN_UNALIGNED_DATA(bp); + return &mip->smp_atomic_word; +} + +ERTS_GLB_INLINE erts_atomic_t * +erts_binary_to_magic_indirection(Binary *bp) +{ + ErtsMagicIndirectionWord *mip; + ASSERT(bp->flags & BIN_FLAG_MAGIC); + ASSERT(ERTS_MAGIC_BIN_ATYPE(bp) == ERTS_ALC_T_MINDIRECTION); + mip = ERTS_MAGIC_BIN_UNALIGNED_DATA(bp); + return &mip->atomic_word; +} + #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ #endif /* !ERL_BINARY_H__ */ -- cgit v1.2.3 From 7c06ca6231b812965305522284dd9f2653ced98d Mon Sep 17 00:00:00 2001 From: Andrew Dryga Date: Tue, 14 Feb 2017 11:30:41 +0200 Subject: Fixed typos in erts --- erts/emulator/beam/beam_emu.c | 10 +++++----- erts/emulator/beam/bif.c | 4 ++-- erts/emulator/beam/binary.c | 4 ++-- erts/emulator/beam/dist.c | 4 ++-- erts/emulator/beam/dist.h | 4 ++-- erts/emulator/beam/dtrace-wrapper.h | 2 +- erts/emulator/beam/erl_alloc.types | 2 +- erts/emulator/beam/erl_alloc_util.c | 4 ++-- erts/emulator/beam/erl_bif_binary.c | 2 +- erts/emulator/beam/erl_gc.c | 2 +- erts/emulator/beam/erl_lock_count.h | 2 +- erts/emulator/beam/erl_monitors.c | 4 ++-- erts/emulator/beam/erl_process.c | 2 +- erts/emulator/beam/erl_ptab.c | 2 +- erts/emulator/beam/erl_time_sup.c | 2 +- erts/emulator/beam/io.c | 2 +- 16 files changed, 26 insertions(+), 26 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 5e275c8fc5..49f932a1c2 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -126,7 +126,7 @@ do { \ /* * We reuse some of fields in the save area in the process structure. - * This is safe to do, since this space is only activly used when + * This is safe to do, since this space is only actively used when * the process is switched out. */ #define REDS_IN(p) ((p)->def_arg_reg[5]) @@ -220,7 +220,7 @@ BeamInstr* em_call_bif_e; /* NOTE These should be the only variables containing trace instructions. ** Sometimes tests are form the instruction value, and sometimes -** for the refering variable (one of these), and rouge references +** for the referring variable (one of these), and rouge references ** will most likely cause chaos. */ BeamInstr beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ @@ -2993,7 +2993,7 @@ do { \ } /* - * An error occured in an arithmetic operation or test that could + * An error occurred in an arithmetic operation or test that could * appear either in a head or in a body. * In a head, execution should continue at failure address in Arg(0). * In a body, Arg(0) == 0 and an exception should be raised. @@ -6268,7 +6268,7 @@ apply_bif_error_adjustment(Process *p, Export *ep, * stackframe correct. Without the following adjustment, * 'p->cp' will point into the function that called * current function when handling the error. We add a - * dummy stackframe in order to achive this. + * dummy stackframe in order to achieve this. * * Note that these BIFs unconditionally will cause * an exception to be raised. That is, our modifications @@ -6612,7 +6612,7 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re #ifndef ERTS_SMP if (ERTS_PROC_IS_EXITING(c_p)) { /* - * See comment in the begining of the function... + * See comment in the beginning of the function... * * This second test is needed since gc might be traced. */ diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 2190c47262..4c9b9887c1 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -467,7 +467,7 @@ demonitor_local_port(Process *origin, Eterm ref, Eterm target) } /* Can return atom true, false, yield, internal_error, badarg or - * THE_NON_VALUE if error occured or trap has been set up + * THE_NON_VALUE if error occurred or trap has been set up */ static BIF_RETTYPE demonitor(Process *c_p, Eterm ref, Eterm *multip) @@ -597,7 +597,7 @@ BIF_RETTYPE demonitor_2(BIF_ALIST_2) switch (demonitor(BIF_P, BIF_ARG_1, &multi)) { case THE_NON_VALUE: - /* If other error occured or trap has been set up - pass through */ + /* If other error occurred or trap has been set up - pass through */ BIF_RET(THE_NON_VALUE); case am_false: if (info) diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index d38097fb12..8f907ee8a2 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -102,7 +102,7 @@ new_binary(Process *p, byte *buf, Uint len) pb->flags = 0; /* - * Miscellanous updates. Return the tagged binary. + * Miscellaneous updates. Return the tagged binary. */ OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); return make_binary(pb); @@ -139,7 +139,7 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, Uint len) pb->flags = 0; /* - * Miscellanous updates. Return the tagged binary. + * Miscellaneous updates. Return the tagged binary. */ OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); return make_binary(pb); diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 390f2a0db3..0d25eb123e 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -264,7 +264,7 @@ static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp) goto done; if (mon->type == MON_ORIGIN) { - /* local pid is beeing monitored */ + /* local pid is being monitored */ rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref); /* ASSERT(rmon != NULL); nope, can happen during process exit */ if (rmon != NULL) { @@ -795,7 +795,7 @@ erts_dsig_send_unlink(ErtsDSigData *dsdp, Eterm local, Eterm remote) } -/* A local process that's beeing monitored by a remote one exits. We send: +/* A local process that's being monitored by a remote one exits. We send: {DOP_MONITOR_P_EXIT, Local pid or name, Remote pid, ref, reason}, which is rather sad as only the ref is needed, no pid's... */ int diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index e82b416286..39670de506 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -115,8 +115,8 @@ extern int erts_is_alive; * erts_dsig_prepare() prepares a send of a distributed signal. * One of the values defined below are returned. If the returned * value is another than ERTS_DSIG_PREP_CONNECTED, the - * distributed signal cannot be sent before apropriate actions - * have been taken. Apropriate actions would typically be setting + * distributed signal cannot be sent before appropriate actions + * have been taken. Appropriate actions would typically be setting * up the connection. */ diff --git a/erts/emulator/beam/dtrace-wrapper.h b/erts/emulator/beam/dtrace-wrapper.h index 6f70d5961e..f6fc28b801 100644 --- a/erts/emulator/beam/dtrace-wrapper.h +++ b/erts/emulator/beam/dtrace-wrapper.h @@ -74,7 +74,7 @@ #if defined(_SDT_PROBE) && !defined(STAP_PROBE11) /* SLF: This is Ubuntu 11-style SystemTap hackery */ -/* work arround for missing STAP macro */ +/* workaround for missing STAP macro */ #define STAP_PROBE11(provider,name,arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10,arg11) \ _SDT_PROBE(provider, name, 11, \ (arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10,arg11)) diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index f88985b7f5..295db2fe9b 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -65,7 +65,7 @@ # --- Allocator declarations ------------------------------------------------- # -# If, and only if, the same thread performes *all* allocations, +# If, and only if, the same thread performs *all* allocations, # reallocations and deallocations of all memory types that are handled # by a specific allocator ( in type declaration), set # for this specific allocator to false; otherwise, set diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index d346f8c8b6..a6bac06478 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -5186,7 +5186,7 @@ erts_alcu_sz_info(Allctr_t *allctr, ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); - /* Update sbc values not continously updated */ + /* Update sbc values not continuously updated */ allctr->sbcs.blocks.curr.no = allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no; allctr->sbcs.blocks.max.no = allctr->sbcs.max.no; @@ -5272,7 +5272,7 @@ erts_alcu_info(Allctr_t *allctr, ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); - /* Update sbc values not continously updated */ + /* Update sbc values not continuously updated */ allctr->sbcs.blocks.curr.no = allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no; allctr->sbcs.blocks.max.no = allctr->sbcs.max.no; diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index e57cd06cec..64a5c1cb29 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -380,7 +380,7 @@ static void ac_compute_failure_functions(ACTrie *act, ACNode **qbuff) qbuff[qt++] = child; /* Search for correct failure function, follow the parent's failure function until you find a similar transition - funtion to this child's */ + function to this child's */ r = parent->h; while (r != NULL && r->g[i] == NULL) { r = r->h; diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index cb3a189e8f..8cc7fc0142 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -2834,7 +2834,7 @@ sweep_off_heap(Process *p, int fullsweep) prev = &MSO(p).first; ptr = MSO(p).first; - /* Firts part of the list will reside on the (old) new-heap. + /* First part of the list will reside on the (old) new-heap. * Keep if moved, otherwise deref. */ while (ptr) { diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h index 4f838f7faa..71cd73ee27 100644 --- a/erts/emulator/beam/erl_lock_count.h +++ b/erts/emulator/beam/erl_lock_count.h @@ -129,7 +129,7 @@ typedef struct { typedef struct erts_lcnt_lock_stats_s { /* "tries" and "colls" needs to be atomic since - * trylock busy does not aquire a lock and there + * trylock busy does not acquire a lock and there * is no post action to rectify the situation */ diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c index bdfc7f30d9..f813f19dbc 100644 --- a/erts/emulator/beam/erl_monitors.c +++ b/erts/emulator/beam/erl_monitors.c @@ -24,7 +24,7 @@ * key in the monitor case and the pid of the linked process as key in the * link case. Lookups the order of the references is somewhat special. Local * references are strictly smaller than remote references and are sorted - * by inlined comparision functionality. Remote references are handled by the + * by inlined comparison functionality. Remote references are handled by the * usual cmp function. * Each Monitor is tagged with different tags depending on which end of the * monitor it is. @@ -154,7 +154,7 @@ static ErtsMonitor *create_monitor(Uint type, Eterm ref, Eterm pid, Eterm name) n->type = (Uint16) type; n->balance = 0; /* Always the same initial value */ n->name = name; /* atom() or [] */ - CP_LINK_VAL(n->ref, hp, ref); /*XXX Unneccesary check, never immediate*/ + CP_LINK_VAL(n->ref, hp, ref); /*XXX Unnecessary check, never immediate*/ CP_LINK_VAL(n->pid, hp, pid); return n; diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 87d71fdd22..c5a7b67764 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -5050,7 +5050,7 @@ check_balance(ErtsRunQueue *c_rq) sched_util_balancing = 1; /* * In order to avoid renaming a large amount of fields - * we write utilization values instead of lenght values + * we write utilization values instead of length values * in the 'max_len' and 'migration_limit' fields... */ for (qix = 0; qix < blnc_no_rqs; qix++) { diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c index a132b1d334..828a029e84 100644 --- a/erts/emulator/beam/erl_ptab.c +++ b/erts/emulator/beam/erl_ptab.c @@ -474,7 +474,7 @@ erts_ptab_init_table(ErtsPTab *ptab, * we don't want to shrink the size to ERTS_PTAB_MAX_SIZE/2. * * In order to fix this, we insert a pointer from the table - * to the invalid_element, wich will be interpreted as a + * to the invalid_element, which will be interpreted as a * slot currently being modified. This way we will be able to * have ERTS_PTAB_MAX_SIZE-1 valid elements in the table while * still having a table size of the power of 2. diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index 1d747e5fab..ed40539b78 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -1502,7 +1502,7 @@ static time_t gregday(int year, int month, int day) pyear = gyear - 1; ndays = (pyear/4) - (pyear/100) + (pyear/400) + pyear*365 + 366; } - /* number of days in all months preceeding month */ + /* number of days in all months preceding month */ for (m = 1; m < month; m++) ndays += mdays[m]; /* Extra day if leap year and March or later */ diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index dad24d7ef6..7e49e9c48f 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -3431,7 +3431,7 @@ void erts_init_io(int port_tab_size, NULL, (ErtsPTabElementCommon *) &erts_invalid_port.common, port_tab_size, - common_element_size, /* Doesn't need to be excact */ + common_element_size, /* Doesn't need to be exact */ "port_table", legacy_port_tab, 1); -- cgit v1.2.3 From d73423cdba178c166f25e00a4608ccc8d0465937 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 14 Feb 2017 15:53:04 +0100 Subject: erts: Fix round/1 for large floats 1> round(6209607916799025.0). 6209607916799026 Problem: Adding/subtracting 0.5 and large double floats between (1 bsl 52) and (1 bsl 53) does not give reliable results. Solution: Use round() function in math.h. --- erts/emulator/beam/erl_bif_guard.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c index b42d2dc28b..74cf7cf11a 100644 --- a/erts/emulator/beam/erl_bif_guard.c +++ b/erts/emulator/beam/erl_bif_guard.c @@ -157,7 +157,7 @@ BIF_RETTYPE round_1(BIF_ALIST_1) GET_DOUBLE(BIF_ARG_1, f); /* round it and return the resultant integer */ - res = double_to_integer(BIF_P, (f.fd > 0.0) ? f.fd + 0.5 : f.fd - 0.5); + res = double_to_integer(BIF_P, round(f.fd)); BIF_RET(res); } @@ -597,8 +597,7 @@ Eterm erts_gc_round_1(Process* p, Eterm* reg, Uint live) } GET_DOUBLE(arg, f); - return gc_double_to_integer(p, (f.fd > 0.0) ? f.fd + 0.5 : f.fd - 0.5, - reg, live); + return gc_double_to_integer(p, round(f.fd), reg, live); } Eterm erts_gc_trunc_1(Process* p, Eterm* reg, Uint live) -- cgit v1.2.3 From 118de47d703e303aea7f4575849a37c11416ba14 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 14 Feb 2017 18:26:31 +0100 Subject: erts: Add deallocation veto for magic destructors A magic destructor can return 0 and thereby take control and prolong the lifetime of a magic binary. --- erts/emulator/beam/beam_load.c | 5 +++-- erts/emulator/beam/binary.c | 6 ++++-- erts/emulator/beam/dist.c | 4 +++- erts/emulator/beam/dist.h | 2 +- erts/emulator/beam/erl_bif_binary.c | 15 ++++++++------- erts/emulator/beam/erl_bif_info.c | 4 ++-- erts/emulator/beam/erl_bif_re.c | 3 ++- erts/emulator/beam/erl_binary.h | 19 +++++++++++-------- erts/emulator/beam/erl_db_util.c | 5 +++-- erts/emulator/beam/erl_db_util.h | 2 +- erts/emulator/beam/erl_map.c | 3 ++- erts/emulator/beam/erl_nif.c | 3 ++- erts/emulator/beam/erl_ptab.c | 6 ++++-- erts/emulator/beam/erl_unicode.c | 3 ++- erts/emulator/beam/external.c | 6 ++++-- 15 files changed, 52 insertions(+), 34 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index e75e7afd54..48206a75a8 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -488,7 +488,7 @@ typedef struct LoaderState { static void free_loader_state(Binary* magic); static ErlHeapFragment* new_literal_fragment(Uint size); static void free_literal_fragment(ErlHeapFragment*); -static void loader_state_dtor(Binary* magic); +static int loader_state_dtor(Binary* magic); #ifdef HIPE static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, @@ -1026,7 +1026,7 @@ static void free_literal_fragment(ErlHeapFragment* bp) /* * This destructor function can safely be called multiple times. */ -static void +static int loader_state_dtor(Binary* magic) { LoaderState* stp = ERTS_MAGIC_BIN_DATA(magic); @@ -1111,6 +1111,7 @@ loader_state_dtor(Binary* magic) */ ASSERT(stp->genop_blocks == 0); + return 1; } #ifdef HIPE diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index d38097fb12..be43f6e1ac 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -350,9 +350,10 @@ typedef struct { Uint bitoffs; } ErtsB2LState; -static void b2l_state_destructor(Binary *mbp) +static int b2l_state_destructor(Binary *mbp) { ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == b2l_state_destructor); + return 1; } static BIF_RETTYPE @@ -723,12 +724,13 @@ list_to_binary_engine(ErtsL2BState *sp) } } -static void +static int l2b_state_destructor(Binary *mbp) { ErtsL2BState *sp = ERTS_MAGIC_BIN_DATA(mbp); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == l2b_state_destructor); DESTROY_SAVED_ESTACK(&sp->buf.iolist.estack); + return 1; } static ERTS_INLINE Eterm diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 390f2a0db3..da1083100b 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -713,7 +713,7 @@ static void clear_dist_entry(DistEntry *dep) } } -void erts_dsend_context_dtor(Binary* ctx_bin) +int erts_dsend_context_dtor(Binary* ctx_bin) { ErtsSendContext* ctx = ERTS_MAGIC_BIN_DATA(ctx_bin); switch (ctx->dss.phase) { @@ -730,6 +730,8 @@ void erts_dsend_context_dtor(Binary* ctx_bin) } if (ctx->dep_to_deref) erts_deref_dist_entry(ctx->dep_to_deref); + + return 1; } Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx) diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index e82b416286..8799e54057 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -375,7 +375,7 @@ extern int erts_dsig_send_monitor(ErtsDSigData *, Eterm, Eterm, Eterm); extern int erts_dsig_send_m_exit(ErtsDSigData *, Eterm, Eterm, Eterm, Eterm); extern int erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx); -extern void erts_dsend_context_dtor(Binary*); +extern int erts_dsend_context_dtor(Binary*); extern Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx); extern int erts_dist_command(Port *prt, int reds); diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index e57cd06cec..deab371e3e 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -239,13 +239,13 @@ static void dump_ac_node(ACNode *node, int indent, int ch); /* * Callback for the magic binary */ -static void cleanup_my_data_ac(Binary *bp) +static int cleanup_my_data_ac(Binary *bp) { - return; + return 1; } -static void cleanup_my_data_bm(Binary *bp) +static int cleanup_my_data_bm(Binary *bp) { - return; + return 1; } /* @@ -2067,7 +2067,7 @@ static int do_search_backward(CommonData *cd, Uint *posp, Uint *redsp) } } -static void cleanup_common_data(Binary *bp) +static int cleanup_common_data(Binary *bp) { int i; CommonData *cd; @@ -2084,7 +2084,7 @@ static void cleanup_common_data(Binary *bp) break; } } - return; + return 1; } static BIF_RETTYPE do_longest_common(Process *p, Eterm list, int direction) @@ -2563,7 +2563,7 @@ typedef struct { #define BINARY_COPY_LOOP_FACTOR 100 -static void cleanup_copy_bin_state(Binary *bp) +static int cleanup_copy_bin_state(Binary *bp) { CopyBinState *cbs = (CopyBinState *) ERTS_MAGIC_BIN_DATA(bp); if (cbs->result != NULL) { @@ -2583,6 +2583,7 @@ static void cleanup_copy_bin_state(Binary *bp) break; } cbs->source_type = BC_TYPE_EMPTY; + return 1; } /* diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index be113b7c88..5a8776076e 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -3567,9 +3567,9 @@ BIF_RETTYPE error_logger_warning_map_0(BIF_ALIST_0) static erts_smp_atomic_t available_internal_state; -static void empty_magic_ref_destructor(Binary *bin) +static int empty_magic_ref_destructor(Binary *bin) { - + return 1; } BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index ba183f24e8..a66b05c6ff 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -600,10 +600,11 @@ static void cleanup_restart_context(RestartContext *rc) } } -static void cleanup_restart_context_bin(Binary *bp) +static int cleanup_restart_context_bin(Binary *bp) { RestartContext *rc = ERTS_MAGIC_BIN_DATA(bp); cleanup_restart_context(rc); + return 1; } /* diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index 4d9c4d6eac..db259be2a7 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -65,7 +65,7 @@ typedef struct magic_binary ErtsMagicBinary; struct magic_binary { ERTS_INTERNAL_BINARY_FIELDS SWord orig_size; - void (*destructor)(Binary *); + int (*destructor)(Binary *); Uint32 refn[ERTS_REF_NUMBERS]; ErtsAlcType_t alloc_type; union { @@ -335,12 +335,12 @@ ERTS_GLB_INLINE Binary *erts_bin_realloc_fnf(Binary *bp, Uint size); ERTS_GLB_INLINE Binary *erts_bin_realloc(Binary *bp, Uint size); ERTS_GLB_INLINE void erts_bin_free(Binary *bp); ERTS_GLB_INLINE Binary *erts_create_magic_binary_x(Uint size, - void (*destructor)(Binary *), + int (*destructor)(Binary *), ErtsAlcType_t alloc_type, int unaligned); ERTS_GLB_INLINE Binary *erts_create_magic_binary(Uint size, - void (*destructor)(Binary *)); -ERTS_GLB_INLINE Binary *erts_create_magic_indirection(void (*destructor)(Binary *)); + int (*destructor)(Binary *)); +ERTS_GLB_INLINE Binary *erts_create_magic_indirection(int (*destructor)(Binary *)); ERTS_GLB_INLINE erts_smp_atomic_t *erts_smp_binary_to_magic_indirection(Binary *bp); ERTS_GLB_INLINE erts_atomic_t *erts_binary_to_magic_indirection(Binary *bp); @@ -471,7 +471,10 @@ ERTS_GLB_INLINE void erts_bin_free(Binary *bp) { if (bp->flags & BIN_FLAG_MAGIC) { - ERTS_MAGIC_BIN_DESTRUCTOR(bp)(bp); + if (!ERTS_MAGIC_BIN_DESTRUCTOR(bp)(bp)) { + /* Destructor took control of the deallocation */ + return; + } erts_magic_ref_remove_bin(ERTS_MAGIC_BIN_REFN(bp)); erts_free(ERTS_MAGIC_BIN_ATYPE(bp), (void *) bp); } @@ -482,7 +485,7 @@ erts_bin_free(Binary *bp) } ERTS_GLB_INLINE Binary * -erts_create_magic_binary_x(Uint size, void (*destructor)(Binary *), +erts_create_magic_binary_x(Uint size, int (*destructor)(Binary *), ErtsAlcType_t alloc_type, int unaligned) { @@ -504,14 +507,14 @@ erts_create_magic_binary_x(Uint size, void (*destructor)(Binary *), } ERTS_GLB_INLINE Binary * -erts_create_magic_binary(Uint size, void (*destructor)(Binary *)) +erts_create_magic_binary(Uint size, int (*destructor)(Binary *)) { return erts_create_magic_binary_x(size, destructor, ERTS_ALC_T_BINARY, 0); } ERTS_GLB_INLINE Binary * -erts_create_magic_indirection(void (*destructor)(Binary *)) +erts_create_magic_indirection(int (*destructor)(Binary *)) { return erts_create_magic_binary_x(sizeof(ErtsMagicIndirectionWord), destructor, diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 070e29578f..3ac2bdd3d6 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1702,17 +1702,18 @@ error: /* Here is were we land when compilation failed. */ /* ** Free a match program (in a binary) */ -void erts_db_match_prog_destructor(Binary *bprog) +int erts_db_match_prog_destructor(Binary *bprog) { MatchProg *prog; if (bprog == NULL) - return; + return 1; prog = Binary2MatchProg(bprog); if (prog->term_save != NULL) { free_message_buffer(prog->term_save); } if (prog->saved_program_buf != NULL) free_message_buffer(prog->saved_program_buf); + return 1; } void diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index b2a3bb6c20..471fefe3cb 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -356,7 +356,7 @@ Eterm db_add_counter(Eterm** hpp, Wterm counter, Eterm incr); Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags); Binary *db_match_set_compile(Process *p, Eterm matchexpr, Uint flags); -void erts_db_match_prog_destructor(Binary *); +int erts_db_match_prog_destructor(Binary *); typedef struct match_prog { ErlHeapFragment *term_save; /* Only if needed, a list of message diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index 990be8efb2..9062e44b10 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -1188,12 +1188,13 @@ typedef struct HashmapMergeContext_ { #endif } HashmapMergeContext; -static void hashmap_merge_ctx_destructor(Binary* ctx_bin) +static int hashmap_merge_ctx_destructor(Binary* ctx_bin) { HashmapMergeContext* ctx = (HashmapMergeContext*) ERTS_MAGIC_BIN_DATA(ctx_bin); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(ctx_bin) == hashmap_merge_ctx_destructor); PSTACK_DESTROY_SAVED(&ctx->pstack); + return 1; } BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1) { diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 3674a29bf8..61ae57a7f0 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2140,7 +2140,7 @@ static void rollback_opened_resource_types(void) } -static void nif_resource_dtor(Binary* bin) +static int nif_resource_dtor(Binary* bin) { ErlNifResource* resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(bin); ErlNifResourceType* type = resource->type; @@ -2159,6 +2159,7 @@ static void nif_resource_dtor(Binary* bin) steal_resource_type(type); erts_free(ERTS_ALC_T_NIF, type); } + return 1; } void* enif_alloc_resource(ErlNifResourceType* type, size_t size) diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c index a132b1d334..ce28708ece 100644 --- a/erts/emulator/beam/erl_ptab.c +++ b/erts/emulator/beam/erl_ptab.c @@ -733,7 +733,7 @@ erts_ptab_delete_element(ErtsPTab *ptab, * erts_ptab_list() implements BIFs listing the content of the table, * e.g. erlang:processes/0. */ -static void cleanup_ptab_list_bif_data(Binary *bp); +static int cleanup_ptab_list_bif_data(Binary *bp); static int ptab_list_bif_engine(Process *c_p, Eterm *res_accp, Binary *mbp); @@ -787,7 +787,7 @@ erts_ptab_list(Process *c_p, ErtsPTab *ptab) return ret_val; } -static void +static int cleanup_ptab_list_bif_data(Binary *bp) { ErtsPTabListBifData *ptlbdp = ERTS_MAGIC_BIN_DATA(bp); @@ -875,6 +875,8 @@ cleanup_ptab_list_bif_data(Binary *bp) ERTS_PTAB_LIST_DBG_TRACE(ptlbdp->debug.caller, return); ERTS_PTAB_LIST_DBG_CLEANUP(ptlbdp); + + return 1; } static int diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index dd7c589c3d..01629db9ad 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -123,10 +123,11 @@ static void cleanup_restart_context(RestartContext *rc) } } -static void cleanup_restart_context_bin(Binary *bp) +static int cleanup_restart_context_bin(Binary *bp) { RestartContext *rc = ERTS_MAGIC_BIN_DATA(bp); cleanup_restart_context(rc); + return 1; } static RestartContext *get_rc_from_bin(Eterm mref) diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index f9cba0aec0..205a7711ec 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1398,12 +1398,13 @@ static void b2t_destroy_context(B2TContext* context) } } -static void b2t_context_destructor(Binary *context_bin) +static int b2t_context_destructor(Binary *context_bin) { B2TContext* ctx = (B2TContext*) ERTS_MAGIC_BIN_DATA(context_bin); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor); b2t_destroy_context(ctx); + return 1; } static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1) @@ -1808,7 +1809,7 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) { #endif #define TERM_TO_BINARY_MEMCPY_FACTOR 8 -static void ttb_context_destructor(Binary *context_bin) +static int ttb_context_destructor(Binary *context_bin) { TTBContext *context = ERTS_MAGIC_BIN_DATA(context_bin); if (context->alive) { @@ -1842,6 +1843,7 @@ static void ttb_context_destructor(Binary *context_bin) break; } } + return 1; } static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint flags, -- cgit v1.2.3 From fc5a4a227968001cb031802c5a122cdbb0323b25 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 16 Feb 2017 19:11:54 +0100 Subject: Handle magic refs in db_cleanup_offheap_comp() --- erts/emulator/beam/erl_db_util.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 3ac2bdd3d6..6f30b1d3dd 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -3104,6 +3104,11 @@ void db_cleanup_offheap_comp(DbTerm* obj) erts_erase_fun_entry(u.fun->fe); } break; + case REF_SUBTAG: + ASSERT(is_magic_ref_thing(u.hdr)); + if (erts_refc_dectest(&u.mref->mb->refc, 0) == 0) + erts_bin_free((Binary *)u.mref->mb); + break; default: ASSERT(is_external_header(u.hdr->thing_word)); ASSERT(u.pb != &tmp); -- cgit v1.2.3 From 21a737586fbaff6794f7b737ea215b8daa35ea3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Magnus=20L=C3=A5ng?= Date: Sun, 19 Feb 2017 10:51:10 +0100 Subject: erts: Drop workarounds for ErLLVM w/ LLVM < 3.9 --- erts/emulator/beam/erl_process.h | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index ce0989883c..8bf372dad5 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -958,9 +958,11 @@ struct process { Eterm* stop; /* Stack top */ Eterm* heap; /* Heap start */ Eterm* hend; /* Heap end */ + Eterm* abandoned_heap; Uint heap_sz; /* Size of heap in words */ Uint min_heap_size; /* Minimum size of heap (in words). */ Uint min_vheap_size; /* Minimum size of virtual heap (in words). */ + Uint max_heap_size; /* Maximum size of heap (in words). */ #if !defined(NO_FPE_SIGNALS) || defined(HIPE) volatile unsigned long fp_exception; @@ -972,16 +974,6 @@ struct process { struct hipe_process_state hipe; #endif - /* - * Moved to after "struct hipe_process_state hipe", as a temporary fix for - * LLVM hard-coding offsetof(struct process, hipe.nstack) (sic!) - * (see void X86FrameLowering::adjustForHiPEPrologue(...) in - * lib/Target/X86/X86FrameLowering.cpp). - * - * Used to be below "Eterm* hend". - */ - Eterm* abandoned_heap; - /* * Saved x registers. */ @@ -1061,7 +1053,6 @@ struct process { Eterm *old_hend; /* Heap pointers for generational GC. */ Eterm *old_htop; Eterm *old_heap; - Uint max_heap_size; /* Maximum size of heap (in words). */ Uint16 gen_gcs; /* Number of (minor) generational GCs. */ Uint16 max_gen_gcs; /* Max minor gen GCs before fullsweep. */ ErlOffHeap off_heap; /* Off-heap data updated by copy_struct(). */ -- cgit v1.2.3 From 7dc39b7f02bf1e12541e91298a44d09d052d8fde Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 16 Feb 2017 19:23:51 +0100 Subject: Fix driver monitor implementation --- erts/emulator/beam/io.c | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 7e49e9c48f..e0cc756887 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -7609,10 +7609,21 @@ erl_drv_convert_time_unit(ErlDrvTime val, static void ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon) { - ERTS_CT_ASSERT(sizeof(ErtsOIRefStorage) <= sizeof(ErlDrvMonitor)); - erts_oiref_storage_save((ErtsOIRefStorage *) mon, ref); + ERTS_CT_ASSERT(ERTS_REF_THING_SIZE*sizeof(Uint) <= sizeof(ErlDrvMonitor)); + ASSERT(is_internal_ordinary_ref(ref)); + sys_memcpy((void *) mon, (void *) internal_ref_val(ref), + ERTS_REF_THING_SIZE*sizeof(Uint)); } +static Eterm driver_monitor_to_ref(Eterm *hp, const ErlDrvMonitor *mon) +{ + Eterm ref; + ERTS_CT_ASSERT(ERTS_REF_THING_SIZE*sizeof(Uint) <= sizeof(ErlDrvMonitor)); + sys_memcpy((void *) hp, (void *) mon, ERTS_REF_THING_SIZE*sizeof(Uint)); + ref = make_internal_ref(hp); + ASSERT(is_internal_ordinary_ref(ref)); + return ref; +} static int do_driver_monitor_process(Port *prt, ErlDrvTermData process, @@ -7669,13 +7680,12 @@ int driver_monitor_process(ErlDrvPort drvport, static int do_driver_demonitor_process(Port *prt, const ErlDrvMonitor *monitor) { Eterm heap[ERTS_REF_THING_SIZE]; - Eterm *hp = &heap[0]; Process *rp; Eterm ref; ErtsMonitor *mon; Eterm to; - ref = erts_oiref_storage_make_ref((ErtsOIRefStorage *) monitor, &hp), + ref = driver_monitor_to_ref(heap, monitor); mon = erts_lookup_monitor(ERTS_P_MONITORS(prt), ref); if (mon == NULL) { @@ -7731,9 +7741,8 @@ static ErlDrvTermData do_driver_get_monitored_process(Port *prt,const ErlDrvMoni ErtsMonitor *mon; Eterm to; Eterm heap[ERTS_REF_THING_SIZE]; - Eterm *hp = &heap[0]; - ref = erts_oiref_storage_make_ref((ErtsOIRefStorage *) monitor, &hp), + ref = driver_monitor_to_ref(heap, monitor); mon = erts_lookup_monitor(ERTS_P_MONITORS(prt), ref); if (mon == NULL) { @@ -7767,12 +7776,11 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort drvport, return ret; } - int driver_compare_monitors(const ErlDrvMonitor *monitor1, const ErlDrvMonitor *monitor2) { - return erts_oiref_storage_cmp((ErtsOIRefStorage *) monitor1, - (ErtsOIRefStorage *) monitor2); + return sys_memcmp((void *) monitor1, (void *) monitor2, + ERTS_REF_THING_SIZE*sizeof(Eterm)); } void erts_fire_port_monitor(Port *prt, Eterm ref) -- cgit v1.2.3 From 5adbf961a3c79a6782f8be8336ec26594754e9e8 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 20 Feb 2017 19:16:48 +0100 Subject: erts: Add enif_compare_monitors # Conflicts: # erts/emulator/test/nif_SUITE_data/nif_SUITE.c --- erts/emulator/beam/erl_nif.c | 5 +++++ erts/emulator/beam/erl_nif_api_funcs.h | 2 ++ 2 files changed, 7 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 62191c4abf..cd6fbf276b 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -3202,6 +3202,11 @@ int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monit return 0; } +int enif_compare_monitors(const ErlNifMonitor *monitor1, + const ErlNifMonitor *monitor2) +{ + return memcmp(monitor1,monitor2,sizeof(ErlNifMonitor)); +} /*************************************************************************** ** load_nif/2 ** diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index 1fc6842acc..01d9e386ed 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -179,6 +179,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_select,(ErlNifEnv* env, ErlNifEvent e, enum ErlNi ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type_x,(ErlNifEnv*, const char* name_str, const ErlNifResourceTypeInit*, ErlNifResourceFlags flags, ErlNifResourceFlags* tried)); ERL_NIF_API_FUNC_DECL(int, enif_monitor_process,(ErlNifEnv*,void* obj,const ErlNifPid*,ErlDrvMonitor *monitor)); ERL_NIF_API_FUNC_DECL(int, enif_demonitor_process,(ErlNifEnv*,void* obj,const ErlDrvMonitor *monitor)); +ERL_NIF_API_FUNC_DECL(int, enif_compare_monitors,(const ErlNifMonitor*,const ErlNifMonitor*)); /* ** ADD NEW ENTRIES HERE (before this comment) !!! @@ -340,6 +341,7 @@ ERL_NIF_API_FUNC_DECL(int, enif_demonitor_process,(ErlNifEnv*,void* obj,const Er # define enif_open_resource_type_x ERL_NIF_API_FUNC_MACRO(enif_open_resource_type_x) # define enif_monitor_process ERL_NIF_API_FUNC_MACRO(enif_monitor_process) # define enif_demonitor_process ERL_NIF_API_FUNC_MACRO(enif_demonitor_process) +# define enif_compare_monitors ERL_NIF_API_FUNC_MACRO(enif_compare_monitors) /* ** ADD NEW ENTRIES HERE (before this comment) -- cgit v1.2.3 From cb2a5bd9e86ba49d9bbc83b3d8383fbe0cc90715 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 14 Feb 2017 19:25:16 +0100 Subject: erts: Avoid revival of dying resource by dec_term --- erts/emulator/beam/erl_binary.h | 8 ++-- erts/emulator/beam/erl_nif.c | 88 ++++++++++++++++++++--------------------- erts/emulator/beam/global.h | 12 +++--- 3 files changed, 53 insertions(+), 55 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index db259be2a7..946d3cfe03 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -79,11 +79,9 @@ struct magic_binary { } u; }; -#ifdef ARCH_32 -#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 4 -#else -#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 0 -#endif +#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN \ + (offsetof(ErtsMagicBinary,u.aligned.data) - \ + offsetof(ErtsMagicBinary,u.unaligned.data)) typedef union { Binary binary; diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 7e7b78ec14..daec4ac9e9 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2141,7 +2141,7 @@ static void rollback_opened_resource_types(void) struct destroy_monitor_ctx { - Binary* resource_bin; + ErtsResource* resource; int exiting_procs; int scheduler; }; @@ -2188,30 +2188,26 @@ static void destroy_one_monitor(ErtsMonitor* mon, void* context) #endif } if (is_exiting) { - ctx->exiting_procs++; - /* +1 for exiting process */ - erts_refc_inc(&ctx->resource_bin->refc, ctx->exiting_procs); + ctx->resource->monitors->pending_failed_fire++; } /* ToDo: Delay destruction after monitor_locks */ if (rmon) { ASSERT(rmon->type == MON_NIF_TARGET); - ASSERT(&ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rmon->u.pid)->binary == ctx->resource_bin); + ASSERT(rmon->u.resource == ctx->resource); erts_destroy_monitor(rmon); } erts_destroy_monitor(mon); } -static int destroy_all_monitors(ErtsMonitor* monitors, Binary* bin) +static void destroy_all_monitors(ErtsMonitor* monitors, ErtsResource* resource) { struct destroy_monitor_ctx ctx; execution_state(NULL, NULL, &ctx.scheduler); - ctx.resource_bin = bin; - ctx.exiting_procs = 0; + ctx.resource = resource; erts_sweep_monitors(monitors, &destroy_one_monitor, &ctx); - return ctx.exiting_procs == 0; } @@ -2252,31 +2248,32 @@ static int nif_resource_dtor(Binary* bin) if (resource->monitors) { ErtsResourceMonitors* rm = resource->monitors; - ErtsMonitor* root; ASSERT(type->down); erts_smp_mtx_lock(&rm->lock); ASSERT(erts_refc_read(&bin->refc, 0) == 0); - root = rm->root; - if (root) { + if (rm->root) { + ASSERT(!rm->is_dying); + destroy_all_monitors(rm->root, resource); rm->root = NULL; - if (!destroy_all_monitors(root, bin)) { - /* - * Resource death struggle prolonged to serve exiting process(es). - * Destructor will be called again when last exiting process - * tries to fire its MON_NIF_TARGET monitor (and fails). - * - * This resource is doomed. It has no "real" references and - * should get not get called upon to do anything except the - * final destructor call. - */ - ASSERT(erts_refc_read(&bin->refc, 1)); -#ifdef DEBUG - resource->dbg_is_dying = 1; -#endif - erts_smp_mtx_unlock(&rm->lock); - return 0; - } + } + if (rm->pending_failed_fire) { + /* + * Resource death struggle prolonged to serve exiting process(es). + * Destructor will be called again when last exiting process + * tries to fire its MON_NIF_TARGET monitor (and fails). + * + * This resource is doomed. It has no "real" references and + * should get not get called upon to do anything except the + * final destructor call. + * + * We keep refc at 0 and use a separate counter for exiting + * processes to avoid resource getting revived by "dec_term". + */ + ASSERT(!rm->is_dying); + rm->is_dying = 1; + erts_smp_mtx_unlock(&rm->lock); + return 0; } erts_smp_mtx_unlock(&rm->lock); erts_smp_mtx_destroy(&rm->lock); @@ -2326,24 +2323,24 @@ void erts_fire_nif_monitor(ErtsResource* resource, Eterm pid, Eterm ref) erts_smp_mtx_lock(&rmp->lock); rmon = erts_remove_monitor(&rmp->root, ref); if (!rmon) { + int free_me = (--rmp->pending_failed_fire == 0) && rmp->is_dying; + ASSERT(rmp->pending_failed_fire >= 0); erts_smp_mtx_unlock(&rmp->lock); - /* -1 for exiting process */ - if (erts_refc_dectest(&bin->binary.refc, 0) == 0) { + + if (free_me) { + ASSERT(erts_refc_read(&bin->binary.refc, 0) == 0); erts_bin_free(&bin->binary); } return; } - ASSERT(!resource->dbg_is_dying); - if (erts_refc_inctest(&bin->binary.refc, 1) < 2) { + ASSERT(!rmp->is_dying); + if (erts_refc_inc_unless(&bin->binary.refc, 0, 0) == 0) { /* * Racing resource destruction. * To avoid a more complex refc-dance with destructing thread * we avoid calling 'down' and just silently remove the monitor. - * There are no real references left to this resource and we have - * monitors_lock, so it's safe to reset refc back to zero. * This can happen even for non smp as destructor calls may be scheduled. */ - erts_refc_init(&bin->binary.refc, 0); erts_smp_mtx_unlock(&rmp->lock); } else { @@ -2390,13 +2387,14 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t data_sz) erts_refc_inc(&bin->refc, 1); #ifdef DEBUG erts_refc_init(&resource->nif_refc, 1); - resource->dbg_is_dying = 0; #endif erts_refc_inc(&resource->type->refc, 2); if (type->down) { resource->monitors = (ErtsResourceMonitors*) (resource->data + monitors_offs); erts_smp_mtx_init(&resource->monitors->lock, "resource_monitors"); resource->monitors->root = NULL; + resource->monitors->pending_failed_fire = 0; + resource->monitors->is_dying = 0; resource->monitors->user_data_sz = data_sz; } else { @@ -2411,7 +2409,7 @@ void enif_release_resource(void* obj) ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == NIF_RESOURCE_DTOR); - ASSERT(!resource->dbg_is_dying); + ASSERT(!(resource->monitors && resource->monitors->is_dying)); #ifdef DEBUG erts_refc_dec(&resource->nif_refc, 0); #endif @@ -2426,7 +2424,7 @@ void enif_keep_resource(void* obj) ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == NIF_RESOURCE_DTOR); - ASSERT(!resource->dbg_is_dying); + ASSERT(!(resource->monitors && resource->monitors->is_dying)); #ifdef DEBUG erts_refc_inc(&resource->nif_refc, 1); #endif @@ -2436,7 +2434,7 @@ void enif_keep_resource(void* obj) Eterm erts_bld_resource_ref(Eterm** hpp, ErlOffHeap* oh, ErtsResource* resource) { ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); - ASSERT(!resource->dbg_is_dying); + ASSERT(!(resource->monitors && resource->monitors->is_dying)); return erts_mk_magic_ref(hpp, oh, &bin->binary); } @@ -2445,6 +2443,7 @@ ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj) ErtsResource* resource = DATA_TO_RESOURCE(obj); ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); Eterm* hp = alloc_heap(env, ERTS_MAGIC_REF_THING_SIZE); + ASSERT(!(resource->monitors && resource->monitors->is_dying)); return erts_mk_magic_ref(&hp, &MSO(env->proc), &bin->binary); } @@ -3136,7 +3135,7 @@ int enif_monitor_process(ErlNifEnv* env, void* obj, const ErlNifPid* target_pid, ASSERT(ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rsrc)->magic_binary.destructor == NIF_RESOURCE_DTOR); - ASSERT(!rsrc->dbg_is_dying); + ASSERT(!(rsrc->monitors && rsrc->monitors->is_dying)); ASSERT(!rsrc->monitors == !rsrc->type->down); @@ -3193,7 +3192,9 @@ int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monit { int scheduler; ErtsResource* rsrc = DATA_TO_RESOURCE(obj); +#ifdef DEBUG ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rsrc); +#endif Process *rp; ErtsMonitor *mon; ErtsMonitor *rmon = NULL; @@ -3202,7 +3203,7 @@ int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monit int is_exiting; ASSERT(bin->magic_binary.destructor == NIF_RESOURCE_DTOR); - ASSERT(!rsrc->dbg_is_dying); + ASSERT(!(rsrc->monitors && rsrc->monitors->is_dying)); execution_state(env, NULL, &scheduler); @@ -3252,8 +3253,7 @@ int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monit #endif } if (is_exiting) { - /* +1 for exiting process */ - erts_refc_inc(&bin->binary.refc, 2); + rsrc->monitors->pending_failed_fire++; } erts_smp_mtx_unlock(&rsrc->monitors->lock); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index d4a1225bdd..776f2c599b 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -87,6 +87,9 @@ typedef struct { erts_smp_mtx_t lock; ErtsMonitor* root; + int pending_failed_fire; + int is_dying; + size_t user_data_sz; } ErtsResourceMonitors; @@ -94,14 +97,11 @@ typedef struct ErtsResource_ { struct enif_resource_type_t* type; ErtsResourceMonitors* monitors; -#ifdef ARCH_32 - byte align__[4]; -#endif #ifdef DEBUG erts_refc_t nif_refc; - int dbg_is_dying; -# ifdef ARCH_64 - byte dbg_align__[4]; +#else +# ifdef ARCH_32 + byte align__[4]; # endif #endif char data[1]; -- cgit v1.2.3 From 28e6aeb8c0cc0e8b8821be957e3ee9c1eb015de6 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 14 Feb 2017 16:41:19 +0100 Subject: Fix enif_select for windows --- erts/emulator/beam/erl_nif.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 9ff28a30cc..ac45f3ac81 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -138,9 +138,11 @@ typedef struct void* ref_bin; }ErlNifBinary; -//#ifndef ERL_SYS_DRV -typedef int ErlNifEvent; /* An event to be selected on. */ -//#endif +#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) +typedef void* ErlNifEvent; /* FIXME: Use 'HANDLE' somehow without breaking existing source */ +#else +typedef int ErlNifEvent; +#endif /* Return bits from enif_select: */ #define ERL_NIF_SELECT_STOP_CALLED (1 << 0) -- cgit v1.2.3 From af7cf70ca22a34add7836963d086ca0764f4fbae Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 20 Feb 2017 20:20:29 +0100 Subject: Fix ErlNifMonitor handling --- erts/emulator/beam/erl_nif.c | 7 ++----- erts/emulator/beam/global.h | 1 + erts/emulator/beam/io.c | 6 +++--- 3 files changed, 6 insertions(+), 8 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index daec4ac9e9..e6da4c1a76 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2305,9 +2305,6 @@ void erts_resource_stop(ErtsResource* resource, ErlNifEvent e, post_nif_noproc(&msg_env); } -/* SVERK SVERK: Move to ?.h */ -void erts_ref_to_driver_monitor(Eterm ref, ErlDrvMonitor*); - void erts_fire_nif_monitor(ErtsResource* resource, Eterm pid, Eterm ref) { ErtsMonitor* rmon; @@ -3207,8 +3204,8 @@ int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monit execution_state(env, NULL, &scheduler); - memcpy(ref_heap, monitor, sizeof(Eterm)*ERTS_REF_THING_SIZE); - ref = make_internal_ref(ref_heap); + ref = erts_driver_monitor_to_ref(ref_heap, monitor); + erts_smp_mtx_lock(&rsrc->monitors->lock); mon = erts_remove_monitor(&rsrc->monitors->root, ref); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 776f2c599b..c4c848f49f 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1176,6 +1176,7 @@ void erts_stale_drv_select(Eterm, ErlDrvPort, ErlDrvEvent, int, int); Port *erts_get_heart_port(void); void erts_emergency_close_ports(void); void erts_ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon); +Eterm erts_driver_monitor_to_ref(Eterm* hp, const ErlDrvMonitor *mon); #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT) void erts_lcnt_enable_io_lock_count(int enable); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 9b525cc100..84dc00f641 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -7615,7 +7615,7 @@ void erts_ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon) ERTS_REF_THING_SIZE*sizeof(Uint)); } -static Eterm driver_monitor_to_ref(Eterm *hp, const ErlDrvMonitor *mon) +Eterm erts_driver_monitor_to_ref(Eterm *hp, const ErlDrvMonitor *mon) { Eterm ref; ERTS_CT_ASSERT(ERTS_REF_THING_SIZE*sizeof(Uint) <= sizeof(ErlDrvMonitor)); @@ -7685,7 +7685,7 @@ static int do_driver_demonitor_process(Port *prt, const ErlDrvMonitor *monitor) ErtsMonitor *mon; Eterm to; - ref = driver_monitor_to_ref(heap, monitor); + ref = erts_driver_monitor_to_ref(heap, monitor); mon = erts_lookup_monitor(ERTS_P_MONITORS(prt), ref); if (mon == NULL) { @@ -7742,7 +7742,7 @@ static ErlDrvTermData do_driver_get_monitored_process(Port *prt,const ErlDrvMoni Eterm to; Eterm heap[ERTS_REF_THING_SIZE]; - ref = driver_monitor_to_ref(heap, monitor); + ref = erts_driver_monitor_to_ref(heap, monitor); mon = erts_lookup_monitor(ERTS_P_MONITORS(prt), ref); if (mon == NULL) { -- cgit v1.2.3 From 3671dd2d426208cf0639530b294da083b5e75acd Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Tue, 21 Feb 2017 09:52:51 +0100 Subject: erts: Emasculate binaries that are scheduled in port_control --- erts/emulator/beam/io.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 31396d06c6..9bf29ee230 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -4884,10 +4884,16 @@ erts_port_control(Process* c_p, ASSERT(!tmp_alloced); if (*ebinp == HEADER_SUB_BIN) ebinp = binary_val(((ErlSubBin *) ebinp)->orig); + if (*ebinp != HEADER_PROC_BIN) copy = 1; else { - binp = ((ProcBin *) ebinp)->val; + ProcBin *pb = (ProcBin *) ebinp; + + if (pb->flags) + erts_emasculate_writable_binary(pb); + + binp = pb->val; ASSERT(bufp <= bufp + size); ASSERT(binp->orig_bytes <= bufp && bufp + size <= binp->orig_bytes + binp->orig_size); -- cgit v1.2.3 From faaf8ffe2902de80c91fbed1e74b062b94edd792 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 21 Feb 2017 15:03:30 +0100 Subject: erts: Fix literal size bug when only old instance exists fix for already merged but not releases 808b2f4d53e446aed07f85716c5c4b85abb3d18a --- erts/emulator/beam/break.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 5d34c83c1a..71934e1376 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -381,10 +381,12 @@ info(fmtfn_t to, void *to_arg) static int code_size(struct erl_module_instance* modi) { - ErtsLiteralArea* lit = modi->code_hdr->literal_area; int size = modi->code_length; - if (lit) { - size += (lit->end - lit->start) * sizeof(Eterm); + + if (modi->code_hdr) { + ErtsLiteralArea* lit = modi->code_hdr->literal_area; + if (lit) + size += (lit->end - lit->start) * sizeof(Eterm); } return size; } -- cgit v1.2.3 From d4e2691d35fc7e1964e58be524d826752001f2a5 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 21 Feb 2017 15:16:59 +0100 Subject: erts: Beautify loaded() by removing some unnecessary conditions and remove unused and faulty summation for 'cur' and 'old'. --- erts/emulator/beam/break.c | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 71934e1376..2efdd1ab89 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -408,13 +408,9 @@ loaded(fmtfn_t to, void *to_arg) * Calculate and print totals. */ for (i = 0; i < module_code_size(code_ix); i++) { - if ((modp = module_code(i, code_ix)) != NULL && - ((modp->curr.code_length != 0) || - (modp->old.code_length != 0))) { + if ((modp = module_code(i, code_ix)) != NULL) { cur += code_size(&modp->curr); - if (modp->old.code_length != 0) { - old += code_size(&modp->old); - } + old += code_size(&modp->old); } } erts_print(to, to_arg, "Current code: %d\n", cur); @@ -430,26 +426,20 @@ loaded(fmtfn_t to, void *to_arg) /* * Interactive dump; keep it brief. */ - if (modp != NULL && - ((modp->curr.code_length != 0) || - (modp->old.code_length != 0))) { - erts_print(to, to_arg, "%T", make_atom(modp->module)); - cur += code_size(&modp->curr); - erts_print(to, to_arg, " %d", code_size(&modp->curr)); - if (modp->old.code_length != 0) { - erts_print(to, to_arg, " (%d old)", - code_size(&modp->old)); - old += code_size(&modp->old); - } + if (modp != NULL && ((modp->curr.code_length != 0) || + (modp->old.code_length != 0))) { + erts_print(to, to_arg, "%T %d", make_atom(modp->module), + code_size(&modp->curr)); + if (modp->old.code_length != 0) + erts_print(to, to_arg, " (%d old)", code_size(&modp->old)); erts_print(to, to_arg, "\n"); } } else { /* * To crash dump; make it parseable. */ - if (modp != NULL && - ((modp->curr.code_length != 0) || - (modp->old.code_length != 0))) { + if (modp != NULL && ((modp->curr.code_length != 0) || + (modp->old.code_length != 0))) { erts_print(to, to_arg, "=mod:"); erts_print(to, to_arg, "%T", make_atom(modp->module)); erts_print(to, to_arg, "\n"); -- cgit v1.2.3 From d8c0f76c800051629cd9cf7279bf075435708a1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Wed, 22 Feb 2017 16:52:29 +0100 Subject: erts: Introduce erts_internal:maps_to_list/2 --- erts/emulator/beam/bif.tab | 1 + erts/emulator/beam/erl_map.c | 66 +++++++++++++++++++++++++++++++++++++++----- 2 files changed, 60 insertions(+), 7 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index a6510ace81..ce2cffa498 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -671,3 +671,4 @@ bif math:floor/1 bif math:ceil/1 bif math:fmod/2 bif os:set_signal/2 +bif erts_internal:maps_to_list/2 diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index 9062e44b10..a80f5d6a16 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -91,7 +91,7 @@ static BIF_RETTYPE hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB, int swap_ static Export hashmap_merge_trap_export; static BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1); static Uint hashmap_subtree_size(Eterm node); -static Eterm hashmap_to_list(Process *p, Eterm map); +static Eterm hashmap_to_list(Process *p, Eterm map, Sint n); static Eterm hashmap_keys(Process *p, Eterm map); static Eterm hashmap_values(Process *p, Eterm map); static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node, Eterm *value); @@ -161,13 +161,58 @@ BIF_RETTYPE maps_to_list_1(BIF_ALIST_1) { BIF_RET(res); } else if (is_hashmap(BIF_ARG_1)) { - return hashmap_to_list(BIF_P, BIF_ARG_1); + return hashmap_to_list(BIF_P, BIF_ARG_1, -1); } BIF_P->fvalue = BIF_ARG_1; BIF_ERROR(BIF_P, BADMAP); } +/* erts_internal:maps_to_list/2 + * + * This function should be removed once iterators are in place. + * Never document it. + * Never encourage its usage. + * + * A negative value in ARG 2 means the entire map. + */ + +BIF_RETTYPE erts_internal_maps_to_list_2(BIF_ALIST_2) { + Sint m; + if (term_to_Sint(BIF_ARG_2, &m)) { + if (is_flatmap(BIF_ARG_1)) { + Uint n; + Eterm* hp; + Eterm *ks,*vs, res, tup; + flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1); + + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + n = flatmap_get_size(mp); + + if (m >= 0) { + n = m < n ? m : n; + } + + hp = HAlloc(BIF_P, (2 + 3) * n); + res = NIL; + + while(n--) { + tup = TUPLE2(hp, ks[n], vs[n]); hp += 3; + res = CONS(hp, tup, res); hp += 2; + } + + BIF_RET(res); + } else if (is_hashmap(BIF_ARG_1)) { + return hashmap_to_list(BIF_P, BIF_ARG_1, m); + } + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); + } + BIF_ERROR(BIF_P, BADARG); +} + + /* maps:find/2 * return value if key *matches* a key in the map */ @@ -1917,15 +1962,22 @@ BIF_RETTYPE maps_values_1(BIF_ALIST_1) { BIF_ERROR(BIF_P, BADMAP); } -static Eterm hashmap_to_list(Process *p, Eterm node) { +static Eterm hashmap_to_list(Process *p, Eterm node, Sint m) { DECLARE_WSTACK(stack); Eterm *hp, *kv; - Eterm res = NIL; + Eterm tup, res = NIL; + Uint n = hashmap_size(node); - hp = HAlloc(p, hashmap_size(node) * (2 + 3)); + if (m >= 0) { + n = m < n ? m : n; + } + + hp = HAlloc(p, n * (2 + 3)); hashmap_iterator_init(&stack, node, 0); - while ((kv=hashmap_iterator_next(&stack)) != NULL) { - Eterm tup = TUPLE2(hp, CAR(kv), CDR(kv)); + while (n--) { + kv = hashmap_iterator_next(&stack); + ASSERT(kv != NULL); + tup = TUPLE2(hp, CAR(kv), CDR(kv)); hp += 3; res = CONS(hp, tup, res); hp += 2; -- cgit v1.2.3 From 0e566efe903fd2482c51762f1939bc292dca43e7 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 23 Feb 2017 19:26:07 +0100 Subject: Remove debug printout and comment --- erts/emulator/beam/break.c | 1 - 1 file changed, 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 787f1e4b7e..6780c28580 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -196,7 +196,6 @@ static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext) break; case MON_NIF_TARGET: { ErtsResource* rsrc = mon->u.resource; - /*SVERK: Print resource-ref? */ erts_print(to, to_arg, "%s{from,{%T,%T},%T}", prefix, rsrc->type->module, rsrc->type->name, mon->ref); break; -- cgit v1.2.3 From cfe4b5c4aaa0d86f9d98095515ae6582a053cc47 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Mon, 27 Feb 2017 10:11:47 +0100 Subject: erts: Fix emasculation of binaries in port_control This fixes a bug introduced in 3671dd2d --- erts/emulator/beam/io.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 9bf29ee230..38e403d91a 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -4889,10 +4889,18 @@ erts_port_control(Process* c_p, copy = 1; else { ProcBin *pb = (ProcBin *) ebinp; + int offset = bufp - pb->val->orig_bytes; - if (pb->flags) + ASSERT(pb->val->orig_bytes <= bufp + && bufp + size <= pb->val->orig_bytes + pb->val->orig_size); + + if (pb->flags) { erts_emasculate_writable_binary(pb); + /* The procbin may have been reallocated, so update bufp */ + bufp = pb->val->orig_bytes + offset; + } + binp = pb->val; ASSERT(bufp <= bufp + size); ASSERT(binp->orig_bytes <= bufp -- cgit v1.2.3 From c0f82eef818a71a8cb475d9b27a3c30074fe3b77 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Mon, 27 Feb 2017 12:01:50 +0100 Subject: erts: Fix proc arity value when exiting Not setting it correctly will cause the swapin later to fail in debug build as REDS_IN will overwrite the value in reg[5]. --- erts/emulator/beam/beam_emu.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 49f932a1c2..8be0f58227 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1324,6 +1324,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) goto do_schedule1; do_schedule: + ASSERT(c_p->arity < 6); ASSERT(c_p->debug_reds_in == REDS_IN(c_p)); if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) reds_used = REDS_IN(c_p) - FCALLS; @@ -1924,6 +1925,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); SWAPOUT; c_p->flags &= ~F_DELAY_GC; + c_p->arity = 0; goto do_schedule; /* Will be rescheduled for exit */ } ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); @@ -2166,6 +2168,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) * in limbo forever. */ SWAPOUT; + c_p->arity = 0; goto do_schedule; } #endif -- cgit v1.2.3 From 4bbbf4ff9b3fe4be24bf8d57780c84d5b3ca0f77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Mon, 27 Feb 2017 15:27:05 +0100 Subject: erts: Convert small sub-binaries to heap-binaries In many cases sub-binaries costs more memory than converting them to heap-binaries. Sub-binaries also has a hidden cost of pinning larger binaries in memory. By converting binaries this cost is reduced. Byte aligned sub-binaries upto 24 bytes (64-bit) or 12 bytes (32-bit) are converted. --- erts/emulator/beam/erl_binary.h | 18 +---------- erts/emulator/beam/erl_bits.h | 17 ++++++++++ erts/emulator/beam/erl_gc.c | 33 +++++++++++++++++++ erts/emulator/beam/erl_gc.h | 70 ++++++++++++++++++++++++++++------------- 4 files changed, 100 insertions(+), 38 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index 946d3cfe03..6ff71ec6d1 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -156,6 +156,7 @@ typedef union { #include "erl_threads.h" #include "bif.h" #include "erl_bif_unique.h" +#include "erl_bits.h" /* * Maximum number of bytes to place in a heap binary. @@ -163,23 +164,6 @@ typedef union { #define ERL_ONHEAP_BIN_LIMIT 64 -/* - * This structure represents a SUB_BINARY. - * - * Note: The last field (orig) is not counted in arityval in the header. - * This simplifies garbage collection. - */ - -typedef struct erl_sub_bin { - Eterm thing_word; /* Subtag SUB_BINARY_SUBTAG. */ - Uint size; /* Binary size in bytes. */ - Uint offs; /* Offset into original binary. */ - byte bitsize; - byte bitoffs; - byte is_writable; /* The underlying binary is writable */ - Eterm orig; /* Original binary (REFC or HEAP binary). */ -} ErlSubBin; - #define ERL_SUB_BIN_SIZE (sizeof(ErlSubBin)/sizeof(Eterm)) #define HEADER_SUB_BIN _make_header(ERL_SUB_BIN_SIZE-2,_TAG_HEADER_SUB_BIN) diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h index 4bd5b24157..1b4546722f 100644 --- a/erts/emulator/beam/erl_bits.h +++ b/erts/emulator/beam/erl_bits.h @@ -21,6 +21,23 @@ #ifndef __ERL_BITS_H__ #define __ERL_BITS_H__ +/* + * This structure represents a SUB_BINARY. + * + * Note: The last field (orig) is not counted in arityval in the header. + * This simplifies garbage collection. + */ + +typedef struct erl_sub_bin { + Eterm thing_word; /* Subtag SUB_BINARY_SUBTAG. */ + Uint size; /* Binary size in bytes. */ + Uint offs; /* Offset into original binary. */ + byte bitsize; + byte bitoffs; + byte is_writable; /* The underlying binary is writable */ + Eterm orig; /* Original binary (REFC or HEAP binary). */ +} ErlSubBin; + /* * This structure represents a binary to be matched. */ diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 8cc7fc0142..26f6b6339d 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -3256,6 +3256,39 @@ reply_gc_info(void *vgcirp) gcireq_free(vgcirp); } +void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig) { + Eterm *ptr = *pp; + Eterm *htop = *hpp; + Eterm gval; + ErlSubBin *sb = (ErlSubBin *)ptr; + ErlHeapBin *hb = (ErlHeapBin *)htop; + Eterm *real_bin; + byte *bs; + + real_bin = binary_val(follow_moved(sb->orig, (Eterm)0)); + + if (*real_bin == HEADER_PROC_BIN) { + bs = ((ProcBin *) real_bin)->bytes + sb->offs; + } else { + bs = (byte *)(&(((ErlHeapBin *) real_bin)->data)) + sb->offs; + } + + hb->thing_word = header_heap_bin(sb->size); + hb->size = sb->size; + sys_memcpy((byte *)hb->data, bs, sb->size); + + gval = make_boxed(htop); + *orig = gval; + *ptr = gval; + + ptr += ERL_SUB_BIN_SIZE; + htop += heap_bin_size(sb->size); + + *hpp = htop; + *pp = ptr; +} + + Eterm erts_gc_info_request(Process *c_p) { diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 1cce426d21..2e2be2ee47 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -28,6 +28,8 @@ #define ERTS_POTENTIALLY_LONG_GC_HSIZE (128*1024) /* Words */ #include "erl_map.h" +#include "erl_fun.h" +#include "erl_bits.h" #define IS_MOVED_BOXED(x) (!is_header((x))) #define IS_MOVED_CONS(x) (is_non_value((x))) @@ -45,27 +47,53 @@ do { \ HTOP += 2; /* update tospace htop */ \ } while(0) -#define MOVE_BOXED(PTR,HDR,HTOP,ORIG) \ -do { \ - Eterm gval; \ - Sint nelts; \ - \ - ASSERT(is_header(HDR)); \ - nelts = header_arity(HDR); \ - switch ((HDR) & _HEADER_SUBTAG_MASK) { \ - case SUB_BINARY_SUBTAG: nelts++; break; \ - case MAP_SUBTAG: \ - if (is_flatmap_header(HDR)) nelts+=flatmap_get_size(PTR) + 1; \ - else nelts += hashmap_bitcount(MAP_HEADER_VAL(HDR)); \ - break; \ - case FUN_SUBTAG: nelts+=((ErlFunThing*)(PTR))->num_free+1; break; \ - } \ - gval = make_boxed(HTOP); \ - *ORIG = gval; \ - *HTOP++ = HDR; \ - *PTR++ = gval; \ - while (nelts--) *HTOP++ = *PTR++; \ -} while(0) +#define MOVE_BOXED(PTR,HDR,HTOP,ORIG) \ + do { \ + move_boxed(&(PTR), HDR, &(HTOP), ORIG); \ + } while(0) + +void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig); +ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig) +{ + Eterm gval; + Sint nelts; + Eterm *ptr = *pp; + Eterm *htop = *hpp; + + ASSERT(is_header(hdr)); + nelts = header_arity(hdr); + switch ((hdr) & _HEADER_SUBTAG_MASK) { + case SUB_BINARY_SUBTAG: + { + ErlSubBin *sb = (ErlSubBin *)ptr; + /* convert sub-binary to heap-binary if applicable */ + if (sb->bitsize == 0 && sb->bitoffs == 0 && + sb->is_writable == 0 && sb->size <= sizeof(Eterm) * 3) { + erts_sub_binary_to_heap_binary(pp, hpp, orig); + return; + } + } + nelts++; + break; + case MAP_SUBTAG: + if (is_flatmap_header(hdr)) nelts+=flatmap_get_size(ptr) + 1; + else nelts += hashmap_bitcount(MAP_HEADER_VAL(hdr)); + break; + case FUN_SUBTAG: nelts+=((ErlFunThing*)(ptr))->num_free+1; break; + } + gval = make_boxed(htop); + *orig = gval; + *htop++ = hdr; + *ptr++ = gval; + while (nelts--) *htop++ = *ptr++; + + *hpp = htop; + *pp = ptr; +} +#endif #define ErtsInYoungGen(TPtr, Ptr, OldHeap, OldHeapSz) \ (!erts_is_literal((TPtr), (Ptr)) \ -- cgit v1.2.3 From 6b51f402e4e656776565233021edcea8f4d1590b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Tue, 28 Feb 2017 15:22:35 +0100 Subject: erts: Refactor MOVE_BOXED to inline function --- erts/emulator/beam/copy.c | 2 +- erts/emulator/beam/erl_gc.c | 30 +++++++++++++++--------------- erts/emulator/beam/erl_gc.h | 5 ----- 3 files changed, 16 insertions(+), 21 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 85db23393c..a8e37c0173 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -1990,7 +1990,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite if (is_header(val)) { struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp; ASSERT(ptr + header_arity(val) < end); - MOVE_BOXED(ptr, val, hp, &dummy_ref); + move_boxed(&ptr, val, &hp, &dummy_ref); switch (val & _HEADER_SUBTAG_MASK) { case REF_SUBTAG: if (is_ordinary_ref_thing(hdr)) diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 26f6b6339d..a8c8bf3db5 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1173,7 +1173,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, ASSERT(is_boxed(val)); *g_ptr++ = val; } else if (ErtsInArea(ptr, area, area_size)) { - MOVE_BOXED(ptr,val,old_htop,g_ptr++); + move_boxed(&ptr,val,&old_htop,g_ptr++); } else { g_ptr++; } @@ -1479,9 +1479,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, ASSERT(is_boxed(val)); *g_ptr++ = val; } else if (ErtsInArea(ptr, mature, mature_size)) { - MOVE_BOXED(ptr,val,old_htop,g_ptr++); + move_boxed(&ptr,val,&old_htop,g_ptr++); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - MOVE_BOXED(ptr,val,n_htop,g_ptr++); + move_boxed(&ptr,val,&n_htop,g_ptr++); } else { g_ptr++; } @@ -1538,9 +1538,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, ASSERT(is_boxed(val)); *n_hp++ = val; } else if (ErtsInArea(ptr, mature, mature_size)) { - MOVE_BOXED(ptr,val,old_htop,n_hp++); + move_boxed(&ptr,val,&old_htop,n_hp++); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - MOVE_BOXED(ptr,val,n_htop,n_hp++); + move_boxed(&ptr,val,&n_htop,n_hp++); } else { n_hp++; } @@ -1574,10 +1574,10 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, *origptr = val; mb->base = binary_bytes(val); } else if (ErtsInArea(ptr, mature, mature_size)) { - MOVE_BOXED(ptr,val,old_htop,origptr); + move_boxed(&ptr,val,&old_htop,origptr); mb->base = binary_bytes(mb->orig); } else if (ErtsInYoungGen(*origptr, ptr, oh, oh_size)) { - MOVE_BOXED(ptr,val,n_htop,origptr); + move_boxed(&ptr,val,&n_htop,origptr); mb->base = binary_bytes(mb->orig); } } @@ -1792,7 +1792,7 @@ full_sweep_heaps(Process *p, Eterm* ptr; Eterm val; Eterm gval = *g_ptr; - + switch (primary_tag(gval)) { case TAG_PRIMARY_BOXED: { @@ -1802,7 +1802,7 @@ full_sweep_heaps(Process *p, ASSERT(is_boxed(val)); *g_ptr++ = val; } else if (!erts_is_literal(gval, ptr)) { - MOVE_BOXED(ptr,val,n_htop,g_ptr++); + move_boxed(&ptr,val,&n_htop,g_ptr++); } else { g_ptr++; } @@ -2104,7 +2104,7 @@ sweep(Eterm *n_hp, Eterm *n_htop, ASSERT(is_boxed(val)); *n_hp++ = val; } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) { - MOVE_BOXED(ptr,val,n_htop,n_hp++); + move_boxed(&ptr,val,&n_htop,n_hp++); } else { n_hp++; } @@ -2129,7 +2129,7 @@ sweep(Eterm *n_hp, Eterm *n_htop, if (header_is_bin_matchstate(gval)) { ErlBinMatchState *ms = (ErlBinMatchState*) n_hp; ErlBinMatchBuffer *mb = &(ms->mb); - Eterm* origptr; + Eterm* origptr; origptr = &(mb->orig); ptr = boxed_val(*origptr); val = *ptr; @@ -2137,7 +2137,7 @@ sweep(Eterm *n_hp, Eterm *n_htop, *origptr = val; mb->base = binary_bytes(*origptr); } else if (ERTS_IS_IN_SWEEP_AREA(*origptr, ptr)) { - MOVE_BOXED(ptr,val,n_htop,origptr); + move_boxed(&ptr,val,&n_htop,origptr); mb->base = binary_bytes(*origptr); } } @@ -2200,7 +2200,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, ASSERT(is_boxed(val)); *heap_ptr++ = val; } else if (ErtsInArea(ptr, src, src_size)) { - MOVE_BOXED(ptr,val,htop,heap_ptr++); + move_boxed(&ptr,val,&htop,heap_ptr++); } else { heap_ptr++; } @@ -2233,7 +2233,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, *origptr = val; mb->base = binary_bytes(*origptr); } else if (ErtsInArea(ptr, src, src_size)) { - MOVE_BOXED(ptr,val,htop,origptr); + move_boxed(&ptr,val,&htop,origptr); mb->base = binary_bytes(*origptr); } } @@ -2266,7 +2266,7 @@ move_one_area(Eterm* n_htop, char* src, Uint src_size) ASSERT(val != ERTS_HOLE_MARKER); if (is_header(val)) { ASSERT(ptr + header_arity(val) < end); - MOVE_BOXED(ptr, val, n_htop, &dummy_ref); + move_boxed(&ptr, val, &n_htop, &dummy_ref); } else { /* must be a cons cell */ ASSERT(ptr+1 < end); diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 2e2be2ee47..8e09904766 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -47,11 +47,6 @@ do { \ HTOP += 2; /* update tospace htop */ \ } while(0) -#define MOVE_BOXED(PTR,HDR,HTOP,ORIG) \ - do { \ - move_boxed(&(PTR), HDR, &(HTOP), ORIG); \ - } while(0) - void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig); ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig); -- cgit v1.2.3 From 3e638439e80e388aa66a2047735a47c954cd70c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Tue, 28 Feb 2017 15:32:58 +0100 Subject: erts: Refactor MOVE_CONS to inline function --- erts/emulator/beam/copy.c | 2 +- erts/emulator/beam/erl_gc.c | 18 +++++++++--------- erts/emulator/beam/erl_gc.h | 32 ++++++++++++++++++-------------- 3 files changed, 28 insertions(+), 24 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index a8e37c0173..e567eabc82 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -2007,7 +2007,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite } else { /* must be a cons cell */ ASSERT(ptr+1 < end); - MOVE_CONS(ptr, val, hp, &dummy_ref); + move_cons(&ptr, val, &hp, &dummy_ref); ptr += 2; } } diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index a8c8bf3db5..50805d9cd9 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1184,7 +1184,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, if (IS_MOVED_CONS(val)) { /* Moved */ *g_ptr++ = ptr[1]; } else if (ErtsInArea(ptr, area, area_size)) { - MOVE_CONS(ptr,val,old_htop,g_ptr++); + move_cons(&ptr,val,&old_htop,g_ptr++); } else { g_ptr++; } @@ -1494,9 +1494,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, if (IS_MOVED_CONS(val)) { /* Moved */ *g_ptr++ = ptr[1]; } else if (ErtsInArea(ptr, mature, mature_size)) { - MOVE_CONS(ptr,val,old_htop,g_ptr++); + move_cons(&ptr,val,&old_htop,g_ptr++); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - MOVE_CONS(ptr,val,n_htop,g_ptr++); + move_cons(&ptr,val,&n_htop,g_ptr++); } else { g_ptr++; } @@ -1552,9 +1552,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; } else if (ErtsInArea(ptr, mature, mature_size)) { - MOVE_CONS(ptr,val,old_htop,n_hp++); + move_cons(&ptr,val,&old_htop,n_hp++); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - MOVE_CONS(ptr,val,n_htop,n_hp++); + move_cons(&ptr,val,&n_htop,n_hp++); } else { n_hp++; } @@ -1815,7 +1815,7 @@ full_sweep_heaps(Process *p, if (IS_MOVED_CONS(val)) { *g_ptr++ = ptr[1]; } else if (!erts_is_literal(gval, ptr)) { - MOVE_CONS(ptr,val,n_htop,g_ptr++); + move_cons(&ptr,val,&n_htop,g_ptr++); } else { g_ptr++; } @@ -2116,7 +2116,7 @@ sweep(Eterm *n_hp, Eterm *n_htop, if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) { - MOVE_CONS(ptr,val,n_htop,n_hp++); + move_cons(&ptr,val,&n_htop,n_hp++); } else { n_hp++; } @@ -2212,7 +2212,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, if (IS_MOVED_CONS(val)) { *heap_ptr++ = ptr[1]; } else if (ErtsInArea(ptr, src, src_size)) { - MOVE_CONS(ptr,val,htop,heap_ptr++); + move_cons(&ptr,val,&htop,heap_ptr++); } else { heap_ptr++; } @@ -2270,7 +2270,7 @@ move_one_area(Eterm* n_htop, char* src, Uint src_size) } else { /* must be a cons cell */ ASSERT(ptr+1 < end); - MOVE_CONS(ptr, val, n_htop, &dummy_ref); + move_cons(&ptr, val, &n_htop, &dummy_ref); ptr += 2; } } diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 8e09904766..414aff1e06 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -33,23 +33,27 @@ #define IS_MOVED_BOXED(x) (!is_header((x))) #define IS_MOVED_CONS(x) (is_non_value((x))) +void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig); + +ERTS_GLB_INLINE void move_cons(Eterm **pp, Eterm car, Eterm **hpp, Eterm *orig); +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE void move_cons(Eterm **pp, Eterm car, Eterm **hpp, Eterm *orig) +{ + Eterm *ptr = *pp; + Eterm *htop = *hpp; + Eterm gval; -#define MOVE_CONS(PTR,CAR,HTOP,ORIG) \ -do { \ - Eterm gval; \ - \ - HTOP[0] = CAR; /* copy car */ \ - HTOP[1] = PTR[1]; /* copy cdr */ \ - gval = make_list(HTOP); /* new location */ \ - *ORIG = gval; /* redirect original reference */ \ - PTR[0] = THE_NON_VALUE; /* store forwarding indicator */ \ - PTR[1] = gval; /* store forwarding address */ \ - HTOP += 2; /* update tospace htop */ \ -} while(0) + htop[0] = car; /* copy car */ + htop[1] = ptr[1]; /* copy cdr */ + gval = make_list(htop); /* new location */ + *orig = gval; /* redirect original reference */ + ptr[0] = THE_NON_VALUE; /* store forwarding indicator */ + ptr[1] = gval; /* store forwarding address */ + *hpp += 2; /* update tospace htop */ +} +#endif -void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig); ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig); - #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig) { -- cgit v1.2.3 From 0d6dc895744c34c9c52fd42f4801a8a941864ae3 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 13 Feb 2017 18:22:39 +0100 Subject: Introduce references as table identifiers --- erts/emulator/beam/erl_db.c | 98 ++++++++++++++++++++++++++++++++++++++-- erts/emulator/beam/erl_db_util.h | 2 + 2 files changed, 96 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 9f077dd407..d886c2546a 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -43,6 +43,7 @@ #include "erl_db.h" #include "bif.h" #include "big.h" +#include "erl_binary.h" erts_smp_atomic_t erts_ets_misc_mem_size; @@ -113,6 +114,75 @@ static struct { #define MARK_SLOT_DEAD(i) (meta_main_tab[(i)].u.next_free |= 2) #define GET_ANY_SLOT_TAB(i) ((DbTable*)(meta_main_tab[(i)].u.next_free & ~(1|2))) /* dead or alive */ +static void schedule_free_dbtable(DbTable* tb); + +static void table_dec_refc(DbTable *tb, erts_aint_t min_val) +{ + if (erts_smp_refc_dectest(&tb->common.refc, min_val) == 0) + schedule_free_dbtable(tb); +} + +static int +db_table_tid_destructor(Binary *unused) +{ + return 1; +} + +static ERTS_INLINE void +make_btid(DbTable *tb) +{ + Binary *btid = erts_create_magic_indirection(db_table_tid_destructor); + erts_smp_atomic_t *tbref = erts_smp_binary_to_magic_indirection(btid); + erts_smp_atomic_init_nob(tbref, (erts_aint_t) tb); + tb->common.btid = btid; + /* + * Table and magic indirection refer eachother, + * and table is refered once by being alive... + */ + erts_smp_refc_init(&tb->common.refc, 2); + erts_refc_inc(&btid->refc, 1); +} + +static DbTable * +tid2tab(Eterm tid) +{ + DbTable *tb; + Binary *btid; + erts_smp_atomic_t *tbref; + if (!is_internal_magic_ref(tid)) + return NULL; + + btid = erts_magic_ref2bin(tid); + if (ERTS_MAGIC_BIN_DESTRUCTOR(btid) != db_table_tid_destructor) + return NULL; + + tbref = erts_smp_binary_to_magic_indirection(btid); + tb = (DbTable *) erts_smp_atomic_read_nob(tbref); + + ASSERT(!tb || tb->common.btid == btid); + + return tb; +} + +static ERTS_INLINE void +tid_clear(DbTable *tb) +{ + DbTable *rtb; + Binary *btid = tb->common.btid; + erts_smp_atomic_t *tbref = erts_smp_binary_to_magic_indirection(btid); + rtb = (DbTable *) erts_smp_atomic_xchg_nob(tbref, (erts_aint_t) NULL); + ASSERT(!rtb || tb == rtb); + if (rtb) + table_dec_refc(tb, 1); +} + +static ERTS_INLINE Eterm +make_tid(Process *c_p, DbTable *tb) +{ + Eterm *hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE); + return erts_mk_magic_ref(&hp, &c_p->off_heap, tb->common.btid); +} + static ERTS_INLINE erts_smp_rwmtx_t * get_meta_main_tab_lock(unsigned slot) { @@ -252,6 +322,10 @@ free_dbtable(void *vtb) erts_smp_mtx_destroy(&tb->common.fixlock); #endif ASSERT(is_immed(tb->common.heir_data)); + + if (tb->common.btid && erts_refc_dectest(&tb->common.btid->refc, 0) == 0) + erts_bin_free(tb->common.btid); + erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable)); } @@ -266,6 +340,7 @@ static void schedule_free_dbtable(DbTable* tb) * Caller is *not* allowed to access the specialized part * (hash or tree) of *tb after this function has returned. */ + ASSERT(erts_smp_refc_read(&tb->common.refc, 0) == 0); ASSERT(erts_smp_refc_read(&tb->common.ref, 0) == 0); erts_schedule_thr_prgr_later_cleanup_op(free_dbtable, (void *) tb, @@ -375,7 +450,7 @@ DbTable* db_get_table_aux(Process *p, db_lock_kind_t kind, int meta_already_locked) { - DbTable *tb = NULL; + DbTable *tb; erts_smp_rwmtx_t *mtl = NULL; /* @@ -385,7 +460,10 @@ DbTable* db_get_table_aux(Process *p, */ ASSERT(erts_get_scheduler_data()); - if (is_small(id)) { + tb = tid2tab(id); + if (tb) + id = tb->common.id; + else if (is_small(id)) { Uint slot = unsigned_val(id) & meta_main_tab_slot_mask; if (!meta_already_locked) { mtl = get_meta_main_tab_lock(slot); @@ -1505,6 +1583,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) meth->db_create(BIF_P, tb); ASSERT(cret == DB_ERROR_NONE); + make_btid(tb); + erts_smp_spin_lock(&meta_main_tab_main_lock); if (meta_main_tab_cnt >= db_max_tabs) { @@ -1539,6 +1619,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) tb->common.id = ret; tb->common.slot = slot; /* store slot for erase */ + if (!is_named) + ret = make_tid(BIF_P, tb); + mmtl = get_meta_main_tab_lock(slot); erts_smp_rwmtx_rwlock(mmtl); meta_main_tab[slot].u.tb = tb; @@ -1551,11 +1634,13 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) free_slot(slot); erts_smp_rwmtx_rwunlock(mmtl); + tid_clear(tb); + db_lock(tb,LCK_WRITE); free_heir_data(tb); tb->common.meth->db_free_table(tb); - schedule_free_dbtable(tb); db_unlock(tb,LCK_WRITE); + table_dec_refc(tb, 0); BIF_ERROR(BIF_P, BADARG); } @@ -1740,6 +1825,8 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) UnUseTmpHeap(3,BIF_P); } + tid_clear(tb); + mmtl = get_meta_main_tab_lock(tb->common.slot); #ifdef ERTS_SMP if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) { @@ -2969,6 +3056,7 @@ void init_db(ErtsDbSpinCount db_spin_count) erts_smp_atomic_read_nob(&init_tb.common.memory_size)); meta_pid_to_tab->common.id = NIL; + meta_pid_to_tab->common.btid = NULL; meta_pid_to_tab->common.the_name = am_true; meta_pid_to_tab->common.status = (DB_NORMAL | DB_BAG | DB_PUBLIC | DB_FINE_LOCKED); #ifdef ERTS_SMP @@ -3000,6 +3088,7 @@ void init_db(ErtsDbSpinCount db_spin_count) erts_smp_atomic_read_nob(&init_tb.common.memory_size)); meta_pid_to_fixed_tab->common.id = NIL; + meta_pid_to_fixed_tab->common.btid = NULL; meta_pid_to_fixed_tab->common.the_name = am_true; meta_pid_to_fixed_tab->common.status = (DB_NORMAL | DB_BAG | DB_PUBLIC | DB_FINE_LOCKED); #ifdef ERTS_SMP @@ -3295,6 +3384,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) #endif first_call = (tb->common.status & DB_DELETE) == 0; if (first_call) { + tid_clear(tb); /* Clear all access bits. */ tb->common.status &= ~(DB_PROTECTED | DB_PUBLIC @@ -3707,7 +3797,7 @@ static int free_table_cont(Process *p, make_small(tb->common.slot)); db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); } - schedule_free_dbtable(tb); + table_dec_refc(tb, 0); BUMP_REDS(p, 100); return 0; } diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 471fefe3cb..3f507b559f 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -212,6 +212,7 @@ typedef struct db_fixation { */ typedef struct db_table_common { + erts_smp_refc_t refc; /* reference count of table struct */ erts_smp_refc_t ref; /* fixation counter */ #ifdef ERTS_SMP erts_smp_rwmtx_t rwlock; /* rw lock on table */ @@ -225,6 +226,7 @@ typedef struct db_table_common { Uint64 heir_started_interval; /* To further identify the heir */ Eterm the_name; /* an atom */ Eterm id; /* atom | integer */ + Binary *btid; DbTableMethod* meth; /* table methods */ erts_smp_atomic_t nitems; /* Total number of items in table */ erts_smp_atomic_t memory_size;/* Total memory size. NOTE: in bytes! */ -- cgit v1.2.3 From ee7494d5d5c3388a8bb1f012f6411a90f89d8ea6 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 13 Feb 2017 18:36:00 +0100 Subject: Rename fixation count in ets table to avoid confusion --- erts/emulator/beam/erl_db.c | 20 ++++++++++---------- erts/emulator/beam/erl_db_util.h | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index d886c2546a..ad11d72d5f 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -341,7 +341,7 @@ static void schedule_free_dbtable(DbTable* tb) * (hash or tree) of *tb after this function has returned. */ ASSERT(erts_smp_refc_read(&tb->common.refc, 0) == 0); - ASSERT(erts_smp_refc_read(&tb->common.ref, 0) == 0); + ASSERT(erts_smp_refc_read(&tb->common.fix_count, 0) == 0); erts_schedule_thr_prgr_later_cleanup_op(free_dbtable, (void *) tb, &tb->release.data, @@ -678,11 +678,11 @@ done: */ static ERTS_INLINE void local_fix_table(DbTable* tb) { - erts_smp_refc_inc(&tb->common.ref, 1); + erts_smp_refc_inc(&tb->common.fix_count, 1); } static ERTS_INLINE void local_unfix_table(DbTable* tb) { - if (erts_smp_refc_dectest(&tb->common.ref, 0) == 0) { + if (erts_smp_refc_dectest(&tb->common.fix_count, 0) == 0) { ASSERT(IS_HASH_TABLE(tb->common.status)); db_unfix_table_hash(&(tb->hash)); } @@ -1565,7 +1565,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) tb->common.type = status & ERTS_ETS_TABLE_TYPES; /* Note, 'type' is *read only* from now on... */ #endif - erts_smp_refc_init(&tb->common.ref, 0); + erts_smp_refc_init(&tb->common.fix_count, 0); db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ), "db_tab", "db_tab_fix"); tb->common.keypos = keypos; @@ -3072,7 +3072,7 @@ void init_db(ErtsDbSpinCount db_spin_count) meta_pid_to_tab->common.meth = &db_hash; meta_pid_to_tab->common.compress = 0; - erts_smp_refc_init(&meta_pid_to_tab->common.ref, 0); + erts_smp_refc_init(&meta_pid_to_tab->common.fix_count, 0); /* Neither rwlock or fixlock used db_init_lock(meta_pid_to_tab, "meta_pid_to_tab", "meta_pid_to_tab_FIX");*/ @@ -3104,7 +3104,7 @@ void init_db(ErtsDbSpinCount db_spin_count) meta_pid_to_fixed_tab->common.meth = &db_hash; meta_pid_to_fixed_tab->common.compress = 0; - erts_smp_refc_init(&meta_pid_to_fixed_tab->common.ref, 0); + erts_smp_refc_init(&meta_pid_to_fixed_tab->common.fix_count, 0); /* Neither rwlock or fixlock used db_init_lock(meta_pid_to_fixed_tab, "meta_pid_to_fixed_tab", "meta_pid_to_fixed_tab_FIX");*/ @@ -3466,7 +3466,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) if ((*pp)->pid == pid) { DbFixation* fix = *pp; erts_aint_t diff = -((erts_aint_t) fix->counter); - erts_smp_refc_add(&tb->common.ref,diff,0); + erts_smp_refc_add(&tb->common.fix_count,diff,0); *pp = fix->next; erts_db_free(ERTS_ALC_T_DB_FIXATION, tb, fix, sizeof(DbFixation)); @@ -3542,7 +3542,7 @@ static void fix_table_locked(Process* p, DbTable* tb) #ifdef ERTS_SMP erts_smp_mtx_lock(&tb->common.fixlock); #endif - erts_smp_refc_inc(&tb->common.ref,1); + erts_smp_refc_inc(&tb->common.fix_count,1); fix = tb->common.fixations; if (fix == NULL) { tb->common.time.monotonic @@ -3598,7 +3598,7 @@ static void unfix_table_locked(Process* p, DbTable* tb, for (pp = &tb->common.fixations; *pp != NULL; pp = &(*pp)->next) { if ((*pp)->pid == p->common.id) { DbFixation* fix = *pp; - erts_smp_refc_dec(&tb->common.ref,0); + erts_smp_refc_dec(&tb->common.fix_count,0); --(fix->counter); ASSERT(fix->counter >= 0); if (fix->counter > 0) { @@ -3647,7 +3647,7 @@ static void free_fixations_locked(DbTable *tb) fix = tb->common.fixations; while (fix != NULL) { erts_aint_t diff = -((erts_aint_t) fix->counter); - erts_smp_refc_add(&tb->common.ref,diff,0); + erts_smp_refc_add(&tb->common.fix_count,diff,0); next_fix = fix->next; db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC); db_erase_bag_exact2(meta_pid_to_fixed_tab, diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 3f507b559f..60dedd5713 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -213,7 +213,7 @@ typedef struct db_fixation { typedef struct db_table_common { erts_smp_refc_t refc; /* reference count of table struct */ - erts_smp_refc_t ref; /* fixation counter */ + erts_smp_refc_t fix_count;/* fixation counter */ #ifdef ERTS_SMP erts_smp_rwmtx_t rwlock; /* rw lock on table */ erts_smp_mtx_t fixlock; /* Protects fixations,megasec,sec,microsec */ @@ -263,7 +263,7 @@ typedef struct db_table_common { (DB_BAG | DB_SET | DB_DUPLICATE_BAG))) #define IS_TREE_TABLE(Status) (!!((Status) & \ DB_ORDERED_SET)) -#define NFIXED(T) (erts_smp_refc_read(&(T)->common.ref,0)) +#define NFIXED(T) (erts_smp_refc_read(&(T)->common.fix_count,0)) #define IS_FIXED(T) (NFIXED(T) != 0) /* -- cgit v1.2.3 From d1ad45f0940697f04f334c078a2287cd51e45ad5 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 17 Feb 2017 18:23:11 +0100 Subject: Implement ets:all() using scheduler specific data --- erts/emulator/beam/bif.tab | 2 +- erts/emulator/beam/break.c | 2 + erts/emulator/beam/erl_alloc.types | 1 + erts/emulator/beam/erl_db.c | 559 +++++++++++++++++++++++++++++++------ erts/emulator/beam/erl_db.h | 42 ++- erts/emulator/beam/erl_db_util.h | 9 +- erts/emulator/beam/erl_init.c | 1 + erts/emulator/beam/erl_process.c | 50 ++++ erts/emulator/beam/erl_process.h | 16 +- 9 files changed, 590 insertions(+), 92 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index a6510ace81..08c8c3adbd 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -324,7 +324,7 @@ bif erlang:match_spec_test/3 # Bifs in ets module. # -bif ets:all/0 +bif ets:internal_request_all/0 bif ets:new/2 bif ets:delete/1 bif ets:delete/2 diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 4dad5736c5..0b40d70cb7 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -512,6 +512,8 @@ do_break(void) erts_free_read_env(mode); #endif /* __WIN32__ */ + ASSERT(erts_smp_thr_progress_is_blocking()); + erts_printf("\n" "BREAK: (a)bort (c)ontinue (p)roc info (i)nfo (l)oaded\n" " (v)ersion (k)ill (D)b-tables (d)istribution\n"); diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 3d5de72ee7..32f84c8593 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -227,6 +227,7 @@ type DB_DMC_ERROR ETS ETS db_dmc_error type DB_DMC_ERR_INFO ETS ETS db_dmc_error_info type DB_TERM ETS ETS db_term type DB_PROC_CLEANUP SHORT_LIVED ETS db_proc_cleanup_state +type ETS_ALL_REQ SHORT_LIVED ETS ets_all_request type INSTR_INFO LONG_LIVED SYSTEM instr_info type LOGGER_DSBUF TEMPORARY SYSTEM logger_dsbuf type TMP_DSBUF TEMPORARY SYSTEM tmp_dsbuf diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index ad11d72d5f..8c4c76e187 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -114,7 +114,12 @@ static struct { #define MARK_SLOT_DEAD(i) (meta_main_tab[(i)].u.next_free |= 2) #define GET_ANY_SLOT_TAB(i) ((DbTable*)(meta_main_tab[(i)].u.next_free & ~(1|2))) /* dead or alive */ +static void +send_ets_transfer_message(Process *c_p, Process *proc, + ErtsProcLocks *locks, + DbTable *tb, Eterm heir_data); static void schedule_free_dbtable(DbTable* tb); +static void delete_sched_table(Process *c_p, DbTable *tb); static void table_dec_refc(DbTable *tb, erts_aint_t min_val) { @@ -164,16 +169,32 @@ tid2tab(Eterm tid) return tb; } +static ERTS_INLINE int +is_table_alive(DbTable *tb) +{ + erts_smp_atomic_t *tbref; + DbTable *rtb; + + tbref = erts_smp_binary_to_magic_indirection(tb->common.btid); + rtb = (DbTable *) erts_smp_atomic_read_nob(tbref); + + ASSERT(!rtb || rtb == tb); + + return !!rtb; +} + static ERTS_INLINE void -tid_clear(DbTable *tb) +tid_clear(Process *c_p, DbTable *tb) { DbTable *rtb; Binary *btid = tb->common.btid; erts_smp_atomic_t *tbref = erts_smp_binary_to_magic_indirection(btid); rtb = (DbTable *) erts_smp_atomic_xchg_nob(tbref, (erts_aint_t) NULL); ASSERT(!rtb || tb == rtb); - if (rtb) + if (rtb) { table_dec_refc(tb, 1); + delete_sched_table(c_p, tb); + } } static ERTS_INLINE Eterm @@ -348,6 +369,99 @@ static void schedule_free_dbtable(DbTable* tb) sizeof(DbTable)); } +static ERTS_INLINE void +save_sched_table(Process *c_p, DbTable *tb) +{ + ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); + DbTable *first; + + ASSERT(esdp); + esdp->ets_tables.count++; + erts_smp_refc_inc(&tb->common.refc, 1); + + first = esdp->ets_tables.clist; + if (!first) { + tb->common.all.next = tb->common.all.prev = tb; + esdp->ets_tables.clist = tb; + } + else { + tb->common.all.prev = first->common.all.prev; + tb->common.all.next = first; + tb->common.all.prev->common.all.next = tb; + first->common.all.prev = tb; + } +} + +static ERTS_INLINE void +remove_sched_table(ErtsSchedulerData *esdp, DbTable *tb) +{ + ErtsEtsAllYieldData *eaydp; + ASSERT(esdp); + ASSERT(erts_get_ref_numbers_thr_id(ERTS_MAGIC_BIN_REFN(tb->common.btid)) + == (Uint32) esdp->no); + + ASSERT(esdp->ets_tables.count > 0); + esdp->ets_tables.count--; + + eaydp = ERTS_SCHED_AUX_YIELD_DATA(esdp, ets_all); + if (eaydp->ongoing) { + /* ets:all() op process list from last to first... */ + if (eaydp->tab == tb) { + if (eaydp->tab == esdp->ets_tables.clist) + eaydp->tab = NULL; + else + eaydp->tab = tb->common.all.prev; + } + } + + if (tb->common.all.next == tb) { + ASSERT(tb->common.all.prev == tb); + ASSERT(esdp->ets_tables.clist == tb); + esdp->ets_tables.clist = NULL; + } + else { +#ifdef DEBUG + DbTable *tmp = esdp->ets_tables.clist; + do { + if (tmp == tb) break; + tmp = tmp->common.all.next; + } while (tmp != esdp->ets_tables.clist); + ASSERT(tmp == tb); +#endif + tb->common.all.prev->common.all.next = tb->common.all.next; + tb->common.all.next->common.all.prev = tb->common.all.prev; + + if (esdp->ets_tables.clist == tb) + esdp->ets_tables.clist = tb->common.all.next; + + } + + table_dec_refc(tb, 0); +} + +static void +scheduled_remove_sched_table(void *vtb) +{ + remove_sched_table(erts_get_scheduler_data(), (DbTable *) vtb); +} + +static void +delete_sched_table(Process *c_p, DbTable *tb) +{ + ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); + Uint32 sid; + + ASSERT(esdp); + + ASSERT(tb->common.btid); + sid = erts_get_ref_numbers_thr_id(ERTS_MAGIC_BIN_REFN(tb->common.btid)); + ASSERT(1 <= sid && sid <= erts_no_schedulers); + if (sid == (Uint32) esdp->no) + remove_sched_table(esdp, tb); + else + erts_schedule_misc_aux_work((int) sid, scheduled_remove_sched_table, tb); +} + static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock, char *rwname, char* fixname) { @@ -1322,6 +1436,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) { DbTable* tb; Eterm ret; + Eterm old_name; erts_smp_rwmtx_t *lck1, *lck2; #ifdef HARDDEBUG @@ -1338,12 +1453,10 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) (void) meta_name_tab_bucket(BIF_ARG_2, &lck1); - if (is_small(BIF_ARG_1)) { - Uint slot = unsigned_val(BIF_ARG_1) & meta_main_tab_slot_mask; - lck2 = get_meta_main_tab_lock(slot); - } - else if (is_atom(BIF_ARG_1)) { - (void) meta_name_tab_bucket(BIF_ARG_1, &lck2); + if (is_atom(BIF_ARG_1)) { + old_name = BIF_ARG_1; + named_tab: + (void) meta_name_tab_bucket(old_name, &lck2); if (lck1 == lck2) lck2 = NULL; else if (lck1 > lck2) { @@ -1353,7 +1466,18 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) } } else { - BIF_ERROR(BIF_P, BADARG); + Uint slot; + tb = tid2tab(BIF_ARG_1); + if (!tb) + BIF_ERROR(BIF_P, BADARG); + else { + if (is_atom(tb->common.id)) { + old_name = tb->common.id; + goto named_tab; + } + slot = unsigned_val(tb->common.id) & meta_main_tab_slot_mask; + lck2 = get_meta_main_tab_lock(slot); + } } erts_smp_rwmtx_rwlock(lck1); @@ -1378,7 +1502,10 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) tb->common.id = tb->common.the_name = BIF_ARG_2; done: - ret = tb->common.id; + if (is_atom(tb->common.id)) + ret = tb->common.id; + else + ret = BIF_ARG_1; db_unlock(tb, LCK_WRITE); erts_smp_rwmtx_rwunlock(lck1); if (lck2) @@ -1622,6 +1749,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) if (!is_named) ret = make_tid(BIF_P, tb); + save_sched_table(BIF_P, tb); + mmtl = get_meta_main_tab_lock(slot); erts_smp_rwmtx_rwlock(mmtl); meta_main_tab[slot].u.tb = tb; @@ -1634,7 +1763,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) free_slot(slot); erts_smp_rwmtx_rwunlock(mmtl); - tid_clear(tb); + tid_clear(BIF_P, tb); db_lock(tb,LCK_WRITE); free_heir_data(tb); @@ -1825,7 +1954,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) UnUseTmpHeap(3,BIF_P); } - tid_clear(tb); + tid_clear(BIF_P, tb); mmtl = get_meta_main_tab_lock(tb->common.slot); #ifdef ERTS_SMP @@ -1913,12 +2042,8 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3) db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); db_unlock(tb,LCK_WRITE); - erts_send_message(BIF_P, to_proc, &to_locks, - TUPLE4(buf, am_ETS_TRANSFER, - tb->common.id, - from_pid, - BIF_ARG_3), - 0); + send_ets_transfer_message(BIF_P, to_proc, &to_locks, + tb, BIF_ARG_3); erts_smp_proc_unlock(to_proc, to_locks); UnUseTmpHeap(5,BIF_P); BIF_RET(am_true); @@ -2188,46 +2313,254 @@ BIF_RETTYPE ets_select_delete_2(BIF_ALIST_2) return result; } -/* -** Return a list of tables on this node -*/ -BIF_RETTYPE ets_all_0(BIF_ALIST_0) +/* + * ets:all/0 + * + * ets:all() calls ets:internal_request_all/0 which + * requests information about all tables from + * each scheduler thread. Each scheduler replies + * to the calling process with information about + * existing tables created on that specific scheduler. + */ + +struct ErtsEtsAllReq_ { + erts_smp_atomic32_t refc; + Process *proc; + ErtsOIRefStorage ref; + ErtsEtsAllReqList list[1]; /* one per scheduler */ +}; + +#define ERTS_ETS_ALL_REQ_SIZE \ + (sizeof(ErtsEtsAllReq) \ + + (sizeof(ErtsEtsAllReqList) \ + * (erts_no_schedulers - 1))) + +typedef struct { + ErtsEtsAllReq *ongoing; + ErlHeapFragment *hfrag; + DbTable *tab; + ErtsEtsAllReq *queue; +} ErtsEtsAllData; + +/* Tables handled before yielding */ +#define ERTS_ETS_ALL_TB_YCNT 200 +/* + * Min yield count required before starting + * an operation that will require yield. + */ +#define ERTS_ETS_ALL_TB_YCNT_START 10 + +#ifdef DEBUG +/* Test yielding... */ +#undef ERTS_ETS_ALL_TB_YCNT +#undef ERTS_ETS_ALL_TB_YCNT_START +#define ERTS_ETS_ALL_TB_YCNT 10 +#define ERTS_ETS_ALL_TB_YCNT_START 1 +#endif + +static int +ets_all_reply(ErtsSchedulerData *esdp, ErtsEtsAllReq **reqpp, + ErlHeapFragment **hfragpp, DbTable **tablepp, + int *yield_count_p) { - DbTable* tb; - Eterm previous; - int i; - Eterm* hp; - Eterm* hendp; - int t_tabs_cnt; - int t_top; + ErtsEtsAllReq *reqp = *reqpp; + ErlHeapFragment *hfragp = *hfragpp; + int ycount = *yield_count_p; + DbTable *tb, *first; + Uint sz; + Eterm list, msg, ref, *hp; + ErlOffHeap *ohp; + ErtsMessage *mp; - erts_smp_spin_lock(&meta_main_tab_main_lock); - t_tabs_cnt = meta_main_tab_cnt; - t_top = meta_main_tab_top; - erts_smp_spin_unlock(&meta_main_tab_main_lock); + /* + * - save_sched_table() inserts at end of circular list. + * + * - This function scans from the end so we know that + * the amount of tables to scan wont grow even if we + * yield. + * + * - remove_sched_table() updates the table we yielded + * on if it removes it. + */ + + if (hfragp) { + /* Restart of a yielded operation... */ + ASSERT(hfragp->used_size < hfragp->alloc_size); + ohp = &hfragp->off_heap; + hp = &hfragp->mem[hfragp->used_size]; + list = *hp; + hfragp->used_size = hfragp->alloc_size; + first = esdp->ets_tables.clist; + tb = *tablepp; + } + else { + /* A new operation... */ + ASSERT(!*tablepp); - hp = HAlloc(BIF_P, 2*t_tabs_cnt); - hendp = hp + 2*t_tabs_cnt; + /* Max heap size needed... */ + sz = esdp->ets_tables.count; + sz *= ERTS_MAGIC_REF_THING_SIZE + 2; + sz += 3 + ERTS_REF_THING_SIZE; + hfragp = new_message_buffer(sz); - previous = NIL; - for(i = 0; i < t_top; i++) { - erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(i); - erts_smp_rwmtx_rlock(mmtl); - if (IS_SLOT_ALIVE(i)) { - if (hp == hendp) { - /* Racing table creator, grab some more heap space */ - t_tabs_cnt = 10; - hp = HAlloc(BIF_P, 2*t_tabs_cnt); - hendp = hp + 2*t_tabs_cnt; - } - tb = meta_main_tab[i].u.tb; - previous = CONS(hp, tb->common.id, previous); - hp += 2; - } - erts_smp_rwmtx_runlock(mmtl); + hp = &hfragp->mem[0]; + ohp = &hfragp->off_heap; + list = NIL; + first = esdp->ets_tables.clist; + tb = first ? first->common.all.prev : NULL; + } + + if (tb) { + while (1) { + if (is_table_alive(tb)) { + Eterm tid; + if (is_atom(tb->common.id)) + tid = tb->common.id; + else + tid = erts_mk_magic_ref(&hp, ohp, tb->common.btid); + list = CONS(hp, tid, list); + hp += 2; + } + + if (tb == first) + break; + + tb = tb->common.all.prev; + + if (--ycount <= 0) { + sz = hp - &hfragp->mem[0]; + ASSERT(hfragp->alloc_size > sz + 1); + *hp = list; + hfragp->used_size = sz; + *hfragpp = hfragp; + *reqpp = reqp; + *tablepp = tb; + *yield_count_p = 0; + return 1; /* Yield! */ + } + } + } + + ref = erts_oiref_storage_make_ref(&reqp->ref, &hp); + msg = TUPLE2(hp, ref, list); + hp += 3; + + sz = hp - &hfragp->mem[0]; + ASSERT(sz <= hfragp->alloc_size); + + hfragp = erts_resize_message_buffer(hfragp, sz, &msg, 1); + + mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = hfragp; + + erts_queue_message(reqp->proc, 0, mp, msg, am_system); + + erts_proc_dec_refc(reqp->proc); + + if (erts_smp_atomic32_dec_read_nob(&reqp->refc) == 0) + erts_free(ERTS_ALC_T_ETS_ALL_REQ, reqp); + + *reqpp = NULL; + *hfragpp = NULL; + *tablepp = NULL; + *yield_count_p = ycount; + + return 0; +} + +int +erts_handle_yielded_ets_all_request(ErtsSchedulerData *esdp, + ErtsEtsAllYieldData *eaydp) +{ + int ix = (int) esdp->no - 1; + int yc = ERTS_ETS_ALL_TB_YCNT; + + while (1) { + if (!eaydp->ongoing) { + ErtsEtsAllReq *ongoing; + + if (!eaydp->queue) + return 0; /* All work completed! */ + + if (yc < ERTS_ETS_ALL_TB_YCNT_START && yc > esdp->ets_tables.count) + return 1; /* Yield! */ + + eaydp->ongoing = ongoing = eaydp->queue; + if (ongoing->list[ix].next == ongoing) + eaydp->queue = NULL; + else { + ongoing->list[ix].next->list[ix].prev = ongoing->list[ix].prev; + ongoing->list[ix].prev->list[ix].next = ongoing->list[ix].next; + eaydp->queue = ongoing->list[ix].next; + } + ASSERT(!eaydp->hfrag); + ASSERT(!eaydp->tab); + } + + if (ets_all_reply(esdp, &eaydp->ongoing, &eaydp->hfrag, &eaydp->tab, &yc)) + return 1; /* Yield! */ + } +} + +static void +handle_ets_all_request(void *vreq) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsEtsAllYieldData *eayp = ERTS_SCHED_AUX_YIELD_DATA(esdp, ets_all); + ErtsEtsAllReq *req = (ErtsEtsAllReq *) vreq; + + if (!eayp->ongoing && !eayp->queue) { + /* No ets:all() operations ongoing... */ + ErlHeapFragment *hf = NULL; + DbTable *tb = NULL; + int yc = ERTS_ETS_ALL_TB_YCNT; + if (ets_all_reply(esdp, &req, &hf, &tb, &yc)) { + /* Yielded... */ + ASSERT(hf); + eayp->ongoing = req; + eayp->hfrag = hf; + eayp->tab = tb; + erts_notify_new_aux_yield_work(esdp); + } + } + else { + /* Ongoing ets:all() operations; queue up this request... */ + int ix = (int) esdp->no - 1; + if (!eayp->queue) { + req->list[ix].next = req; + req->list[ix].prev = req; + eayp->queue = req; + } + else { + req->list[ix].next = eayp->queue; + req->list[ix].prev = eayp->queue->list[ix].prev; + eayp->queue->list[ix].prev = req; + req->list[ix].prev->list[ix].next = req; + } } - HRelease(BIF_P, hendp, hp); - BIF_RET(previous); +} + +BIF_RETTYPE ets_internal_request_all_0(BIF_ALIST_0) +{ + Eterm ref = erts_make_ref(BIF_P); + ErtsEtsAllReq *req = erts_alloc(ERTS_ALC_T_ETS_ALL_REQ, + ERTS_ETS_ALL_REQ_SIZE); + erts_smp_atomic32_init_nob(&req->refc, + (erts_aint32_t) erts_no_schedulers); + erts_oiref_storage_save(&req->ref, ref); + req->proc = BIF_P; + erts_proc_add_refc(BIF_P, (Sint) erts_no_schedulers); + +#ifdef ERTS_SMP + if (erts_no_schedulers > 1) + erts_schedule_multi_misc_aux_work(1, + erts_no_schedulers, + handle_ets_all_request, + (void *) req); +#endif + + handle_ets_all_request((void *) req); + BIF_RET(ref); } @@ -2788,7 +3121,7 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1) */ if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ)) == NULL) { - if (is_atom(BIF_ARG_1) || is_small(BIF_ARG_1)) { + if (is_atom(BIF_ARG_1) || is_ref(BIF_ARG_1)) { BIF_RET(am_undefined); } BIF_ERROR(BIF_P, BADARG); @@ -2850,7 +3183,7 @@ BIF_RETTYPE ets_info_2(BIF_ALIST_2) Eterm ret = THE_NON_VALUE; if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ)) == NULL) { - if (is_atom(BIF_ARG_1) || is_small(BIF_ARG_1)) { + if (is_atom(BIF_ARG_1) || is_ref(BIF_ARG_1)) { BIF_RET(am_undefined); } BIF_ERROR(BIF_P, BADARG); @@ -3140,6 +3473,19 @@ void init_db(ErtsDbSpinCount db_spin_count) ms_delete_all = CONS(hp, ms_delete_all,NIL); } +void +erts_ets_sched_spec_data_init(ErtsSchedulerData *esdp) +{ + ErtsEtsAllYieldData *eaydp = ERTS_SCHED_AUX_YIELD_DATA(esdp, ets_all); + eaydp->ongoing = NULL; + eaydp->hfrag = NULL; + eaydp->tab = NULL; + eaydp->queue = NULL; + esdp->ets_tables.clist = NULL; + esdp->ets_tables.count = 0; +} + + #define ARRAY_CHUNK 100 typedef enum { @@ -3288,17 +3634,46 @@ retry: ASSERT(arityval(*tpv) == 1); heir_data = tpv[1]; } - erts_send_message(p, to_proc, &to_locks, - TUPLE4(buf, - am_ETS_TRANSFER, - tb->common.id, - p->common.id, - heir_data), - 0); + send_ets_transfer_message(p, to_proc, &to_locks, tb, heir_data); erts_smp_proc_unlock(to_proc, to_locks); return !0; } +static void +send_ets_transfer_message(Process *c_p, Process *proc, + ErtsProcLocks *locks, + DbTable *tb, Eterm heir_data) +{ + Uint hsz, hd_sz; + ErtsMessage *mp; + Eterm *hp; + ErlOffHeap *ohp; + Eterm tid, hd_copy, msg, sender; + + hsz = 5; + if (is_not_atom(tb->common.id)) + hsz += ERTS_MAGIC_REF_THING_SIZE; + if (is_immed(heir_data)) + hd_sz = 0; + else { + hd_sz = size_object(heir_data); + hsz += hd_sz; + } + + mp = erts_alloc_message_heap(proc, locks, hsz, &hp, &ohp); + if (is_atom(tb->common.id)) + tid = tb->common.id; + else + tid = erts_mk_magic_ref(&hp, ohp, tb->common.btid); + if (!hd_sz) + hd_copy = heir_data; + else + hd_copy = copy_struct(heir_data, hd_sz, &hp, ohp); + sender = c_p->common.id; + msg = TUPLE4(hp, am_ETS_TRANSFER, tid, sender, hd_copy); + erts_queue_message(proc, *locks, mp, msg, sender); +} + /* * erts_db_process_exiting() is called when a process terminates. * It returns 0 when completely done, and !0 when it wants to @@ -3384,7 +3759,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) #endif first_call = (tb->common.status & DB_DELETE) == 0; if (first_call) { - tid_clear(tb); + tid_clear(c_p, tb); /* Clear all access bits. */ tb->common.status &= ~(DB_PROTECTED | DB_PUBLIC @@ -3975,21 +4350,30 @@ static void print_table(fmtfn_t to, void *to_arg, int show, DbTable* tb) erts_print(to, to_arg, "Read Concurrency: %T\n", table_info(NULL, tb, am_read_concurrency)); } +typedef struct { + fmtfn_t to; + void *to_arg; + int show; +} ErtsPrintDbInfo; + +static void +db_info_print(DbTable *tb, void *vpdbip) +{ + ErtsPrintDbInfo *pdbip = (ErtsPrintDbInfo *) vpdbip; + erts_print(pdbip->to, pdbip->to_arg, "=ets:%T\n", tb->common.owner); + erts_print(pdbip->to, pdbip->to_arg, "Slot: %bpu\n", (Uint) tb); + print_table(pdbip->to, pdbip->to_arg, pdbip->show, tb); +} + void db_info(fmtfn_t to, void *to_arg, int show) /* Called by break handler */ { - int i; - for (i=0; i < db_max_tabs; i++) - if (IS_SLOT_ALIVE(i)) { - erts_print(to, to_arg, "=ets:%T\n", meta_main_tab[i].u.tb->common.owner); - erts_print(to, to_arg, "Slot: %d\n", i); - print_table(to, to_arg, show, meta_main_tab[i].u.tb); - } -#ifdef DEBUG - erts_print(to, to_arg, "=internal_ets: Process to table index\n"); - print_table(to, to_arg, show, meta_pid_to_tab); - erts_print(to, to_arg, "=internal_ets: Process to fixation index\n"); - print_table(to, to_arg, show, meta_pid_to_fixed_tab); -#endif + ErtsPrintDbInfo pdbi; + + pdbi.to = to; + pdbi.to_arg = to_arg; + pdbi.show = show; + + erts_db_foreach_table(db_info_print, &pdbi); } Uint @@ -4004,15 +4388,22 @@ erts_get_ets_misc_mem_size(void) void erts_db_foreach_table(void (*func)(DbTable *, void *), void *arg) { - int i, j; - j = 0; - for(i = 0; (i < db_max_tabs && j < meta_main_tab_cnt); i++) { - if (IS_SLOT_ALIVE(i)) { - j++; - (*func)(meta_main_tab[i].u.tb, arg); - } + int ix; + + ASSERT(erts_smp_thr_progress_is_blocking()); + + for (ix = 0; ix < erts_no_schedulers; ix++) { + ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix); + DbTable *first = esdp->ets_tables.clist; + if (first) { + DbTable *tb = first; + do { + if (is_table_alive(tb)) + (*func)(tb, arg); + tb = tb->common.all.next; + } while (tb != first); + } } - ASSERT(j == meta_main_tab_cnt); } /* SMP Note: May only be used when system is locked */ diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index 852440ff31..1c53401590 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -24,8 +24,37 @@ * */ -#ifndef __DB_H__ -#define __DB_H__ +#ifndef ERTS_DB_SCHED_SPEC_TYPES__ +#define ERTS_DB_SCHED_SPEC_TYPES__ + +union db_table; +typedef union db_table DbTable; + +typedef struct ErtsEtsAllReq_ ErtsEtsAllReq; + +typedef struct { + ErtsEtsAllReq *next; + ErtsEtsAllReq *prev; +} ErtsEtsAllReqList; + +typedef struct { + ErtsEtsAllReq *ongoing; + ErlHeapFragment *hfrag; + DbTable *tab; + ErtsEtsAllReq *queue; +} ErtsEtsAllYieldData; + +typedef struct { + Uint count; + DbTable *clist; +} ErtsEtsTables; + +#endif /* ERTS_DB_SCHED_SPEC_TYPES__ */ + +#ifndef ERTS_ONLY_SCHED_SPEC_ETS_DATA + +#ifndef ERL_DB_H__ +#define ERL_DB_H__ #include "sys.h" #undef ERL_THR_PROGRESS_TSD_TYPE_ONLY @@ -46,6 +75,12 @@ typedef struct { ErtsThrPrgrLaterOp data; } DbTableRelease; +struct ErtsSchedulerData_; +int erts_handle_yielded_ets_all_request(struct ErtsSchedulerData_ *esdp, + ErtsEtsAllYieldData *eadp); + +void erts_ets_sched_spec_data_init(struct ErtsSchedulerData_ *esdp); + /* * So, the structure for a database table, NB this is only * interesting in db.c. @@ -95,7 +130,7 @@ Eterm erts_ets_restore_meta_state(Process* p, Eterm target_state); Uint erts_db_get_max_tabs(void); -#endif +#endif /* ERL_DB_H__ */ #if defined(ERTS_WANT_DB_INTERNAL__) && !defined(ERTS_HAVE_DB_INTERNAL__) #define ERTS_HAVE_DB_INTERNAL__ @@ -271,3 +306,4 @@ erts_db_free_nt(ErtsAlcType_t type, void *ptr, Uint size) #endif /* #if defined(ERTS_WANT_DB_INTERNAL__) && !defined(ERTS_HAVE_DB_INTERNAL__) */ +#endif /* !ERTS_ONLY_SCHED_SPEC_ETS_DATA */ diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 60dedd5713..3c13035ec5 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -75,9 +75,6 @@ typedef struct db_term { */ } DbTerm; -union db_table; -typedef union db_table DbTable; - #define DB_MUST_RESIZE 1 #define DB_NEW_OBJECT 2 #define DB_INC_TRY_GROW 4 @@ -203,6 +200,11 @@ typedef struct db_fixation { struct db_fixation *next; } DbFixation; +typedef struct { + DbTable *next; + DbTable *prev; +} DbTableList; + /* * This structure contains data for all different types of database * tables. Note that these fields must match the same fields @@ -214,6 +216,7 @@ typedef struct db_fixation { typedef struct db_table_common { erts_smp_refc_t refc; /* reference count of table struct */ erts_smp_refc_t fix_count;/* fixation counter */ + DbTableList all; #ifdef ERTS_SMP erts_smp_rwmtx_t rwlock; /* rw lock on table */ erts_smp_mtx_t fixlock; /* Protects fixations,megasec,sec,microsec */ diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index b2c307e826..61477af316 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -2307,6 +2307,7 @@ erl_start(int argc, char **argv) #endif set_main_stack_size(); erts_sched_init_time_sup(esdp); + erts_ets_sched_spec_data_init(esdp); process_main(esdp->x_reg_array, esdp->f_reg_array); } #endif diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 52d9f9ddf7..b816ba59fb 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -602,6 +602,7 @@ dbg_chk_aux_work_val(erts_aint32_t value) valid |= ERTS_SSI_AUX_WORK_REAP_PORTS; #endif valid |= ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED; + valid |= ERTS_SSI_AUX_WORK_YIELD; if (~valid & value) erts_exit(ERTS_ABORT_EXIT, @@ -690,6 +691,8 @@ erts_pre_init_process(void) = "SET_TMO"; erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX] = "MSEG_CACHE_CHECK"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_YIELD_IX] + = "YIELD"; erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_REAP_PORTS_IX] = "REAP_PORTS"; erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX] @@ -2557,6 +2560,48 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) return aux_work & ~ERTS_SSI_AUX_WORK_REAP_PORTS; } +void +erts_notify_new_aux_yield_work(ErtsSchedulerData *esdp) +{ + ASSERT(esdp == erts_get_scheduler_data()); + /* Always called by the scheduler itself... */ + set_aux_work_flags_wakeup_nob(esdp->ssi, ERTS_SSI_AUX_WORK_YIELD); +} + +static ERTS_INLINE erts_aint32_t +handle_yield(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) +{ + int yield = 0; + /* + * Yield operations are always requested by the scheduler itself. + * + * The following handlers should *not* set the ERTS_SSI_AUX_WORK_YIELD + * flag in order to indicate more work. They should instead return + * information so this "main handler" can manipulate the flag... + * + * The following handlers should be able to handle being called + * even though no work is to be done... + */ + + /* Various yielding operations... */ + + yield |= erts_handle_yielded_ets_all_request(awdp->esdp, + &awdp->yield.ets_all); + + /* + * Other yielding operations... + * + */ + + if (!yield) { + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_YIELD); + return aux_work & ~ERTS_SSI_AUX_WORK_YIELD; + } + + return aux_work; +} + + #if HAVE_ERTS_MSEG static ERTS_INLINE erts_aint32_t @@ -2697,6 +2742,9 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) handle_mseg_cache_check); #endif + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_YIELD, + handle_yield); + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_REAP_PORTS, handle_reap_ports); @@ -8727,6 +8775,8 @@ sched_thread_func(void *vesdp) ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL); #endif + erts_ets_sched_spec_data_init(esdp); + process_main(esdp->x_reg_array, esdp->f_reg_array); /* No schedulers should *ever* terminate */ diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index ce0989883c..58f1e15c23 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -63,6 +63,9 @@ typedef struct process Process; #define ERTS_ONLY_INCLUDE_TRACE_FLAGS #include "erl_trace.h" #undef ERTS_ONLY_INCLUDE_TRACE_FLAGS +#define ERTS_ONLY_SCHED_SPEC_ETS_DATA +#include "erl_db.h" +#undef ERTS_ONLY_SCHED_SPEC_ETS_DATA #ifdef HIPE #include "hipe_process.h" @@ -312,6 +315,7 @@ typedef enum { ERTS_SSI_AUX_WORK_PENDING_EXITERS_IX, ERTS_SSI_AUX_WORK_SET_TMO_IX, ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX, + ERTS_SSI_AUX_WORK_YIELD_IX, ERTS_SSI_AUX_WORK_REAP_PORTS_IX, ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX, /* SHOULD be last flag index */ @@ -348,6 +352,8 @@ typedef enum { (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX) #define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \ (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX) +#define ERTS_SSI_AUX_WORK_YIELD \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_YIELD_IX) #define ERTS_SSI_AUX_WORK_REAP_PORTS \ (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_REAP_PORTS_IX) #define ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED \ @@ -612,6 +618,10 @@ typedef struct { ErtsDelayedAuxWorkWakeupJob *job; } delayed_wakeup; #endif + struct { + ErtsEtsAllYieldData ets_all; + /* Other yielding operations... */ + } yield; struct { struct { erts_aint32_t flags; @@ -621,6 +631,10 @@ typedef struct { } debug; } ErtsAuxWorkData; +#define ERTS_SCHED_AUX_YIELD_DATA(ESDP, NAME) \ + (&(ESDP)->aux_work_data.yield.NAME) +void erts_notify_new_aux_yield_work(ErtsSchedulerData *esdp); + #ifdef ERTS_DIRTY_SCHEDULERS typedef enum { ERTS_DIRTY_CPU_SCHEDULER, @@ -688,7 +702,7 @@ struct ErtsSchedulerData_ { ErtsSchedWallTime sched_wall_time; ErtsGCInfo gc_info; ErtsPortTaskHandle nosuspend_port_task_handle; - + ErtsEtsTables ets_tables; #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC erts_alloc_verify_func_t verify_unused_temp_alloc; Allctr_t *verify_unused_temp_alloc_data; -- cgit v1.2.3 From 7488cfc986e1608e73f904071556eb0543d6a346 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Thu, 2 Mar 2017 16:59:38 +0100 Subject: erts: Add enif_inspect_binary emasculation of refc bins --- erts/emulator/beam/erl_nif.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index e6da4c1a76..63a4a997da 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -998,16 +998,14 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) byte* raw_ptr; }u; - if (is_boxed(bin_term) && *binary_val(bin_term) == HEADER_SUB_BIN) { - ErlSubBin* sb = (ErlSubBin*) binary_val(bin_term); - if (sb->is_writable) { - ProcBin* pb = (ProcBin*) binary_val(sb->orig); - ASSERT(pb->thing_word == HEADER_PROC_BIN); - if (pb->flags) { - erts_emasculate_writable_binary(pb); - sb->is_writable = 0; - } - } + if (is_binary(bin_term)) { + ProcBin *pb = (ProcBin*) binary_val(bin_term); + if (pb->thing_word == HEADER_SUB_BIN) { + ErlSubBin* sb = (ErlSubBin*) pb; + pb = (ProcBin*) binary_val(sb->orig); + } + if (pb->thing_word == HEADER_PROC_BIN && pb->flags) + erts_emasculate_writable_binary(pb); } u.tmp = NULL; bin->data = erts_get_aligned_binary_bytes_extra(bin_term, &u.raw_ptr, allocator, @@ -1024,7 +1022,7 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) bin->bin_term = bin_term; bin->size = binary_size(bin_term); bin->ref_bin = NULL; - ADD_READONLY_CHECK(env, bin->data, bin->size); + ADD_READONLY_CHECK(env, bin->data, bin->size); return 1; } -- cgit v1.2.3 From 5cce8836ccffbaf71879b2efbb5788278e7f7e90 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 8 Mar 2017 14:41:54 +0100 Subject: erts: Fix faulty ASSERT for failed dec_term when a recursive call to dec_term() has already done erts_factory_undo and set factory->hp to NULL. --- erts/emulator/beam/external.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 205a7711ec..186b250c7e 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -4010,7 +4010,7 @@ error: factory->hp = hp; /* the largest must be the freshest */ } } - else ASSERT(factory->hp == hp); + else ASSERT(!factory->hp || factory->hp == hp); error_hamt: erts_factory_undo(factory); -- cgit v1.2.3 From 65b04e233e09e3cc2e0fda3c28e155b95c5a4baf Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 7 Mar 2017 17:04:40 +0100 Subject: erts: Do not generate atoms on old latin1 external format term_to_binary will always generate utf8 atoms ATOM_UTF8_EXT and SMALL_ATOM_UTF8_EXT. Old latin1 atoms, ATOM_EXT and SMALL_ATOM_EXT, are still decoded. --- erts/emulator/beam/dist.h | 1 + erts/emulator/beam/external.c | 39 +++------------------------------------ 2 files changed, 4 insertions(+), 36 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index 8f6be1061a..6ed36a478e 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -53,6 +53,7 @@ | DFLAG_EXPORT_PTR_TAG \ | DFLAG_BIT_BINARIES \ | DFLAG_MAP_TAG \ + | DFLAG_UTF8_ATOMS \ | DFLAG_BIG_CREATION) /* opcodes used in distribution messages */ diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 186b250c7e..285ae4ac78 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2093,7 +2093,6 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) { int iix; int len; - int utf8_atoms = (int) (dflags & DFLAG_UTF8_ATOMS); ASSERT(is_atom(atom)); @@ -2122,8 +2121,8 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) if (iix < 0) { Atom *a = atom_tab(atom_val(atom)); len = a->len; - if (utf8_atoms || a->latin1_chars < 0) { - if (len > 255) { + { + if (len > 255) { *ep++ = ATOM_UTF8_EXT; put_int16(len, ep); ep += 2; @@ -2135,32 +2134,6 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) } sys_memcpy((char *) ep, (char *) a->name, len); } - else { - if (a->latin1_chars <= 255 && (dflags & DFLAG_SMALL_ATOM_TAGS)) { - *ep++ = SMALL_ATOM_EXT; - if (len == a->latin1_chars) { - sys_memcpy(ep+1, a->name, len); - } - else { - len = erts_utf8_to_latin1(ep+1, a->name, len); - ASSERT(len == a->latin1_chars); - } - put_int8(len, ep); - ep++; - } - else { - *ep++ = ATOM_EXT; - if (len == a->latin1_chars) { - sys_memcpy(ep+2, a->name, len); - } - else { - len = erts_utf8_to_latin1(ep+2, a->name, len); - ASSERT(len == a->latin1_chars); - } - put_int16(len, ep); - ep += 2; - } - } ep += len; return ep; } @@ -4085,19 +4058,13 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, else { Atom *a = atom_tab(atom_val(obj)); int alen; - if ((dflags & DFLAG_UTF8_ATOMS) || a->latin1_chars < 0) { + { alen = a->len; result += 1 + 1 + alen; if (alen > 255) { result++; /* ATOM_UTF8_EXT (not small) */ } } - else { - alen = a->latin1_chars; - result += 1 + 1 + alen; - if (alen > 255 || !(dflags & DFLAG_SMALL_ATOM_TAGS)) - result++; /* ATOM_EXT (not small) */ - } insert_acache_map(acmp, obj, dflags); } break; -- cgit v1.2.3 From 26c3cd82529836cb5b6eefbf7f92f318fd91f847 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 10 Mar 2017 15:00:46 +0100 Subject: Update copyright year --- erts/emulator/beam/beam_bif_load.c | 2 +- erts/emulator/beam/beam_bp.c | 2 +- erts/emulator/beam/beam_bp.h | 2 +- erts/emulator/beam/beam_emu.c | 2 +- erts/emulator/beam/beam_load.c | 2 +- erts/emulator/beam/break.c | 2 +- erts/emulator/beam/copy.c | 2 +- erts/emulator/beam/dist.c | 2 +- erts/emulator/beam/erl_bif_ddll.c | 2 +- erts/emulator/beam/erl_bif_guard.c | 2 +- erts/emulator/beam/erl_bif_info.c | 2 +- erts/emulator/beam/erl_bif_trace.c | 2 +- erts/emulator/beam/erl_db.c | 2 +- erts/emulator/beam/erl_db_util.c | 2 +- erts/emulator/beam/erl_db_util.h | 2 +- erts/emulator/beam/erl_fun.c | 2 +- erts/emulator/beam/erl_fun.h | 2 +- erts/emulator/beam/erl_gc.c | 2 +- erts/emulator/beam/erl_gc.h | 2 +- erts/emulator/beam/erl_hl_timer.c | 2 +- erts/emulator/beam/erl_init.c | 2 +- erts/emulator/beam/erl_lock_check.c | 2 +- erts/emulator/beam/erl_message.c | 2 +- erts/emulator/beam/erl_monitors.c | 2 +- erts/emulator/beam/erl_nif.c | 2 +- erts/emulator/beam/erl_node_tables.c | 2 +- erts/emulator/beam/erl_node_tables.h | 2 +- erts/emulator/beam/erl_process.c | 2 +- erts/emulator/beam/erl_process_dump.c | 2 +- erts/emulator/beam/erl_trace.c | 2 +- erts/emulator/beam/erl_vm.h | 2 +- erts/emulator/beam/export.c | 2 +- erts/emulator/beam/external.c | 2 +- erts/emulator/beam/global.h | 2 +- erts/emulator/beam/index.c | 2 +- erts/emulator/beam/index.h | 2 +- erts/emulator/beam/io.c | 2 +- erts/emulator/beam/sys.h | 2 +- erts/emulator/beam/utils.c | 2 +- 39 files changed, 39 insertions(+), 39 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 7f5e9ff691..27d6823d3c 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2016. All Rights Reserved. + * Copyright Ericsson AB 1999-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index a66d9f68d9..6d723a4b92 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2016. All Rights Reserved. + * Copyright Ericsson AB 2000-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 9e4ab3cd20..7372b9b258 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2016. All Rights Reserved. + * Copyright Ericsson AB 2000-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 1c83d8f02e..1ad13c32e3 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 88b6e89a02..2bad3ab4c5 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 4c7c910419..74a8a3b852 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 78db141cfc..de68adbeb6 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 52b2174609..3611eeee02 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index bace51bbbb..4948975851 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2016. All Rights Reserved. + * Copyright Ericsson AB 2006-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c index 74cf7cf11a..ea508bd1c4 100644 --- a/erts/emulator/beam/erl_bif_guard.c +++ b/erts/emulator/beam/erl_bif_guard.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2016. All Rights Reserved. + * Copyright Ericsson AB 2006-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 95a56a3de9..7bd45916f5 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2016. All Rights Reserved. + * Copyright Ericsson AB 1999-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 0627526d7e..4fed09f085 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2016. All Rights Reserved. + * Copyright Ericsson AB 1999-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 3cc2b21a20..23c8a5bae9 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index f4db7ca3dd..3c5bfb2552 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2016. All Rights Reserved. + * Copyright Ericsson AB 1998-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 3fb063797e..b30f0580dd 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2016. All Rights Reserved. + * Copyright Ericsson AB 1998-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index fd4d5b1c5c..53a1801b72 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2016. All Rights Reserved. + * Copyright Ericsson AB 2000-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index 0fd0d62343..b1d5cddb34 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2016. All Rights Reserved. + * Copyright Ericsson AB 2000-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 9b7744068e..bb36be0848 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2016. All Rights Reserved. + * Copyright Ericsson AB 2002-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 2521379664..76dd74c866 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2007-2016. All Rights Reserved. + * Copyright Ericsson AB 2007-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index 647fa26811..a73f4ecfc7 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2015-2016. All Rights Reserved. + * Copyright Ericsson AB 2015-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index f65a06c85f..e8b8739852 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2016. All Rights Reserved. + * Copyright Ericsson AB 1997-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 5d4823c077..c53bdc685b 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2016. All Rights Reserved. + * Copyright Ericsson AB 2005-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 019e079e67..cdd771ef7d 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2016. All Rights Reserved. + * Copyright Ericsson AB 1997-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c index c207dea10f..7244ae144f 100644 --- a/erts/emulator/beam/erl_monitors.c +++ b/erts/emulator/beam/erl_monitors.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2016. All Rights Reserved. + * Copyright Ericsson AB 2004-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 4fa15a8dd8..479a2f4809 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2016. All Rights Reserved. + * Copyright Ericsson AB 2009-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 467a05f950..f37393bce4 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2016. All Rights Reserved. + * Copyright Ericsson AB 2001-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index 35051173d0..489da1ba17 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2016. All Rights Reserved. + * Copyright Ericsson AB 2001-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 485949c84a..5c0322a7f6 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 4a74cc2b82..8d66391d55 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2016. All Rights Reserved. + * Copyright Ericsson AB 2003-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index ac9e91e31f..00bb114c26 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2016. All Rights Reserved. + * Copyright Ericsson AB 1999-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index 899e2a275a..3d8c647386 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 00fa452f79..6cf6fed31a 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 0ef8114bac..9436259011 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 26aa39b65a..19286e1310 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c index cd834e2c12..a1f6f54543 100644 --- a/erts/emulator/beam/index.c +++ b/erts/emulator/beam/index.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h index 10f5d1eb39..6c07571df6 100644 --- a/erts/emulator/beam/index.h +++ b/erts/emulator/beam/index.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 38e403d91a..c3eb610fdc 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 79ec34e717..d49edad6dc 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 7f7e38c22a..d90c282c7e 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. -- cgit v1.2.3 From a603a15503c6e11290c354c8c5c18578ccd8acff Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 16 Mar 2017 18:24:43 +0100 Subject: erts: Change HIPE allocations from sys_alloc to long lived, short lived and native stack. --- erts/emulator/beam/beam_load.c | 6 +++--- erts/emulator/beam/erl_alloc.types | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 48206a75a8..6eea963016 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -6321,7 +6321,7 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) */ magic = erts_alloc_loader_state(); stp = ERTS_MAGIC_BIN_DATA(magic); - hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code)); + hipe_code = erts_alloc(ERTS_ALC_T_HIPE_LL, sizeof(*hipe_code)); if (!is_internal_magic_ref(hipe_magic_bin) || !(hipe_magic = erts_magic_ref2bin(hipe_magic_bin), @@ -6556,7 +6556,7 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) } error: - erts_free(ERTS_ALC_T_HIPE, hipe_code); + erts_free(ERTS_ALC_T_HIPE_LL, hipe_code); erts_free_aligned_binary_bytes(temp_alloc); free_loader_state(magic); BIF_ERROR(p, BADARG); @@ -6583,7 +6583,7 @@ int erts_commit_hipe_patch_load(Eterm hipe_magic_bin) /* * Initialise HiPE module */ - hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code)); + hipe_code = erts_alloc(ERTS_ALC_T_HIPE_LL, sizeof(*hipe_code)); hipe_code->text_segment = hipe_stp->text_segment; hipe_code->text_segment_size = hipe_stp->text_segment_size; hipe_code->data_segment = hipe_stp->data_segment; diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 3d5de72ee7..46ab941041 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -350,8 +350,9 @@ type SL_MPATHS SHORT_LIVED SYSTEM sl_migration_paths +if hipe -# Currently most hipe code use this type. -type HIPE SYSTEM SYSTEM hipe_data +type HIPE_LL LONG_LIVED SYSTEM hipe_long_lived +type HIPE_SL SHORT_LIVED SYSTEM hipe_short_lived +type HIPE_STK STANDARD SYSTEM hipe_nstack +if exec_alloc type HIPE_EXEC EXEC CODE hipe_code -- cgit v1.2.3 From 303f4781e485f66e1eb6c5db77ff9523e5f69b7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Mon, 20 Mar 2017 11:49:15 +0100 Subject: erts: Fix erl_async include files for no-threads --- erts/emulator/beam/erl_async.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_async.h b/erts/emulator/beam/erl_async.h index 473c7686e5..c884a5040d 100644 --- a/erts/emulator/beam/erl_async.h +++ b/erts/emulator/beam/erl_async.h @@ -27,7 +27,6 @@ extern int erts_async_max_threads; #define ERTS_ASYNC_THREAD_MAX_STACK_SIZE 8192 /* Kilo words */ extern int erts_async_thread_suggested_stack_size; -#ifdef USE_THREADS #ifdef ERTS_SMP /* @@ -47,6 +46,10 @@ extern int erts_async_thread_suggested_stack_size; # define ERTS_USE_ASYNC_READY_Q 0 #endif +#ifndef USE_THREADS +# undef ERTS_USE_ASYNC_READY_Q +# define ERTS_USE_ASYNC_READY_Q 0 +#endif /* !USE_THREADS */ #if ERTS_USE_ASYNC_READY_Q int erts_check_async_ready(void *); int erts_async_ready_clean(void *, void *); @@ -58,10 +61,7 @@ void *erts_get_async_ready_queue(Uint sched_id); #endif #endif /* ERTS_USE_ASYNC_READY_Q */ -#endif /* USE_THREADS */ - void erts_init_async(void); void erts_exit_flush_async(void); - #endif /* ERL_ASYNC_H__ */ -- cgit v1.2.3 From 92616102d59ae7c51c39176b264b84b32cc21f14 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 20 Mar 2017 17:31:45 +0100 Subject: Fix type of MAKE_HASH --- erts/emulator/beam/erl_process_dict.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c index 8311fde025..7cfdf20341 100644 --- a/erts/emulator/beam/erl_process_dict.c +++ b/erts/emulator/beam/erl_process_dict.c @@ -54,9 +54,9 @@ #define HASH_RANGE(PDict) ((PDict)->usedSlots) #define MAKE_HASH(Term) \ - ((is_small(Term)) ? unsigned_val(Term) : \ + ((is_small(Term)) ? (Uint32) unsigned_val(Term) : \ ((is_atom(Term)) ? \ - atom_val(Term) : \ + (Uint32) atom_val(Term) : \ make_internal_hash(Term))) #define PD_SZ2BYTES(Sz) (sizeof(ProcDict) + ((Sz) - 1)*sizeof(Eterm)) -- cgit v1.2.3 From f92d862c1af230a873c2b2d30fea575ff3dafd56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Tue, 21 Mar 2017 14:37:03 +0100 Subject: erts: Make sigterm signal safe for non-smp beam The signal handler will now schedule a sigterm message instead of sending the message in the signal handler. The signal handler must refrain from memory allocations and thus the event is scheduled. --- erts/emulator/beam/sys.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index d49edad6dc..dd4f05686b 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -481,6 +481,12 @@ extern volatile int erts_break_requested; void erts_do_break_handling(void); #endif +#if !defined(ERTS_SMP) && !defined(__WIN32__) +extern volatile Uint erts_signal_sigterm; +#define ERTS_SIGNAL_SIGTERM erts_signal_sigterm +void erts_handle_signal_sigterm(void); +#endif + #ifdef ERTS_WANT_GOT_SIGUSR1 # ifndef UNIX # define ERTS_GOT_SIGUSR1 0 -- cgit v1.2.3 From 0dec7ddb341b2b7d2643f86dd9e9497b3f5268a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Tue, 21 Mar 2017 10:48:15 +0100 Subject: erts: Don't allocate memory during signal handling Allocations and message sending are now scheduled by a signal, via a signal state bitmap, instead of doing it directly in the signal handler. --- erts/emulator/beam/sys.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index c6ea8049c3..144dd60d21 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -487,6 +487,12 @@ extern volatile int erts_break_requested; void erts_do_break_handling(void); #endif +#if !defined(ERTS_SMP) && !defined(__WIN32__) +extern volatile Uint erts_signal_state; +#define ERTS_SIGNAL_STATE erts_signal_state +void erts_handle_signal_state(void); +#endif + #ifdef ERTS_SMP extern erts_smp_atomic32_t erts_writing_erl_crash_dump; extern erts_tsd_key_t erts_is_crash_dumping_key; -- cgit v1.2.3 From 25ee16c99376650e61f3af2fd6fdc1fc3e36a6de Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 23 Feb 2017 12:08:12 +0100 Subject: list_to_ref/1 --- erts/emulator/beam/bif.c | 126 +++++++++++++++++++++++++++++++++++++++++++++ erts/emulator/beam/bif.tab | 1 + 2 files changed, 127 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 614bedab12..2d37f977c0 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -4254,6 +4254,132 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } +BIF_RETTYPE list_to_ref_1(BIF_ALIST_1) +{ + /* + * A valid reference is on the format + * "#Ref" where N, X, Y, and Z are + * 32-bit integers (i.e., max 10 characters). + */ + Eterm *hp; + Eterm res; + Uint32 refn[ERTS_MAX_REF_NUMBERS]; + int n = 0; + Uint ints[1 + ERTS_MAX_REF_NUMBERS] = {0}; + char* cp; + Sint i; + DistEntry *dep = NULL; + char buf[5 /* #Ref< */ + + (1 + ERTS_MAX_REF_NUMBERS)*(10 + 1) /* N.X.Y.Z> */ + + 1 /* \0 */]; + + /* walk down the list and create a C string */ + if ((i = intlist_to_buf(BIF_ARG_1, buf, sizeof(buf)-1)) < 0) + goto bad; + + buf[i] = '\0'; /* null terminal */ + + cp = &buf[0]; + if (*cp++ != '#') goto bad; + if (*cp++ != 'R') goto bad; + if (*cp++ != 'e') goto bad; + if (*cp++ != 'f') goto bad; + if (*cp++ != '<') goto bad; + + for (i = 0; i < sizeof(ints)/sizeof(Uint); i++) { + if (*cp < '0' || *cp > '9') goto bad; + + while (*cp >= '0' && *cp <= '9') { + ints[i] = 10*ints[i] + (*cp - '0'); + cp++; + } + + n++; + if (ints[i] > ~((Uint32) 0)) goto bad; + if (*cp == '>') break; + if (*cp++ != '.') goto bad; + } + + if (*cp++ != '>') goto bad; + if (*cp != '\0') goto bad; + + if (n < 2) goto bad; + + for (n = 0; i > 0; i--) + refn[n++] = (Uint32) ints[i]; + + ASSERT(n <= ERTS_MAX_REF_NUMBERS); + + dep = erts_channel_no_to_dist_entry(ints[0]); + + if (!dep) + goto bad; + + if(dep == erts_this_dist_entry) { + ErtsMagicBinary *mb; + Uint32 sid; + if (refn[0] > MAX_REFERENCE) goto bad; + if (n != ERTS_REF_NUMBERS) goto bad; + sid = erts_get_ref_numbers_thr_id(refn); + if (sid > erts_no_schedulers) goto bad; + mb = erts_magic_ref_lookup_bin(refn); + if (mb) { + hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE); + res = erts_mk_magic_ref(&hp, &BIF_P->off_heap, + (Binary *) mb); + } + else { + hp = HAlloc(BIF_P, ERTS_REF_THING_SIZE); + write_ref_thing(hp, refn[0], refn[1], refn[2]); + res = make_internal_ref(hp); + } + } + else { + ExternalThing *etp; + ErlNode *enp; + Uint hsz; + int j; + + if (is_nil(dep->cid)) + goto bad; + + enp = erts_find_or_insert_node(dep->sysname, dep->creation); + ASSERT(enp != erts_this_node); + + hsz = EXTERNAL_THING_HEAD_SIZE; +#if defined(ARCH_64) + hsz += n/2 + 1; +#else + hsz += n; +#endif + + etp = (ExternalThing *) HAlloc(BIF_P, hsz); + etp->header = make_external_ref_header(n/2); + etp->next = BIF_P->off_heap.first; + etp->node = enp; + i = 0; +#if defined(ARCH_64) + etp->data.ui32[i] = n; +#endif + for (j = 0; j < n; j++) { + etp->data.ui32[i] = refn[j]; + i++; + } + + BIF_P->off_heap.first = (struct erl_off_heap_header*) etp; + res = make_external_ref(etp); + } + + erts_deref_dist_entry(dep); + BIF_RET(res); + + bad: + if (dep) + erts_deref_dist_entry(dep); + BIF_ERROR(BIF_P, BADARG); +} + + /**********************************************************************/ BIF_RETTYPE group_leader_0(BIF_ALIST_0) diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 08c8c3adbd..56da8703c2 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -88,6 +88,7 @@ bif erlang:list_to_binary/1 bif erlang:list_to_float/1 bif erlang:list_to_integer/1 bif erlang:list_to_pid/1 +bif erlang:list_to_ref/1 bif erlang:list_to_tuple/1 bif erlang:loaded/0 bif erlang:localtime/0 -- cgit v1.2.3 From 8b2a0f9940ffde4533d029a87ee30e290552db61 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 27 Feb 2017 16:18:41 +0100 Subject: Fix node_container_SUITE --- erts/emulator/beam/erl_node_tables.c | 41 ++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 14 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index e2072fe30f..89b627aaf5 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -850,14 +850,15 @@ static Eterm AM_timer; static Eterm AM_delayed_delete_timer; static void setup_reference_table(void); -static Eterm reference_table_term(Uint **hpp, Uint *szp); +static Eterm reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp); static void delete_reference_table(void); -#if BIG_UINT_HEAP_SIZE > 3 /* 2-tuple */ -#define ID_HEAP_SIZE BIG_UINT_HEAP_SIZE -#else -#define ID_HEAP_SIZE 3 /* 2-tuple */ -#endif +#undef ERTS_MAX__ +#define ERTS_MAX__(A, B) ((A) > (B) ? (A) : (B)) + +#define ID_HEAP_SIZE \ + ERTS_MAX__(ERTS_MAGIC_REF_THING_SIZE, \ + ERTS_MAX__(BIG_UINT_HEAP_SIZE, 3)) typedef struct node_referrer_ { struct node_referrer_ *next; @@ -870,6 +871,7 @@ typedef struct node_referrer_ { int system_ref; Eterm id; Uint id_heap[ID_HEAP_SIZE]; + ErlOffHeap off_heap; } NodeReferrer; typedef struct { @@ -942,7 +944,7 @@ erts_get_node_and_dist_references(struct process *proc) /* Get term size */ size = 0; - (void) reference_table_term(NULL, &size); + (void) reference_table_term(NULL, NULL, &size); hp = HAlloc(proc, size); #ifdef DEBUG @@ -951,7 +953,7 @@ erts_get_node_and_dist_references(struct process *proc) #endif /* Write term */ - res = reference_table_term(&hp, NULL); + res = reference_table_term(&hp, &proc->off_heap, NULL); ASSERT(endp == hp); @@ -1048,13 +1050,14 @@ insert_node_referrer(ReferredNode *referred_node, int type, Eterm id) nrp = (NodeReferrer *) erts_alloc(ERTS_ALC_T_NC_TMP, sizeof(NodeReferrer)); nrp->next = referred_node->referrers; + ERTS_INIT_OFF_HEAP(&nrp->off_heap); referred_node->referrers = nrp; if(IS_CONST(id)) nrp->id = id; else { Uint *hp = &nrp->id_heap[0]; - ASSERT(is_big(id) || is_tuple(id)); - nrp->id = copy_struct(id, size_object(id), &hp, NULL); + ASSERT(is_big(id) || is_tuple(id) || is_internal_magic_ref(id)); + nrp->id = copy_struct(id, size_object(id), &hp, &nrp->off_heap); } nrp->heap_ref = 0; nrp->link_ref = 0; @@ -1211,10 +1214,20 @@ insert_links2(ErtsLink *lnk, Eterm id) static void insert_ets_table(DbTable *tab, void *unused) { + ErlOffHeap off_heap; + Eterm heap[ERTS_MAGIC_REF_THING_SIZE]; struct insert_offheap2_arg a; a.type = ETS_REF; - a.id = tab->common.id; + if (tab->common.status & DB_NAMED_TABLE) + a.id = tab->common.the_name; + else { + Eterm *hp = &heap[0]; + ERTS_INIT_OFF_HEAP(&off_heap); + a.id = erts_mk_magic_ref(&hp, &off_heap, tab->common.btid); + } erts_db_foreach_offheap(tab, insert_offheap2, (void *) &a); + if (is_not_atom(a.id)) + erts_cleanup_offheap(&off_heap); } static void @@ -1518,7 +1531,7 @@ setup_reference_table(void) */ static Eterm -reference_table_term(Uint **hpp, Uint *szp) +reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp) { #undef MK_2TUP #undef MK_3TUP @@ -1573,12 +1586,11 @@ reference_table_term(Uint **hpp, Uint *szp) nrid = nrp->id; if (!IS_CONST(nrp->id)) { - Uint nrid_sz = size_object(nrp->id); if (szp) *szp += nrid_sz; if (hpp) - nrid = copy_struct(nrp->id, nrid_sz, hpp, NULL); + nrid = copy_struct(nrp->id, nrid_sz, hpp, ohp); } if (is_internal_pid(nrid) || nrid == am_error_logger) { @@ -1713,6 +1725,7 @@ delete_reference_table(void) NodeReferrer *tnrp; nrp = referred_nodes[i].referrers; while(nrp) { + erts_cleanup_offheap(&nrp->off_heap); tnrp = nrp; nrp = nrp->next; erts_free(ERTS_ALC_T_NC_TMP, (void *) tnrp); -- cgit v1.2.3 From 8d98992b9fa3059554c6cd5ff45ee3b47805fa1c Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 15 Feb 2017 19:19:02 +0100 Subject: erts: Add ERTS_RBT_YIELD_STAT_INIT to erl_rbtree for dynamic initialization of yield state. --- erts/emulator/beam/erl_rbtree.h | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_rbtree.h b/erts/emulator/beam/erl_rbtree.h index 5fefaea978..a10b1b1081 100644 --- a/erts/emulator/beam/erl_rbtree.h +++ b/erts/emulator/beam/erl_rbtree.h @@ -105,7 +105,10 @@ * _rbt_yield_state_t. * * The yield state should be statically initialized by - * ERTS_RBT_YIELD_STAT_INITER. + * ERTS_RBT_YIELD_STAT_INITER + * + * or dynamically initialized with + * ERTS_RBT_YIELD_STAT_INIT(_rbt_yield_state_t *ystate) * * * The following API functions are implemented if corresponding @@ -422,6 +425,13 @@ #ifndef ERTS_RBT_YIELD_STAT_INITER # define ERTS_RBT_YIELD_STAT_INITER {NULL, 0} #endif +#ifndef ERTS_RBT_YIELD_STAT_INIT +# define ERTS_RBT_YIELD_STAT_INIT(YS) \ + do { \ + (YS)->x = NULL; \ + (YS)->up = 0; \ + } while (0) +#endif #define ERTS_RBT_CONCAT_MACRO_VALUES___(X, Y) \ X ## Y -- cgit v1.2.3 From 832091f4dbf27c8bdc572383cbdeba5307d8e41c Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 15 Feb 2017 19:23:28 +0100 Subject: erts: Correct erl_rbtree comments about yielding true is yielding, false is done. and correct return value for unused foreach_ordered__ --- erts/emulator/beam/erl_rbtree.h | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_rbtree.h b/erts/emulator/beam/erl_rbtree.h index a10b1b1081..ebde5caca4 100644 --- a/erts/emulator/beam/erl_rbtree.h +++ b/erts/emulator/beam/erl_rbtree.h @@ -181,8 +181,8 @@ * Operate by calling the operator 'op' on each element. * Order is undefined. * - * Yield when 'ylimit' elements has been processed. Zero is - * returned when yielding, and a non-zero value is returned when + * Yield when 'ylimit' elements has been processed. True is + * returned when yielding, and false is returned when * the whole tree has been processed. The tree should not be * modified until all of it has been processed. * @@ -198,8 +198,8 @@ * Order is undefined. Each element should be destroyed * by 'op'. * - * Yield when 'ylimit' elements has been processed. Zero is - * returned when yielding, and a non-zero value is returned when + * Yield when 'ylimit' elements has been processed. True is + * returned when yielding, and false is returned when * the whole tree has been processed. * * 'arg' is passed as argument to 'op'. @@ -231,8 +231,8 @@ * Operate by calling the operator 'op' on each element from * smallest towards larger elements. * - * Yield when 'ylimit' elements has been processed. Zero is - * returned when yielding, and a non-zero value is returned when + * Yield when 'ylimit' elements has been processed. True is + * returned when yielding, and false is returned when * the whole tree has been processed. The tree should not be * modified until all of it has been processed. * @@ -247,8 +247,8 @@ * Operate by calling the operator 'op' on each element from * largest towards smaller elements. * - * Yield when 'ylimit' elements has been processed. Zero is - * returned when yielding, and a non-zero value is returned when + * Yield when 'ylimit' elements has been processed. True is + * returned when yielding, and false is returned when * the whole tree has been processed. The tree should not be * modified until all of it has been processed. * @@ -299,8 +299,8 @@ * Note that elements are often destroyed in another order * than the order that the elements are operated on. * - * Yield when 'ylimit' elements has been processed. Zero is - * returned when yielding, and a non-zero value is returned when + * Yield when 'ylimit' elements has been processed. True is + * returned when yielding, and false is returned when * the whole tree has been processed. The tree should not be * modified until all of it has been processed. * @@ -321,8 +321,8 @@ * Note that elements are often destroyed in another order * than the order that the elements are operated on. * - * Yield when 'ylimit' elements has been processed. Zero is - * returned when yielding, and a non-zero value is returned when + * Yield when 'ylimit' elements has been processed. True is + * returned when yielding, and false is returned when * the whole tree has been processed. The tree should not be * modified until all of it has been processed. * @@ -1374,7 +1374,7 @@ ERTS_RBT_FUNC__(foreach_ordered__)(ERTS_RBT_T **root, ystate->x = NULL; ystate->up = 0; } - return 1; /* Done */ + return 0; /* Done */ } x = p; } -- cgit v1.2.3 From b174a38c5f09859d3714fe5a81773aa5f2d19417 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 21 Feb 2017 16:07:50 +0100 Subject: erts: Replace meta_pid_to{_fixed}_tab with linked lists from process psd through all owned/fixed tables. As meta_pid_to{_fixed}_tab maps to slot in meta_main_tab which is planned for destruction. In this commit we no longer seize table lock while freeing the table (free_table_cont) as it's not needed and makes the code a bit simpler. Any concurrent operation on the table will only access lock, owner and status and then bail out. --- erts/emulator/beam/erl_bif_info.c | 8 - erts/emulator/beam/erl_db.c | 1030 ++++++++++++++++------------------- erts/emulator/beam/erl_db.h | 4 +- erts/emulator/beam/erl_db_hash.c | 152 +----- erts/emulator/beam/erl_db_hash.h | 10 - erts/emulator/beam/erl_db_util.h | 20 +- erts/emulator/beam/erl_lock_check.c | 6 +- erts/emulator/beam/erl_process.c | 38 +- erts/emulator/beam/erl_process.h | 16 +- erts/emulator/beam/erl_rbtree.h | 28 +- 10 files changed, 545 insertions(+), 767 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 06a73ffea5..025f99330a 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -3634,10 +3634,6 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(TUPLE2(hp, make_small((Uint) words), erts_ets_hash_sizeof_ext_segtab())); } - else if (ERTS_IS_ATOM_STR("DbTable_meta", BIF_ARG_1)) { - /* Used by ets_SUITE (stdlib) */ - BIF_RET(erts_ets_get_meta_state(BIF_P)); - } else if (ERTS_IS_ATOM_STR("check_io_debug", BIF_ARG_1)) { /* Used by driver_SUITE (emulator) */ Uint sz, *szp; @@ -4398,10 +4394,6 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } BIF_RET(am_ok); } - else if (ERTS_IS_ATOM_STR("DbTable_meta", BIF_ARG_1)) { - /* Used by ets_SUITE (stdlib) */ - BIF_RET(erts_ets_restore_meta_state(BIF_P, BIF_ARG_2)); - } else if (ERTS_IS_ATOM_STR("make", BIF_ARG_1)) { if (ERTS_IS_ATOM_STR("magic_ref", BIF_ARG_2)) { Binary *bin = erts_create_magic_binary(0, empty_magic_ref_destructor); diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 8c4c76e187..24de21cc24 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -19,9 +19,7 @@ */ /* - * This file contains the bif interface functions and - * the handling of the "meta tables" ie the tables of - * db tables. + * This file contains the 'ets' bif interface functions. */ /* @@ -75,6 +73,113 @@ enum DbIterSafety { #define DID_TRAP(P,Ret) (!is_value(Ret) && ((P)->freason == TRAP)) +/* + * "fixed_tabs": list of all fixed tables for a process + */ +#ifdef DEBUG +static int fixed_tabs_find(DbFixation* first, DbFixation* fix); +#endif + +static void fixed_tabs_insert(Process* p, DbFixation* fix) +{ + DbFixation* first = erts_psd_get(p, ERTS_PSD_ETS_FIXED_TABLES); + + if (!first) { + fix->tabs.next = fix->tabs.prev = fix; + erts_psd_set(p, ERTS_PSD_ETS_FIXED_TABLES, fix); + } + else { + ASSERT(!fixed_tabs_find(first, fix)); + fix->tabs.prev = first->tabs.prev; + fix->tabs.next = first; + fix->tabs.prev->tabs.next = fix; + first->tabs.prev = fix; + } +} + +static void fixed_tabs_delete(Process *p, DbFixation* fix) +{ + if (fix->tabs.next == fix) { + DbFixation* old; + ASSERT(fix->tabs.prev == fix); + old = erts_psd_set(p, ERTS_PSD_ETS_FIXED_TABLES, NULL); + ASSERT(old == fix); (void)old; + } + else { + DbFixation *first = (DbFixation*) erts_psd_get(p, ERTS_PSD_ETS_FIXED_TABLES); + + ASSERT(fixed_tabs_find(first, fix)); + fix->tabs.prev->tabs.next = fix->tabs.next; + fix->tabs.next->tabs.prev = fix->tabs.prev; + + if (fix == first) + erts_psd_set(p, ERTS_PSD_ETS_FIXED_TABLES, fix->tabs.next); + } +} + +#ifdef DEBUG +static int fixed_tabs_find(DbFixation* first, DbFixation* fix) +{ + DbFixation* p; + + if (!first) { + first = (DbFixation*) erts_psd_get(fix->procs.p, ERTS_PSD_ETS_FIXED_TABLES); + } + p = first; + do { + if (p == fix) + return 1; + ASSERT(p->procs.p == fix->procs.p); + ASSERT(p->tabs.next->tabs.prev == p); + p = p->tabs.next; + } while (p != first); + return 0; +} +#endif + + +/* + * fixing_procs: tree of all processes fixating a table + */ +#define ERTS_RBT_PREFIX fixing_procs +#define ERTS_RBT_T DbFixation +#define ERTS_RBT_KEY_T Process* +#define ERTS_RBT_FLAGS_T int +#define ERTS_RBT_INIT_EMPTY_TNODE(T) \ + do { \ + (T)->procs.parent = NULL; \ + (T)->procs.right = NULL; \ + (T)->procs.left = NULL; \ + } while (0) +#define ERTS_RBT_IS_RED(T) ((T)->procs.is_red) +#define ERTS_RBT_SET_RED(T) ((T)->procs.is_red = 1) +#define ERTS_RBT_IS_BLACK(T) (!(T)->procs.is_red) +#define ERTS_RBT_SET_BLACK(T) ((T)->procs.is_red = 0) +#define ERTS_RBT_GET_FLAGS(T) ((T)->procs.is_red) +#define ERTS_RBT_SET_FLAGS(T, F) ((T)->procs.is_red = (F)) +#define ERTS_RBT_GET_PARENT(T) ((T)->procs.parent) +#define ERTS_RBT_SET_PARENT(T, P) ((T)->procs.parent = (P)) +#define ERTS_RBT_GET_RIGHT(T) ((T)->procs.right) +#define ERTS_RBT_SET_RIGHT(T, R) ((T)->procs.right = (R)) +#define ERTS_RBT_GET_LEFT(T) ((T)->procs.left) +#define ERTS_RBT_SET_LEFT(T, L) ((T)->procs.left = (L)) +#define ERTS_RBT_GET_KEY(T) ((T)->procs.p) +#define ERTS_RBT_IS_LT(KX, KY) ((KX) < (KY)) +#define ERTS_RBT_IS_EQ(KX, KY) ((KX) == (KY)) + +#define ERTS_RBT_WANT_INSERT +#define ERTS_RBT_WANT_LOOKUP +#define ERTS_RBT_WANT_DELETE +#define ERTS_RBT_WANT_FOREACH +#define ERTS_RBT_WANT_FOREACH_DESTROY +#ifdef DEBUG +# define ERTS_RBT_WANT_LOOKUP +#endif +#define ERTS_RBT_UNDEF + +#include "erl_rbtree.h" + + /* ** The main meta table, containing all ets tables. @@ -148,6 +253,12 @@ make_btid(DbTable *tb) erts_refc_inc(&btid->refc, 1); } +static ERTS_INLINE DbTable* btid2tab(Binary* btid) +{ + erts_smp_atomic_t *tbref = erts_smp_binary_to_magic_indirection(btid); + return (DbTable *) erts_smp_atomic_read_nob(tbref); +} + static DbTable * tid2tab(Eterm tid) { @@ -272,8 +383,6 @@ int user_requested_db_max_tabs; int erts_ets_realloc_always_moves; int erts_ets_always_compress; static int db_max_tabs; -static DbTable *meta_pid_to_tab; /* Pid mapped to owned tables */ -static DbTable *meta_pid_to_fixed_tab; /* Pid mapped to fixed tables */ static Eterm ms_delete_all; static Eterm ms_delete_all_buff[8]; /* To compare with for deletion of all objects */ @@ -286,12 +395,9 @@ static void fix_table_locked(Process* p, DbTable* tb); static void unfix_table_locked(Process* p, DbTable* tb, db_lock_kind_t* kind); static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data); static void free_heir_data(DbTable*); -static void free_fixations_locked(DbTable *tb); +static void free_fixations_locked(Process* p, DbTable *tb); -static int free_table_cont(Process *p, - DbTable *tb, - int first, - int clean_meta_tab); +static int free_table_cont(Process *p, DbTable *tb); static void print_table(fmtfn_t to, void *to_arg, int show, DbTable* tb); static BIF_RETTYPE ets_select_delete_1(BIF_ALIST_1); static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1); @@ -328,15 +434,6 @@ free_dbtable(void *vtb) } erts_fprintf(stderr, "ets: free_dbtable(%T) deleted!!!\r\n", tb->common.id); - - erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_tab common.memory_size = %ld\n", - erts_smp_atomic_read_nob(&meta_pid_to_tab->common.memory_size)); - print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_tab); - - - erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_fixed_tab common.memory_size = %ld\n", - erts_smp_atomic_read_nob(&meta_pid_to_fixed_tab->common.memory_size)); - print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_fixed_tab); #endif #ifdef ERTS_SMP erts_smp_rwmtx_destroy(&tb->common.rwlock); @@ -462,6 +559,62 @@ delete_sched_table(Process *c_p, DbTable *tb) erts_schedule_misc_aux_work((int) sid, scheduled_remove_sched_table, tb); } +static ERTS_INLINE void +save_owned_table(Process *c_p, DbTable *tb) +{ + DbTable *first; + + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); + + first = (DbTable*) erts_psd_get(c_p, ERTS_PSD_ETS_OWNED_TABLES); + + erts_smp_refc_inc(&tb->common.refc, 1); + + if (!first) { + tb->common.owned.next = tb->common.owned.prev = tb; + erts_psd_set(c_p, ERTS_PSD_ETS_OWNED_TABLES, tb); + } + else { + tb->common.owned.prev = first->common.owned.prev; + tb->common.owned.next = first; + tb->common.owned.prev->common.owned.next = tb; + first->common.owned.prev = tb; + } + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); +} + +static ERTS_INLINE void +delete_owned_table(Process *p, DbTable *tb) +{ + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + if (tb->common.owned.next == tb) { + DbTable* old; + ASSERT(tb->common.owned.prev == tb); + old = erts_psd_set(p, ERTS_PSD_ETS_OWNED_TABLES, NULL); + ASSERT(old == tb); (void)old; + } + else { + DbTable *first = (DbTable*) erts_psd_get(p, ERTS_PSD_ETS_OWNED_TABLES); +#ifdef DEBUG + DbTable *tmp = first; + do { + if (tmp == tb) break; + tmp = tmp->common.owned.next; + } while (tmp != first); + ASSERT(tmp == tb); +#endif + tb->common.owned.prev->common.owned.next = tb->common.owned.next; + tb->common.owned.next->common.owned.prev = tb->common.owned.prev; + + if (tb == first) + erts_psd_set(p, ERTS_PSD_ETS_OWNED_TABLES, tb->common.owned.next); + } + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + + table_dec_refc(tb, 1); +} + + static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock, char *rwname, char* fixname) { @@ -483,7 +636,6 @@ static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock, static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind) { #ifdef ERTS_SMP - ASSERT(tb != meta_pid_to_tab && tb != meta_pid_to_fixed_tab); if (tb->common.type & DB_FINE_LOCKED) { if (kind == LCK_WRITE) { erts_smp_rwmtx_rwlock(&tb->common.rwlock); @@ -516,8 +668,6 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind) * to follow the tb pointer! */ #ifdef ERTS_SMP - ASSERT(tb != meta_pid_to_tab && tb != meta_pid_to_fixed_tab); - if (tb->common.type & DB_FINE_LOCKED) { if (kind == LCK_WRITE) { ASSERT(tb->common.is_thread_safe); @@ -543,20 +693,6 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind) #endif } - -static ERTS_INLINE void db_meta_lock(DbTable* tb, db_lock_kind_t kind) -{ - ASSERT(tb == meta_pid_to_tab || tb == meta_pid_to_fixed_tab); - ASSERT(kind != LCK_WRITE); - /* As long as we only lock for READ we don't have to lock at all. */ -} - -static ERTS_INLINE void db_meta_unlock(DbTable* tb, db_lock_kind_t kind) -{ - ASSERT(tb == meta_pid_to_tab || tb == meta_pid_to_fixed_tab); - ASSERT(kind != LCK_WRITE); -} - static ERTS_INLINE DbTable* db_get_table_aux(Process *p, Eterm id, @@ -1544,7 +1680,6 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) #ifdef DEBUG int cret; #endif - DeclareTmpHeap(meta_tuple,3,BIF_P); DbTableMethod* meth; erts_smp_rwmtx_t *mmtl; @@ -1701,7 +1836,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) erts_smp_atomic_init_nob(&tb->common.nitems, 0); - tb->common.fixations = NULL; + tb->common.fixing_procs = NULL; tb->common.compress = is_compressed; #ifdef DEBUG @@ -1774,32 +1909,15 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) } BIF_P->flags |= F_USING_DB; /* So we can remove tb if p dies */ + save_owned_table(BIF_P, tb); #ifdef HARDDEBUG erts_fprintf(stderr, "ets:new(%T,%T)=%T; Process: %T, initial: %T:%T/%bpu\n", BIF_ARG_1, BIF_ARG_2, ret, BIF_P->common.id, BIF_P->u.initial[0], BIF_P->u.initial[1], BIF_P->u.initial[2]); - erts_fprintf(stderr, "ets: new: meta_pid_to_tab common.memory_size = %ld\n", - erts_smp_atomic_read_nob(&meta_pid_to_tab->common.memory_size)); - erts_fprintf(stderr, "ets: new: meta_pid_to_fixed_tab common.memory_size = %ld\n", - erts_smp_atomic_read_nob(&meta_pid_to_fixed_tab->common.memory_size)); #endif - UseTmpHeap(3,BIF_P); - - db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC); - if (db_put_hash(meta_pid_to_tab, - TUPLE2(meta_tuple, - BIF_P->common.id, - make_small(slot)), - 0) != DB_ERROR_NONE) { - erts_exit(ERTS_ERROR_EXIT,"Could not update ets metadata."); - } - db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); - - UnUseTmpHeap(3,BIF_P); - BIF_RET(ret); } @@ -1929,7 +2047,6 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) tb->common.status |= DB_DELETE; if (tb->common.owner != BIF_P->common.id) { - DeclareTmpHeap(meta_tuple,3,BIF_P); /* * The table is being deleted by a process other than its owner. @@ -1937,22 +2054,19 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) * current process will be killed (e.g. by an EXIT signal), we will * now transfer the ownership to the current process. */ - UseTmpHeap(3,BIF_P); - db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC); - db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner, - make_small(tb->common.slot)); - - BIF_P->flags |= F_USING_DB; - tb->common.owner = BIF_P->common.id; - - db_put_hash(meta_pid_to_tab, - TUPLE2(meta_tuple, - BIF_P->common.id, - make_small(tb->common.slot)), - 0); - db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); - UnUseTmpHeap(3,BIF_P); - } + + Process *rp = erts_proc_lookup_raw(tb->common.owner); + /* + * Process 'rp' might be exiting, but our table lock prevents it + * from terminating as it cannot complete erts_db_process_exiting(). + */ + ASSERT(!(ERTS_PSFLG_FREE & erts_smp_atomic32_read_nob(&rp->state))); + + delete_owned_table(rp, tb); + BIF_P->flags |= F_USING_DB; + tb->common.owner = BIF_P->common.id; + save_owned_table(BIF_P, tb); + } tid_clear(BIF_P, tb); @@ -1978,10 +2092,10 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) free_heir_data(tb); tb->common.heir = am_none; - free_fixations_locked(tb); - - trap = free_table_cont(BIF_P, tb, 1, 1); + free_fixations_locked(BIF_P, tb); db_unlock(tb, LCK_WRITE); + + trap = free_table_cont(BIF_P, tb); if (trap) { /* * Package the DbTable* pointer into a bignum so that it can be safely @@ -2006,7 +2120,6 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3) { Process* to_proc = NULL; ErtsProcLocks to_locks = ERTS_PROC_LOCK_MAIN; - DeclareTmpHeap(buf,5,BIF_P); Eterm to_pid = BIF_ARG_2; Eterm from_pid; DbTable* tb = NULL; @@ -2028,18 +2141,10 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3) goto badarg; /* or should we be idempotent? return false maybe */ } - UseTmpHeap(5,BIF_P); - db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC); - db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner, - make_small(tb->common.slot)); - + delete_owned_table(BIF_P, tb); to_proc->flags |= F_USING_DB; tb->common.owner = to_pid; - - db_put_hash(meta_pid_to_tab, - TUPLE2(buf,to_pid,make_small(tb->common.slot)), - 0); - db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); + save_owned_table(to_proc, tb); db_unlock(tb,LCK_WRITE); send_ets_transfer_message(BIF_P, to_proc, &to_locks, @@ -3273,7 +3378,6 @@ int erts_ets_rwmtx_spin_count = -1; void init_db(ErtsDbSpinCount db_spin_count) { - DbTable init_tb; int i; Eterm *hp; unsigned bits; @@ -3379,72 +3483,6 @@ void init_db(ErtsDbSpinCount db_spin_count) db_initialize_hash(); db_initialize_tree(); - /*TT*/ - /* Create meta table invertion. */ - erts_smp_atomic_init_nob(&init_tb.common.memory_size, 0); - meta_pid_to_tab = (DbTable*) erts_db_alloc(ERTS_ALC_T_DB_TABLE, - &init_tb, - sizeof(DbTable)); - erts_smp_atomic_init_nob(&meta_pid_to_tab->common.memory_size, - erts_smp_atomic_read_nob(&init_tb.common.memory_size)); - - meta_pid_to_tab->common.id = NIL; - meta_pid_to_tab->common.btid = NULL; - meta_pid_to_tab->common.the_name = am_true; - meta_pid_to_tab->common.status = (DB_NORMAL | DB_BAG | DB_PUBLIC | DB_FINE_LOCKED); -#ifdef ERTS_SMP - meta_pid_to_tab->common.type - = meta_pid_to_tab->common.status & ERTS_ETS_TABLE_TYPES; - /* Note, 'type' is *read only* from now on... */ - meta_pid_to_tab->common.is_thread_safe = 0; -#endif - meta_pid_to_tab->common.keypos = 1; - meta_pid_to_tab->common.owner = NIL; - erts_smp_atomic_init_nob(&meta_pid_to_tab->common.nitems, 0); - meta_pid_to_tab->common.slot = -1; - meta_pid_to_tab->common.meth = &db_hash; - meta_pid_to_tab->common.compress = 0; - - erts_smp_refc_init(&meta_pid_to_tab->common.fix_count, 0); - /* Neither rwlock or fixlock used - db_init_lock(meta_pid_to_tab, "meta_pid_to_tab", "meta_pid_to_tab_FIX");*/ - - if (db_create_hash(NULL, meta_pid_to_tab) != DB_ERROR_NONE) { - erts_exit(ERTS_ERROR_EXIT,"Unable to create ets metadata tables."); - } - - erts_smp_atomic_set_nob(&init_tb.common.memory_size, 0); - meta_pid_to_fixed_tab = (DbTable*) erts_db_alloc(ERTS_ALC_T_DB_TABLE, - &init_tb, - sizeof(DbTable)); - erts_smp_atomic_init_nob(&meta_pid_to_fixed_tab->common.memory_size, - erts_smp_atomic_read_nob(&init_tb.common.memory_size)); - - meta_pid_to_fixed_tab->common.id = NIL; - meta_pid_to_fixed_tab->common.btid = NULL; - meta_pid_to_fixed_tab->common.the_name = am_true; - meta_pid_to_fixed_tab->common.status = (DB_NORMAL | DB_BAG | DB_PUBLIC | DB_FINE_LOCKED); -#ifdef ERTS_SMP - meta_pid_to_fixed_tab->common.type - = meta_pid_to_fixed_tab->common.status & ERTS_ETS_TABLE_TYPES; - /* Note, 'type' is *read only* from now on... */ - meta_pid_to_fixed_tab->common.is_thread_safe = 0; -#endif - meta_pid_to_fixed_tab->common.keypos = 1; - meta_pid_to_fixed_tab->common.owner = NIL; - erts_smp_atomic_init_nob(&meta_pid_to_fixed_tab->common.nitems, 0); - meta_pid_to_fixed_tab->common.slot = -1; - meta_pid_to_fixed_tab->common.meth = &db_hash; - meta_pid_to_fixed_tab->common.compress = 0; - - erts_smp_refc_init(&meta_pid_to_fixed_tab->common.fix_count, 0); - /* Neither rwlock or fixlock used - db_init_lock(meta_pid_to_fixed_tab, "meta_pid_to_fixed_tab", "meta_pid_to_fixed_tab_FIX");*/ - - if (db_create_hash(NULL, meta_pid_to_fixed_tab) != DB_ERROR_NONE) { - erts_exit(ERTS_ERROR_EXIT,"Unable to create ets metadata tables."); - } - /* Non visual BIF to trap to. */ erts_init_trap_export(&ets_select_delete_continue_exp, am_ets, am_atom_put("delete_trap",11), 1, @@ -3486,82 +3524,6 @@ erts_ets_sched_spec_data_init(ErtsSchedulerData *esdp) } -#define ARRAY_CHUNK 100 - -typedef enum { - ErtsDbProcCleanupProgressTables, - ErtsDbProcCleanupProgressFixations, - ErtsDbProcCleanupProgressDone, -} ErtsDbProcCleanupProgress; - -typedef enum { - ErtsDbProcCleanupOpGetTables, - ErtsDbProcCleanupOpDeleteTables, - ErtsDbProcCleanupOpGetFixations, - ErtsDbProcCleanupOpDeleteFixations, - ErtsDbProcCleanupOpDone -} ErtsDbProcCleanupOperation; - -typedef struct { - ErtsDbProcCleanupProgress progress; - ErtsDbProcCleanupOperation op; - struct { - Eterm arr[ARRAY_CHUNK]; - int size; - int ix; - int clean_ix; - } slots; -} ErtsDbProcCleanupState; - - -static void -proc_exit_cleanup_tables_meta_data(Eterm pid, ErtsDbProcCleanupState *state) -{ - ASSERT(state->slots.clean_ix <= state->slots.ix); - if (state->slots.clean_ix < state->slots.ix) { - db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC); - if (state->slots.size < ARRAY_CHUNK - && state->slots.ix == state->slots.size) { - Eterm dummy; - db_erase_hash(meta_pid_to_tab,pid,&dummy); - } - else { - int ix; - /* Need to erase each explicitly */ - for (ix = state->slots.clean_ix; ix < state->slots.ix; ix++) - db_erase_bag_exact2(meta_pid_to_tab, - pid, - state->slots.arr[ix]); - } - db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); - state->slots.clean_ix = state->slots.ix; - } -} - -static void -proc_exit_cleanup_fixations_meta_data(Eterm pid, ErtsDbProcCleanupState *state) -{ - ASSERT(state->slots.clean_ix <= state->slots.ix); - if (state->slots.clean_ix < state->slots.ix) { - db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC); - if (state->slots.size < ARRAY_CHUNK - && state->slots.ix == state->slots.size) { - Eterm dummy; - db_erase_hash(meta_pid_to_fixed_tab,pid,&dummy); - } - else { - int ix; - /* Need to erase each explicitly */ - for (ix = state->slots.clean_ix; ix < state->slots.ix; ix++) - db_erase_bag_exact2(meta_pid_to_fixed_tab, - pid, - state->slots.arr[ix]); - } - db_meta_unlock(meta_pid_to_fixed_tab, LCK_WRITE_REC); - state->slots.clean_ix = state->slots.ix; - } -} - /* In: Table LCK_WRITE ** Return TRUE : ok, table not mine and NOT locked anymore. ** Return FALSE: failed, table still mine (LCK_WRITE) @@ -3570,7 +3532,6 @@ static int give_away_to_heir(Process* p, DbTable* tb) { Process* to_proc; ErtsProcLocks to_locks = ERTS_PROC_LOCK_MAIN; - DeclareTmpHeap(buf,5,p); Eterm to_pid; UWord heir_data; @@ -3614,19 +3575,12 @@ retry: erts_smp_proc_unlock(to_proc, to_locks); return 0; /* heir dead and pid reused, table still mine */ } - UseTmpHeap(5,p); - db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC); - db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner, - make_small(tb->common.slot)); + delete_owned_table(p, tb); to_proc->flags |= F_USING_DB; tb->common.owner = to_pid; - - db_put_hash(meta_pid_to_tab, - TUPLE2(buf,to_pid,make_small(tb->common.slot)), - 0); - db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); - UnUseTmpHeap(5,p); + save_owned_table(to_proc, tb); + db_unlock(tb,LCK_WRITE); heir_data = tb->common.heir_data; if (!is_immed(heir_data)) { @@ -3674,6 +3628,56 @@ send_ets_transfer_message(Process *c_p, Process *proc, erts_queue_message(proc, *locks, mp, msg, sender); } + +/* Auto-release fixation from exiting process */ +static void proc_cleanup_fixed_table(Process* p, DbFixation* fix) +{ + DbTable* tb = btid2tab(fix->tabs.btid); + + ASSERT(fix->procs.p == p); (void)p; + if (tb) { + db_lock(tb, LCK_WRITE_REC); + if (!(tb->common.status & DB_DELETE)) { + erts_aint_t diff; + #ifdef ERTS_SMP + erts_smp_mtx_lock(&tb->common.fixlock); + #endif + + ASSERT(fixing_procs_rbt_lookup(tb->common.fixing_procs, p)); + + diff = -((erts_aint_t) fix->counter); + erts_smp_refc_add(&tb->common.fix_count,diff,0); + fix->counter = 0; + + fixing_procs_rbt_delete(&tb->common.fixing_procs, fix); + + #ifdef ERTS_SMP + erts_smp_mtx_unlock(&tb->common.fixlock); + #endif + if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status)) { + db_unfix_table_hash(&(tb->hash)); + } + + ASSERT(sizeof(DbFixation) == ERTS_ALC_DBG_BLK_SZ(fix)); + ERTS_DB_ALC_MEM_UPDATE_(tb, sizeof(DbFixation), 0); + } + else { + ASSERT(fix->counter == 0); + } + db_unlock(tb, LCK_WRITE_REC); + } + else { + ASSERT(fix->counter == 0); + } + + if (erts_refc_dectest(&fix->tabs.btid->refc, 0) == 0) { + erts_bin_free(fix->tabs.btid); + } + erts_free(ERTS_ALC_T_DB_FIXATION, fix); + ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation)); +} + + /* * erts_db_process_exiting() is called when a process terminates. * It returns 0 when completely done, and !0 when it wants to @@ -3687,277 +3691,157 @@ send_ets_transfer_message(Process *c_p, Process *proc, int erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) { - ErtsDbProcCleanupState *state = (ErtsDbProcCleanupState *) c_p->u.terminate; + typedef struct { + enum { + GET_OWNED_TABLE, + FREE_OWNED_TABLE, + UNFIX_TABLES, + }op; + DbTable *tb; + } CleanupState; + CleanupState *state = (CleanupState *) c_p->u.terminate; Eterm pid = c_p->common.id; - ErtsDbProcCleanupState default_state; - int ret; + CleanupState default_state; if (!state) { state = &default_state; - state->progress = ErtsDbProcCleanupProgressTables; - state->op = ErtsDbProcCleanupOpGetTables; + state->op = GET_OWNED_TABLE; + state->tb = NULL; } - while (!0) { + do { switch (state->op) { - case ErtsDbProcCleanupOpGetTables: - state->slots.size = ARRAY_CHUNK; - db_meta_lock(meta_pid_to_tab, LCK_READ); - ret = db_get_element_array(meta_pid_to_tab, - pid, - 2, - state->slots.arr, - &state->slots.size); - db_meta_unlock(meta_pid_to_tab, LCK_READ); - if (ret == DB_ERROR_BADKEY) { - /* Done with tables; now fixations */ - state->progress = ErtsDbProcCleanupProgressFixations; - state->op = ErtsDbProcCleanupOpGetFixations; - break; - } else if (ret != DB_ERROR_NONE) { - ERTS_DB_INTERNAL_ERROR("Inconsistent ets table metadata"); - } - - state->slots.ix = 0; - state->slots.clean_ix = 0; - state->op = ErtsDbProcCleanupOpDeleteTables; - /* Fall through */ - - case ErtsDbProcCleanupOpDeleteTables: - - while (state->slots.ix < state->slots.size) { - DbTable *tb = NULL; - Sint ix = unsigned_val(state->slots.arr[state->slots.ix]); - erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix); - erts_smp_rwmtx_rlock(mmtl); - if (!IS_SLOT_FREE(ix)) { - tb = GET_ANY_SLOT_TAB(ix); - ASSERT(tb); - } - erts_smp_rwmtx_runlock(mmtl); - if (tb) { - int do_yield; - db_lock(tb, LCK_WRITE); - /* Ownership may have changed since - we looked up the table. */ - if (tb->common.owner != pid) { - do_yield = 0; - db_unlock(tb, LCK_WRITE); - } - else if (tb->common.heir != am_none - && tb->common.heir != pid - && give_away_to_heir(c_p, tb)) { - do_yield = 0; - } - else { - int first_call; -#ifdef HARDDEBUG - erts_fprintf(stderr, - "erts_db_process_exiting(); Table: %T, " - "Process: %T\n", - tb->common.id, pid); -#endif - first_call = (tb->common.status & DB_DELETE) == 0; - if (first_call) { - tid_clear(c_p, tb); - /* Clear all access bits. */ - tb->common.status &= ~(DB_PROTECTED - | DB_PUBLIC - | DB_PRIVATE); - tb->common.status |= DB_DELETE; - - if (is_atom(tb->common.id)) - remove_named_tab(tb, 0); - - free_heir_data(tb); - free_fixations_locked(tb); - } - - do_yield = free_table_cont(c_p, tb, first_call, 0); - db_unlock(tb, LCK_WRITE); - } - if (do_yield) - goto yield; - } - state->slots.ix++; - if (ERTS_BIF_REDS_LEFT(c_p) <= 0) - goto yield; - } + case GET_OWNED_TABLE: { + DbTable* tb; + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); + tb = (DbTable*) erts_psd_get(c_p, ERTS_PSD_ETS_OWNED_TABLES); + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); + + if (!tb) { + /* Done with owned tables; now fixations */ + state->op = UNFIX_TABLES; + break; + } - proc_exit_cleanup_tables_meta_data(pid, state); - state->op = ErtsDbProcCleanupOpGetTables; - break; + ASSERT(tb != state->tb); + state->tb = tb; + db_lock(tb, LCK_WRITE); + /* + * Ownership may have changed since we looked up the table. + */ + if (tb->common.owner != pid) { + db_unlock(tb, LCK_WRITE); + break; + } + if (tb->common.heir != am_none + && tb->common.heir != pid + && give_away_to_heir(c_p, tb)) { + break; + } + tid_clear(c_p, tb); + /* Clear all access bits. */ + tb->common.status &= ~(DB_PROTECTED | DB_PUBLIC | DB_PRIVATE); + tb->common.status |= DB_DELETE; + + if (is_atom(tb->common.id)) + remove_named_tab(tb, 0); + + free_heir_data(tb); + free_fixations_locked(c_p, tb); + db_unlock(tb, LCK_WRITE); + state->op = FREE_OWNED_TABLE; + /* fall through */ + } + case FREE_OWNED_TABLE: + if (free_table_cont(c_p, state->tb)) + goto yield; - case ErtsDbProcCleanupOpGetFixations: - state->slots.size = ARRAY_CHUNK; - db_meta_lock(meta_pid_to_fixed_tab, LCK_READ); - ret = db_get_element_array(meta_pid_to_fixed_tab, - pid, - 2, - state->slots.arr, - &state->slots.size); - db_meta_unlock(meta_pid_to_fixed_tab, LCK_READ); - - if (ret == DB_ERROR_BADKEY) { - /* Done */ - state->progress = ErtsDbProcCleanupProgressDone; - state->op = ErtsDbProcCleanupOpDone; - break; - } else if (ret != DB_ERROR_NONE) { - ERTS_DB_INTERNAL_ERROR("Inconsistent ets fix table metadata"); - } + state->op = GET_OWNED_TABLE; - state->slots.ix = 0; - state->slots.clean_ix = 0; - state->op = ErtsDbProcCleanupOpDeleteFixations; - /* Fall through */ + break; - case ErtsDbProcCleanupOpDeleteFixations: + case UNFIX_TABLES: { + DbFixation* fix; - while (state->slots.ix < state->slots.size) { - DbTable *tb = NULL; - Sint ix = unsigned_val(state->slots.arr[state->slots.ix]); - erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix); - erts_smp_rwmtx_rlock(mmtl); - if (IS_SLOT_ALIVE(ix)) { - tb = meta_main_tab[ix].u.tb; - ASSERT(tb); - } - erts_smp_rwmtx_runlock(mmtl); - if (tb) { - int reds = 0; - - db_lock(tb, LCK_WRITE_REC); - if (!(tb->common.status & DB_DELETE)) { - DbFixation** pp; - - #ifdef ERTS_SMP - erts_smp_mtx_lock(&tb->common.fixlock); - #endif - reds = 10; - - for (pp = &tb->common.fixations; *pp != NULL; - pp = &(*pp)->next) { - if ((*pp)->pid == pid) { - DbFixation* fix = *pp; - erts_aint_t diff = -((erts_aint_t) fix->counter); - erts_smp_refc_add(&tb->common.fix_count,diff,0); - *pp = fix->next; - erts_db_free(ERTS_ALC_T_DB_FIXATION, - tb, fix, sizeof(DbFixation)); - ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation)); - break; - } - } - #ifdef ERTS_SMP - erts_smp_mtx_unlock(&tb->common.fixlock); - #endif - if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status)) { - db_unfix_table_hash(&(tb->hash)); - reds += 40; - } - } - db_unlock(tb, LCK_WRITE_REC); - BUMP_REDS(c_p, reds); - } - state->slots.ix++; - if (ERTS_BIF_REDS_LEFT(c_p) <= 0) - goto yield; - } + fix = (DbFixation*) erts_psd_get(c_p, ERTS_PSD_ETS_FIXED_TABLES); - proc_exit_cleanup_fixations_meta_data(pid, state); - state->op = ErtsDbProcCleanupOpGetFixations; - break; + if (!fix) { + /* Done */ - case ErtsDbProcCleanupOpDone: + if (state != &default_state) + erts_free(ERTS_ALC_T_DB_PROC_CLEANUP, state); + c_p->u.terminate = NULL; + return 0; + } - if (state != &default_state) - erts_free(ERTS_ALC_T_DB_PROC_CLEANUP, state); - c_p->u.terminate = NULL; - return 0; + fixed_tabs_delete(c_p, fix); + proc_cleanup_fixed_table(c_p, fix); + BUMP_REDS(c_p, 10); + break; + } default: ERTS_DB_INTERNAL_ERROR("Bad internal state"); - } - } - - yield: + } - switch (state->progress) { - case ErtsDbProcCleanupProgressTables: - proc_exit_cleanup_tables_meta_data(pid, state); - break; - case ErtsDbProcCleanupProgressFixations: - proc_exit_cleanup_fixations_meta_data(pid, state); - break; - default: - break; - } + } while (ERTS_BIF_REDS_LEFT(c_p) > 0); - ASSERT(c_p->u.terminate == (void *) state - || state == &default_state); + yield: if (state == &default_state) { c_p->u.terminate = erts_alloc(ERTS_ALC_T_DB_PROC_CLEANUP, - sizeof(ErtsDbProcCleanupState)); - sys_memcpy(c_p->u.terminate, - (void*) state, - sizeof(ErtsDbProcCleanupState)); + sizeof(CleanupState)); + sys_memcpy(c_p->u.terminate, (void*) state, sizeof(CleanupState)); } + else + ASSERT(state == c_p->u.terminate); return !0; } + /* SMP note: table only need to be LCK_READ locked */ static void fix_table_locked(Process* p, DbTable* tb) { DbFixation *fix; - DeclareTmpHeap(meta_tuple,3,p); #ifdef ERTS_SMP erts_smp_mtx_lock(&tb->common.fixlock); #endif erts_smp_refc_inc(&tb->common.fix_count,1); - fix = tb->common.fixations; + fix = tb->common.fixing_procs; if (fix == NULL) { tb->common.time.monotonic = erts_get_monotonic_time(erts_proc_sched_data(p)); tb->common.time.offset = erts_get_time_offset(); } else { - for (; fix != NULL; fix = fix->next) { - if (fix->pid == p->common.id) { - ++(fix->counter); + fix = fixing_procs_rbt_lookup(fix, p); + if (fix) { + ASSERT(fixed_tabs_find(NULL, fix)); + ++(fix->counter); + #ifdef ERTS_SMP - erts_smp_mtx_unlock(&tb->common.fixlock); + erts_smp_mtx_unlock(&tb->common.fixlock); #endif - return; - } + return; } } fix = (DbFixation *) erts_db_alloc(ERTS_ALC_T_DB_FIXATION, tb, sizeof(DbFixation)); ERTS_ETS_MISC_MEM_ADD(sizeof(DbFixation)); - fix->pid = p->common.id; + fix->tabs.btid = tb->common.btid; + erts_refc_inc(&fix->tabs.btid->refc, 2); + fix->procs.p = p; fix->counter = 1; - fix->next = tb->common.fixations; - tb->common.fixations = fix; + fixing_procs_rbt_insert(&tb->common.fixing_procs, fix); + #ifdef ERTS_SMP erts_smp_mtx_unlock(&tb->common.fixlock); #endif - p->flags |= F_USING_DB; - UseTmpHeap(3,p); - db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC); - if (db_put_hash(meta_pid_to_fixed_tab, - TUPLE2(meta_tuple, - p->common.id, - make_small(tb->common.slot)), - 0) != DB_ERROR_NONE) { - UnUseTmpHeap(3,p); - erts_exit(ERTS_ERROR_EXIT,"Could not insert ets metadata in safe_fixtable."); - } - UnUseTmpHeap(3,p); - db_meta_unlock(meta_pid_to_fixed_tab, LCK_WRITE_REC); + p->flags |= F_USING_DB; + + fixed_tabs_insert(p, fix); } /* SMP note: May re-lock table @@ -3965,28 +3849,26 @@ static void fix_table_locked(Process* p, DbTable* tb) static void unfix_table_locked(Process* p, DbTable* tb, db_lock_kind_t* kind_p) { - DbFixation** pp; + DbFixation* fix; #ifdef ERTS_SMP erts_smp_mtx_lock(&tb->common.fixlock); #endif - for (pp = &tb->common.fixations; *pp != NULL; pp = &(*pp)->next) { - if ((*pp)->pid == p->common.id) { - DbFixation* fix = *pp; - erts_smp_refc_dec(&tb->common.fix_count,0); - --(fix->counter); - ASSERT(fix->counter >= 0); - if (fix->counter > 0) { - break; - } - *pp = fix->next; + fix = fixing_procs_rbt_lookup(tb->common.fixing_procs, p); + + if (fix) { + erts_smp_refc_dec(&tb->common.fix_count,0); + --(fix->counter); + ASSERT(fix->counter >= 0); + if (fix->counter == 0) { + fixing_procs_rbt_delete(&tb->common.fixing_procs, fix); #ifdef ERTS_SMP erts_smp_mtx_unlock(&tb->common.fixlock); #endif - db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC); - db_erase_bag_exact2(meta_pid_to_fixed_tab, - p->common.id, make_small(tb->common.slot)); - db_meta_unlock(meta_pid_to_fixed_tab, LCK_WRITE_REC); + fixed_tabs_delete(p, fix); + + erts_refc_dec(&fix->tabs.btid->refc, 1); + erts_db_free(ERTS_ALC_T_DB_FIXATION, tb, (void *) fix, sizeof(DbFixation)); ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation)); @@ -4013,29 +3895,83 @@ unlocked: } } -/* Assume that tb is WRITE locked */ -static void free_fixations_locked(DbTable *tb) +struct free_fixations_ctx { - DbFixation *fix; - DbFixation *next_fix; - - fix = tb->common.fixations; - while (fix != NULL) { - erts_aint_t diff = -((erts_aint_t) fix->counter); - erts_smp_refc_add(&tb->common.fix_count,diff,0); - next_fix = fix->next; - db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC); - db_erase_bag_exact2(meta_pid_to_fixed_tab, - fix->pid, - make_small(tb->common.slot)); - db_meta_unlock(meta_pid_to_fixed_tab, LCK_WRITE_REC); - erts_db_free(ERTS_ALC_T_DB_FIXATION, - tb, (void *) fix, sizeof(DbFixation)); - ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation)); - - fix = next_fix; - } - tb->common.fixations = NULL; + Process* p; + DbTable* tb; +}; + +static void free_fixations_op(DbFixation* fix, void* vctx) +{ + struct free_fixations_ctx* ctx = (struct free_fixations_ctx*) vctx; + erts_aint_t diff; +#ifdef DEBUG + DbTable* dbg_tb = btid2tab(fix->tabs.btid); +#endif + + ASSERT(!dbg_tb || dbg_tb == ctx->tb); + ASSERT(fix->counter > 0); + ASSERT(ctx->tb->common.status & DB_DELETE); + + diff = -((erts_aint_t) fix->counter); + erts_smp_refc_add(&ctx->tb->common.fix_count, diff, 0); + +#ifdef ERTS_SMP + if (fix->procs.p != ctx->p) { /* Fixated by other process */ + fix->counter = 0; + + /* Fake memory stats for table */ + ASSERT(sizeof(DbFixation) == ERTS_ALC_DBG_BLK_SZ(fix)); + ERTS_DB_ALC_MEM_UPDATE_(ctx->tb, sizeof(DbFixation), 0); + + erts_schedule_ets_free_fixation(fix->procs.p->common.id, fix); + /* + * Either sys task is scheduled and erts_db_execute_free_fixation() + * will remove 'fix' or process will exit, drop sys task and + * proc_cleanup_fixed_table() will remove 'fix'. + */ + } + else +#endif + { + fixed_tabs_delete(fix->procs.p, fix); + + if (erts_refc_dectest(&fix->tabs.btid->refc, 0) == 0) { + erts_bin_free(fix->tabs.btid); + } + + erts_db_free(ERTS_ALC_T_DB_FIXATION, + ctx->tb, (void *) fix, sizeof(DbFixation)); + ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation)); + } +} + +#ifdef ERTS_SMP +int erts_db_execute_free_fixation(Process* p, DbFixation* fix) +{ + ASSERT(fix->counter == 0); + fixed_tabs_delete(p, fix); + + if (erts_refc_dectest(&fix->tabs.btid->refc, 0) == 0) { + erts_bin_free(fix->tabs.btid); + } + erts_free(ERTS_ALC_T_DB_FIXATION, fix); + ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation)); + return 1; +} +#endif + +static void free_fixations_locked(Process* p, DbTable *tb) +{ + struct free_fixations_ctx ctx; + + ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rwlocked(&tb->common.rwlock)); + + ctx.p = p; + ctx.tb = tb; + fixing_procs_rbt_foreach_destroy(&tb->common.fixing_procs, + free_fixations_op, &ctx); + tb->common.fixing_procs = NULL; } static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data) @@ -4107,9 +4043,7 @@ static BIF_RETTYPE ets_delete_trap(BIF_ALIST_1) ASSERT(*ptr == make_pos_bignum_header(1)); - db_lock(tb, LCK_WRITE); - trap = free_table_cont(p, tb, 0, 1); - db_unlock(tb, LCK_WRITE); + trap = free_table_cont(p, tb); if (trap) { BIF_TRAP1(&ets_delete_continue_exp, p, cont); @@ -4123,21 +4057,11 @@ static BIF_RETTYPE ets_delete_trap(BIF_ALIST_1) /* * free_table_cont() returns 0 when done and !0 when more work is needed. */ -static int free_table_cont(Process *p, - DbTable *tb, - int first, - int clean_meta_tab) +static int free_table_cont(Process *p, DbTable *tb) { Eterm result; erts_smp_rwmtx_t *mmtl; -#ifdef HARDDEBUG - if (!first) { - erts_fprintf(stderr,"ets: free_table_cont %T (continue)\r\n", - tb->common.id); - } -#endif - result = tb->common.meth->db_free_table_continue(tb); if (result == 0) { @@ -4157,27 +4081,36 @@ static int free_table_cont(Process *p, /* Completely done - we will not get called again. */ mmtl = get_meta_main_tab_lock(tb->common.slot); #ifdef ERTS_SMP - if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) { - erts_smp_rwmtx_rwunlock(&tb->common.rwlock); - erts_smp_rwmtx_rwlock(mmtl); - erts_smp_rwmtx_rwlock(&tb->common.rwlock); - } + erts_smp_rwmtx_rwlock(mmtl); #endif free_slot(tb->common.slot); erts_smp_rwmtx_rwunlock(mmtl); - if (clean_meta_tab) { - db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC); - db_erase_bag_exact2(meta_pid_to_tab,tb->common.owner, - make_small(tb->common.slot)); - db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); - } + delete_owned_table(p, tb); table_dec_refc(tb, 0); BUMP_REDS(p, 100); return 0; } } +struct fixing_procs_info_ctx +{ + Process* p; + Eterm list; +}; + +static void fixing_procs_info_op(DbFixation* fix, void* vctx) +{ + struct fixing_procs_info_ctx* ctx = (struct fixing_procs_info_ctx*) vctx; + Eterm* hp; + Eterm tpl; + + hp = HAllocX(ctx->p, 5, 100); + tpl = TUPLE2(hp, fix->procs.p->common.id, make_small(fix->counter)); + hp += 3; + ctx->list = CONS(hp, tpl, ctx->list); +} + static Eterm table_info(Process* p, DbTable* tb, Eterm What) { Eterm ret = THE_NON_VALUE; @@ -4249,9 +4182,9 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) if (IS_FIXED(tb)) { Uint need; Eterm *hp; - Eterm tpl, lst; - DbFixation *fix; + Eterm time; Sint64 mtime; + struct fixing_procs_info_ctx ctx; need = 3; if (use_monotonic) { @@ -4264,19 +4197,15 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) mtime = 0; need += 4; } - for (fix = tb->common.fixations; fix != NULL; fix = fix->next) { - need += 5; - } + ctx.p = p; + ctx.list = NIL; + fixing_procs_rbt_foreach(tb->common.fixing_procs, + fixing_procs_info_op, + &ctx); + hp = HAlloc(p, need); - lst = NIL; - for (fix = tb->common.fixations; fix != NULL; fix = fix->next) { - tpl = TUPLE2(hp,fix->pid,make_small(fix->counter)); - hp += 3; - lst = CONS(hp,tpl,lst); - hp += 2; - } if (use_monotonic) - tpl = (IS_SSMALL(mtime) + time = (IS_SSMALL(mtime) ? make_small(mtime) : erts_sint64_to_big(mtime, &hp)); else { @@ -4284,10 +4213,10 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) erts_make_timestamp_value(&ms, &s, &us, tb->common.time.monotonic, tb->common.time.offset); - tpl = TUPLE3(hp, make_small(ms), make_small(s), make_small(us)); + time = TUPLE3(hp, make_small(ms), make_small(s), make_small(us)); hp += 4; } - ret = TUPLE2(hp, tpl, lst); + ret = TUPLE2(hp, time, ctx.list); } else { ret = am_false; } @@ -4453,37 +4382,6 @@ erts_ets_colliding_names(Process* p, Eterm name, Uint cnt) return list; } -/* - * For testing only - * Retreive meta table size state - */ -Eterm erts_ets_get_meta_state(Process* p) -{ - Eterm* hp = HAlloc(p, 3); - return TUPLE2(hp, - erts_ets_hash_get_memstate(p, &meta_pid_to_tab->hash), - erts_ets_hash_get_memstate(p, &meta_pid_to_fixed_tab->hash)); -} -/* - * For testing only - * Restore a previously retrieved meta table size state. - * We do this to suppress failed memory checks - * caused by the hysteresis of meta tables grow/shrink limits. - */ -Eterm erts_ets_restore_meta_state(Process* p, Eterm meta_state) -{ - Eterm* tv; - Eterm* hp; - if (!is_tuple_arity(meta_state, 2)) - return am_badarg; - - tv = tuple_val(meta_state); - hp = HAlloc(p, 3); - return TUPLE2(hp, - erts_ets_hash_restore_memstate(&meta_pid_to_tab->hash, tv[1]), - erts_ets_hash_restore_memstate(&meta_pid_to_fixed_tab->hash, tv[2])); -} - #ifdef HARDDEBUG /* Here comes some debug functions */ void db_check_tables(void) diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index 1c53401590..cbf4b9e007 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -109,6 +109,7 @@ typedef enum { void init_db(ErtsDbSpinCount); int erts_db_process_exiting(Process *, ErtsProcLocks); +int erts_db_execute_free_fixation(Process*, DbFixation*); void db_info(fmtfn_t, void *, int); void erts_db_foreach_table(void (*)(DbTable *, void *), void *); void erts_db_foreach_offheap(DbTable *, @@ -125,9 +126,6 @@ extern Export ets_select_continue_exp; extern erts_smp_atomic_t erts_ets_misc_mem_size; Eterm erts_ets_colliding_names(Process*, Eterm name, Uint cnt); -Eterm erts_ets_get_meta_state(Process* p); -Eterm erts_ets_restore_meta_state(Process* p, Eterm target_state); - Uint erts_db_get_max_tabs(void); #endif /* ERL_DB_H__ */ diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index bb29d56aa7..3afc58bb79 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -903,70 +903,6 @@ done: RUNLOCK_HASH(lck); return DB_ERROR_NONE; } - -int db_get_element_array(DbTable *tbl, - Eterm key, - int ndex, - Eterm *ret, - int *num_ret) -{ - DbTableHash *tb = &tbl->hash; - HashValue hval; - int ix; - HashDbTerm* b1; - int num = 0; - int retval; - erts_smp_rwmtx_t* lck; - - ASSERT(!IS_FIXED(tbl)); /* no support for fixed tables here */ - - hval = MAKE_HASH(key); - lck = RLOCK_HASH(tb, hval); - ix = hash_to_ix(tb, hval); - b1 = BUCKET(tb, ix); - - while(b1 != 0) { - if (has_live_key(tb,b1,key,hval)) { - if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) { - HashDbTerm* b; - HashDbTerm* b2 = b1->next; - - while(b2 != NULL && has_live_key(tb,b2,key,hval)) { - if (ndex > arityval(b2->dbterm.tpl[0])) { - retval = DB_ERROR_BADITEM; - goto done; - } - b2 = b2->next; - } - - b = b1; - while(b != b2) { - if (num < *num_ret) { - ret[num++] = b->dbterm.tpl[ndex]; - } else { - retval = DB_ERROR_NONE; - goto done; - } - b = b->next; - } - *num_ret = num; - } - else { - ASSERT(*num_ret > 0); - ret[0] = b1->dbterm.tpl[ndex]; - *num_ret = 1; - } - retval = DB_ERROR_NONE; - goto done; - } - b1 = b1->next; - } - retval = DB_ERROR_BADKEY; -done: - RUNLOCK_HASH(lck); - return retval; -} - static int db_member_hash(DbTable *tbl, Eterm key, Eterm *ret) { @@ -1058,54 +994,6 @@ done: return retval; } -/* - * Very internal interface, removes elements of arity two from - * BAG. Used for the PID meta table - */ -int db_erase_bag_exact2(DbTable *tbl, Eterm key, Eterm value) -{ - DbTableHash *tb = &tbl->hash; - HashValue hval; - int ix; - HashDbTerm** bp; - HashDbTerm* b; - erts_smp_rwmtx_t* lck; - int found = 0; - - hval = MAKE_HASH(key); - lck = WLOCK_HASH(tb,hval); - ix = hash_to_ix(tb, hval); - bp = &BUCKET(tb, ix); - b = *bp; - - ASSERT(!IS_FIXED(tb)); - ASSERT((tb->common.status & DB_BAG)); - ASSERT(!tb->common.compress); - - while(b != 0) { - if (has_live_key(tb,b,key,hval)) { - found = 1; - if ((arityval(b->dbterm.tpl[0]) == 2) && - EQ(value, b->dbterm.tpl[2])) { - *bp = b->next; - free_term(tb, b); - erts_smp_atomic_dec_nob(&tb->common.nitems); - b = *bp; - break; - } - } else if (found) { - break; - } - bp = &b->next; - b = b->next; - } - WUNLOCK_HASH(lck); - if (found) { - try_shrink(tb); - } - return DB_ERROR_NONE; -} - /* ** NB, this is for the db_erase/2 bif. */ @@ -2207,7 +2095,7 @@ static int db_free_table_continue_hash(DbTable *tbl) DbTableHash *tb = &tbl->hash; int done; FixedDeletion* fixdel = (FixedDeletion*) erts_smp_atomic_read_acqb(&tb->fixdel); - ERTS_SMP_LC_ASSERT(IS_TAB_WLOCKED(tb)); + ERTS_SMP_LC_ASSERT(IS_TAB_WLOCKED(tb) || (tb->common.status & DB_DELETE)); done = 0; while (fixdel != NULL) { @@ -3007,44 +2895,6 @@ Eterm erts_ets_hash_sizeof_ext_segtab(void) { return make_small(((SIZEOF_EXT_SEGTAB(0)-1) / sizeof(UWord)) + 1); } -/* For testing only */ -Eterm erts_ets_hash_get_memstate(Process* p, DbTableHash* tb) -{ - Eterm seg_cnt; - while (!begin_resizing(tb)) - /*spinn*/; - - seg_cnt = make_small(SLOT_IX_TO_SEG_IX(tb->nslots)); - done_resizing(tb); - return seg_cnt; -} -/* For testing only */ -Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate) -{ - int seg_cnt, target; - - if (!is_small(memstate)) - return make_small(__LINE__); - - target = signed_val(memstate); - if (target < 1) - return make_small(__LINE__); - while (1) { - while (!begin_resizing(tb)) - /*spin*/; - seg_cnt = SLOT_IX_TO_SEG_IX(tb->nslots); - done_resizing(tb); - - if (target == seg_cnt) - return am_ok; - if (IS_FIXED(tb)) - return make_small(__LINE__); - if (target < seg_cnt) - shrink(tb, 0); - else - grow(tb, INT_MAX); - } -} #ifdef HARDDEBUG diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index 6d25c73549..cd02418370 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -88,14 +88,6 @@ int db_get_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret); int db_erase_hash(DbTable *tbl, Eterm key, Eterm *ret); -int db_get_element_array(DbTable *tbl, - Eterm key, - int ndex, - Eterm *ret, - int *num_ret); - -int db_erase_bag_exact2(DbTable *tbl, Eterm key, Eterm value); - /* not yet in method table */ int db_mark_all_deleted_hash(DbTable *tbl); @@ -110,7 +102,5 @@ typedef struct { void db_calc_stats_hash(DbTableHash* tb, DbHashStats*); Eterm erts_ets_hash_sizeof_ext_segtab(void); -Eterm erts_ets_hash_get_memstate(Process*, DbTableHash* tb); -Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate); #endif /* _DB_HASH_H */ diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 3c13035ec5..673ec80604 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -195,9 +195,20 @@ typedef struct db_table_method } DbTableMethod; typedef struct db_fixation { - Eterm pid; + /* Node in fixed_tabs list */ + struct { + struct db_fixation *next, *prev; + Binary* btid; + } tabs; + + /* Node in fixing_procs tree */ + struct { + struct db_fixation *left, *right, *parent; + int is_red; + Process* p; + } procs; + Uint counter; - struct db_fixation *next; } DbFixation; typedef struct { @@ -217,9 +228,10 @@ typedef struct db_table_common { erts_smp_refc_t refc; /* reference count of table struct */ erts_smp_refc_t fix_count;/* fixation counter */ DbTableList all; + DbTableList owned; #ifdef ERTS_SMP erts_smp_rwmtx_t rwlock; /* rw lock on table */ - erts_smp_mtx_t fixlock; /* Protects fixations,megasec,sec,microsec */ + erts_smp_mtx_t fixlock; /* Protects fixing_procs and time */ int is_thread_safe; /* No fine locking inside table needed */ Uint32 type; /* table type, *read only* after creation */ #endif @@ -237,7 +249,7 @@ typedef struct db_table_common { ErtsMonotonicTime monotonic; ErtsMonotonicTime offset; } time; - DbFixation* fixations; /* List of processes who have done safe_fixtable, + DbFixation* fixing_procs; /* Tree of processes who have done safe_fixtable, "local" fixations not included. */ /* All 32-bit fields */ Uint32 status; /* bit masks defined below */ diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 6ff9aea5ab..7f7c06da8e 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -100,12 +100,12 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "dist_entry_links", "address" }, { "code_write_permission", NULL }, { "purge_state", NULL }, - { "proc_status", "pid" }, - { "proc_trace", "pid" }, - { "ports_snapshot", NULL }, { "meta_name_tab", "address" }, { "meta_main_tab_slot", "address" }, { "db_tab", "address" }, + { "proc_status", "pid" }, + { "proc_trace", "pid" }, + { "ports_snapshot", NULL }, { "db_tab_fix", "address" }, { "meta_main_tab_main", NULL }, { "db_hash_slot", "address" }, diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index b816ba59fb..88fae30845 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -449,7 +449,8 @@ typedef enum { ERTS_PSTT_CPC, /* Check Process Code */ ERTS_PSTT_CLA, /* Copy Literal Area */ ERTS_PSTT_COHMQ, /* Change off heap message queue */ - ERTS_PSTT_FTMQ /* Flush trace msg queue */ + ERTS_PSTT_FTMQ, /* Flush trace msg queue */ + ERTS_PSTT_ETS_FREE_FIXATION } ErtsProcSysTaskType; #define ERTS_MAX_PROC_SYS_TASK_ARGS 2 @@ -729,6 +730,16 @@ erts_pre_init_process(void) = ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS; erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].set_locks = ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS; + + erts_psd_required_locks[ERTS_PSD_ETS_OWNED_TABLES].get_locks + = ERTS_PSD_ETS_OWNED_TABLES_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_ETS_OWNED_TABLES].set_locks + = ERTS_PSD_ETS_OWNED_TABLES_SET_LOCKS; + + erts_psd_required_locks[ERTS_PSD_ETS_FIXED_TABLES].get_locks + = ERTS_PSD_ETS_FIXED_TABLES_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_ETS_FIXED_TABLES].set_locks + = ERTS_PSD_ETS_FIXED_TABLES_SET_LOCKS; #endif } @@ -11197,6 +11208,12 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) reds -= erts_flush_trace_messages(c_p, ERTS_PROC_LOCK_MAIN); st_res = am_true; break; +#endif +#ifdef ERTS_SMP + case ERTS_PSTT_ETS_FREE_FIXATION: + reds -= erts_db_execute_free_fixation(c_p, (DbFixation*)st->arg[0]); + st_res = am_true; + break; #endif default: ERTS_INTERNAL_ERROR("Invalid process sys task type"); @@ -11249,7 +11266,8 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) case ERTS_PSTT_GC_MAJOR: case ERTS_PSTT_GC_MINOR: case ERTS_PSTT_CPC: - case ERTS_PSTT_COHMQ: + case ERTS_PSTT_COHMQ: + case ERTS_PSTT_ETS_FREE_FIXATION: st_res = am_false; break; case ERTS_PSTT_CLA: @@ -11603,13 +11621,12 @@ erts_internal_request_system_task_4(BIF_ALIST_4) } static void -erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type) +erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type, void* arg) { Process *rp = erts_proc_lookup(pid); if (rp) { ErtsProcSysTask *st; erts_aint32_t state, fail_state; - int i; st = erts_alloc(ERTS_ALC_T_PROC_SYS_TSK, ERTS_PROC_SYS_TASK_SIZE(0)); @@ -11618,8 +11635,7 @@ erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type) st->reply_tag = NIL; st->req_id = NIL; st->req_id_sz = 0; - for (i = 0; i < ERTS_MAX_PROC_SYS_TASK_ARGS; i++) - st->arg[i] = NIL; + st->arg[0] = (Eterm)arg; ERTS_INIT_OFF_HEAP(&st->off_heap); state = erts_smp_atomic32_read_nob(&rp->state); @@ -11635,7 +11651,13 @@ erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type) void erts_schedule_complete_off_heap_message_queue_change(Eterm pid) { - erts_schedule_generic_sys_task(pid, ERTS_PSTT_COHMQ); + erts_schedule_generic_sys_task(pid, ERTS_PSTT_COHMQ, NULL); +} + +void +erts_schedule_ets_free_fixation(Eterm pid, DbFixation* fix) +{ + erts_schedule_generic_sys_task(pid, ERTS_PSTT_ETS_FREE_FIXATION, fix); } #ifdef ERTS_DIRTY_SCHEDULERS @@ -11683,7 +11705,7 @@ erts_schedule_flush_trace_messages(Process *proc, int force_on_proc) dhndl = erts_thr_progress_unmanaged_delay(); #endif - erts_schedule_generic_sys_task(pid, ERTS_PSTT_FTMQ); + erts_schedule_generic_sys_task(pid, ERTS_PSTT_FTMQ, NULL); #ifdef ERTS_SMP erts_thr_progress_unmanaged_continue(dhndl); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 58f1e15c23..f20dc68799 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -836,14 +836,16 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) #define ERTS_PSD_CALL_TIME_BP 3 #define ERTS_PSD_DELAYED_GC_TASK_QS 4 #define ERTS_PSD_NIF_TRAP_EXPORT 5 -#define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF 6 +#define ERTS_PSD_ETS_OWNED_TABLES 6 +#define ERTS_PSD_ETS_FIXED_TABLES 7 +#define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF 8 -#define ERTS_PSD_SIZE 7 +#define ERTS_PSD_SIZE 9 #if !defined(HIPE) # undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF # undef ERTS_PSD_SIZE -# define ERTS_PSD_SIZE 6 +# define ERTS_PSD_SIZE 8 #endif typedef struct { @@ -871,6 +873,12 @@ typedef struct { #define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_ETS_OWNED_TABLES_GET_LOCKS ERTS_PROC_LOCK_STATUS +#define ERTS_PSD_ETS_OWNED_TABLES_SET_LOCKS ERTS_PROC_LOCK_STATUS + +#define ERTS_PSD_ETS_FIXED_TABLES_GET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_ETS_FIXED_TABLES_SET_LOCKS ERTS_PROC_LOCK_MAIN + typedef struct { ErtsProcLocks get_locks; ErtsProcLocks set_locks; @@ -1805,6 +1813,8 @@ void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *), ErtsThrPrgrLaterOp *, UWord); void erts_schedule_complete_off_heap_message_queue_change(Eterm pid); +struct db_fixation; +void erts_schedule_ets_free_fixation(Eterm pid, struct db_fixation*); void erts_schedule_flush_trace_messages(Process *proc, int force_on_proc); int erts_flush_trace_messages(Process *c_p, ErtsProcLocks locks); diff --git a/erts/emulator/beam/erl_rbtree.h b/erts/emulator/beam/erl_rbtree.h index ebde5caca4..6a42853957 100644 --- a/erts/emulator/beam/erl_rbtree.h +++ b/erts/emulator/beam/erl_rbtree.h @@ -486,12 +486,12 @@ typedef struct { #if defined(ERTS_RBT_HARD_DEBUG) \ && (defined(ERTS_RBT_WANT_DELETE) \ || defined(ERTS_RBT_NEED_INSERT__)) -static void ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root); +static void ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root, ERTS_RBT_T *node); # define ERTS_RBT_NEED_HDBG_CHECK_TREE__ -# define ERTS_RBT_HDBG_CHECK_TREE__(R) \ - ERTS_RBT_FUNC__(hdbg_check_tree)((R)) +# define ERTS_RBT_HDBG_CHECK_TREE__(R,N) \ + ERTS_RBT_FUNC__(hdbg_check_tree)((R),(N)) #else -# define ERTS_RBT_HDBG_CHECK_TREE__(R) ((void) 1) +# define ERTS_RBT_HDBG_CHECK_TREE__(R,N) ((void) 1) #endif #ifdef ERTS_RBT_NEED_ROTATE__ @@ -644,7 +644,7 @@ ERTS_RBT_FUNC__(delete)(ERTS_RBT_T **root, ERTS_RBT_T *n) ERTS_RBT_T null_x; /* null_x is used to get the fixup started when we splice out a node without children. */ - ERTS_RBT_HDBG_CHECK_TREE__(*root); + ERTS_RBT_HDBG_CHECK_TREE__(*root, n); ERTS_RBT_INIT_EMPTY_TNODE(&null_x); @@ -862,7 +862,7 @@ ERTS_RBT_FUNC__(delete)(ERTS_RBT_T **root, ERTS_RBT_T *n) } } - ERTS_RBT_HDBG_CHECK_TREE__(*root); + ERTS_RBT_HDBG_CHECK_TREE__(*root, NULL); } @@ -992,7 +992,7 @@ ERTS_RBT_FUNC__(insert_aux__)(ERTS_RBT_T **root, ERTS_RBT_T *n, int lookup) { ERTS_RBT_KEY_T kn = ERTS_RBT_GET_KEY(n); - ERTS_RBT_HDBG_CHECK_TREE__(*root); + ERTS_RBT_HDBG_CHECK_TREE__(*root, NULL); ERTS_RBT_INIT_EMPTY_TNODE(n); @@ -1014,7 +1014,7 @@ ERTS_RBT_FUNC__(insert_aux__)(ERTS_RBT_T **root, ERTS_RBT_T *n, int lookup) if (lookup && ERTS_RBT_IS_EQ(kn, kx)) { - ERTS_RBT_HDBG_CHECK_TREE__(*root); + ERTS_RBT_HDBG_CHECK_TREE__(*root, NULL); return x; } @@ -1048,7 +1048,7 @@ ERTS_RBT_FUNC__(insert_aux__)(ERTS_RBT_T **root, ERTS_RBT_T *n, int lookup) ERTS_RBT_FUNC__(insert_fixup__)(root, n); } - ERTS_RBT_HDBG_CHECK_TREE__(*root); + ERTS_RBT_HDBG_CHECK_TREE__(*root, n); return NULL; } @@ -1589,15 +1589,17 @@ ERTS_RBT_FUNC__(debug_print)(FILE *filep, ERTS_RBT_T *x, int indent, #ifdef ERTS_RBT_NEED_HDBG_CHECK_TREE__ static void -ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root) +ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root, ERTS_RBT_T *n) { int black_depth = -1, no_black = 0; ERTS_RBT_T *c, *p, *x = root; ERTS_RBT_KEY_T kx; ERTS_RBT_KEY_T kc; - if (!x) + if (!x) { + ERTS_RBT_ASSERT(!n); return; + } ERTS_RBT_ASSERT(!ERTS_RBT_GET_PARENT(x)); @@ -1607,6 +1609,9 @@ ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root) while (1) { + if (x == n) + n = NULL; + if (ERTS_RBT_IS_BLACK(x)) no_black++; else { @@ -1678,6 +1683,7 @@ ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root) if (!p) { ERTS_RBT_ASSERT(root == x); ERTS_RBT_ASSERT(no_black == 0); + ERTS_RBT_ASSERT(!n); return; /* Done */ } -- cgit v1.2.3 From b9388df82dd1149fd603371c6ee12f5d7dbcec30 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 23 Feb 2017 18:24:15 +0100 Subject: erts: Print table id as ref in crashdump and break menu --- erts/emulator/beam/erl_db.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 24de21cc24..7faec9a057 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -4261,7 +4261,19 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) static void print_table(fmtfn_t to, void *to_arg, int show, DbTable* tb) { - erts_print(to, to_arg, "Table: %T\n", tb->common.id); + Eterm tid; + Eterm heap[ERTS_MAGIC_REF_THING_SIZE]; + + if (is_atom(tb->common.id)) { + tid = tb->common.id; + } else { + ErlOffHeap oh; + ERTS_INIT_OFF_HEAP(&oh); + write_magic_ref_thing(heap, &oh, (ErtsMagicBinary *) tb->common.btid); + tid = make_internal_ref(heap); + } + + erts_print(to, to_arg, "Table: %T\n", tid); erts_print(to, to_arg, "Name: %T\n", tb->common.the_name); tb->common.meth->db_print(to, to_arg, show, tb); -- cgit v1.2.3 From e214bb4a0f86a6f956dd473f66b1a6c2f7085591 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 24 Feb 2017 20:08:34 +0100 Subject: erts: Pass tid argument down to trapping functions to get rid of meta table lookup by integer (tb->common.id) --- erts/emulator/beam/erl_db.c | 13 ++++++------- erts/emulator/beam/erl_db_hash.c | 32 +++++++++++++++++--------------- erts/emulator/beam/erl_db_tree.c | 26 +++++++++++++------------- erts/emulator/beam/erl_db_util.h | 16 ++++++++++------ 4 files changed, 46 insertions(+), 41 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 7faec9a057..3f92a69df1 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -2391,7 +2391,7 @@ BIF_RETTYPE ets_select_delete_2(BIF_ALIST_2) if (safety == ITER_UNSAFE) { local_fix_table(tb); } - cret = tb->common.meth->db_select_delete(BIF_P, tb, BIF_ARG_2, &ret); + cret = tb->common.meth->db_select_delete(BIF_P, tb, BIF_ARG_1, BIF_ARG_2, &ret); if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) { fix_table_locked(BIF_P,tb); @@ -2770,7 +2770,7 @@ ets_select3(Process* p, Eterm arg1, Eterm arg2, Eterm arg3) if (safety == ITER_UNSAFE) { local_fix_table(tb); } - cret = tb->common.meth->db_select_chunk(p, tb, + cret = tb->common.meth->db_select_chunk(p, tb, arg1, arg2, chunk_size, 0 /* not reversed */, &ret); @@ -2939,8 +2939,7 @@ ets_select2(Process* p, Eterm arg1, Eterm arg2) local_fix_table(tb); } - cret = tb->common.meth->db_select(p, tb, arg2, - 0, &ret); + cret = tb->common.meth->db_select(p, tb, arg1, arg2, 0, &ret); if (DID_TRAP(p,ret) && safety != ITER_SAFE) { fix_table_locked(p, tb); @@ -3031,7 +3030,7 @@ BIF_RETTYPE ets_select_count_2(BIF_ALIST_2) if (safety == ITER_UNSAFE) { local_fix_table(tb); } - cret = tb->common.meth->db_select_count(BIF_P,tb,BIF_ARG_2, &ret); + cret = tb->common.meth->db_select_count(BIF_P,tb, BIF_ARG_1, BIF_ARG_2, &ret); if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) { fix_table_locked(BIF_P, tb); @@ -3085,7 +3084,7 @@ BIF_RETTYPE ets_select_reverse_3(BIF_ALIST_3) if (safety == ITER_UNSAFE) { local_fix_table(tb); } - cret = tb->common.meth->db_select_chunk(BIF_P,tb, + cret = tb->common.meth->db_select_chunk(BIF_P,tb, BIF_ARG_1, BIF_ARG_2, chunk_size, 1 /* reversed */, &ret); if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) { @@ -3135,7 +3134,7 @@ BIF_RETTYPE ets_select_reverse_2(BIF_ALIST_2) if (safety == ITER_UNSAFE) { local_fix_table(tb); } - cret = tb->common.meth->db_select(BIF_P,tb,BIF_ARG_2, + cret = tb->common.meth->db_select(BIF_P,tb, BIF_ARG_1, BIF_ARG_2, 1 /*reversed*/, &ret); if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) { diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 3afc58bb79..adb2376c34 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -393,20 +393,20 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object,Eterm *ret); static int db_slot_hash(Process *p, DbTable *tbl, Eterm slot_term, Eterm *ret); -static int db_select_chunk_hash(Process *p, DbTable *tbl, +static int db_select_chunk_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Sint chunk_size, int reverse, Eterm *ret); -static int db_select_hash(Process *p, DbTable *tbl, +static int db_select_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, int reverse, Eterm *ret); -static int db_select_count_hash(Process *p, DbTable *tbl, +static int db_select_count_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret); -static int db_select_delete_hash(Process *p, DbTable *tbl, +static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret); static int db_select_continue_hash(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); -static int db_select_count_continue_hash(Process *p, DbTable *tbl, +static int db_select_count_continue_hash(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); static int db_select_delete_continue_hash(Process *p, DbTable *tbl, @@ -1289,14 +1289,14 @@ trap: } -static int db_select_hash(Process *p, DbTable *tbl, +static int db_select_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, int reverse, Eterm *ret) { - return db_select_chunk_hash(p, tbl, pattern, 0, reverse, ret); + return db_select_chunk_hash(p, tbl, tid, pattern, 0, reverse, ret); } -static int db_select_chunk_hash(Process *p, DbTable *tbl, +static int db_select_chunk_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Sint chunk_size, int reverse, /* not used */ Eterm *ret) @@ -1444,7 +1444,7 @@ done: mpb = erts_db_make_match_prog_ref(p,(mpi.mp),&hp); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; - continuation = TUPLE6(hp, tb->common.id,make_small(slot_ix), + continuation = TUPLE6(hp, tid, make_small(slot_ix), make_small(chunk_size), mpb, rest, make_small(rest_size)); @@ -1468,7 +1468,7 @@ trap: (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; hp = HAlloc(p,7+ERTS_MAGIC_REF_THING_SIZE); mpb = erts_db_make_match_prog_ref(p,(mpi.mp),&hp); - continuation = TUPLE6(hp, tb->common.id, make_small(slot_ix), + continuation = TUPLE6(hp, tid, make_small(slot_ix), make_small(chunk_size), mpb, match_list, make_small(got)); @@ -1482,7 +1482,8 @@ trap: } static int db_select_count_hash(Process *p, - DbTable *tbl, + DbTable *tbl, + Eterm tid, Eterm pattern, Eterm *ret) { @@ -1588,7 +1589,7 @@ trap: hp += BIG_UINT_HEAP_SIZE; } mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp); - continuation = TUPLE4(hp, tb->common.id, make_small(slot_ix), + continuation = TUPLE4(hp, tid, make_small(slot_ix), mpb, egot); mpi.mp = NULL; /*otherwise the return macro will destroy it */ @@ -1601,6 +1602,7 @@ trap: static int db_select_delete_hash(Process *p, DbTable *tbl, + Eterm tid, Eterm pattern, Eterm *ret) { @@ -1733,7 +1735,7 @@ trap: hp += BIG_UINT_HEAP_SIZE; } mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp); - continuation = TUPLE4(hp, tb->common.id, make_small(slot_ix), + continuation = TUPLE4(hp, tid, make_small(slot_ix), mpb, egot); mpi.mp = NULL; /*otherwise the return macro will destroy it */ @@ -1847,7 +1849,7 @@ trap: egot = uint_to_big(got, hp); hp += BIG_UINT_HEAP_SIZE; } - continuation = TUPLE4(hp, tb->common.id, make_small(slot_ix), + continuation = TUPLE4(hp, tptr[1], make_small(slot_ix), tptr[3], egot); RET_TO_BIF(bif_trap1(&ets_select_delete_continue_exp, p, @@ -1938,7 +1940,7 @@ trap: egot = uint_to_big(got, hp); hp += BIG_UINT_HEAP_SIZE; } - continuation = TUPLE4(hp, tb->common.id, make_small(slot_ix), + continuation = TUPLE4(hp, tptr[1], make_small(slot_ix), tptr[3], egot); RET_TO_BIF(bif_trap1(&ets_select_count_continue_exp, p, diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index c4ecd2ba37..3c2dd65e61 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -369,18 +369,18 @@ static int db_erase_tree(DbTable *tbl, Eterm key, Eterm *ret); static int db_erase_object_tree(DbTable *tbl, Eterm object,Eterm *ret); static int db_slot_tree(Process *p, DbTable *tbl, Eterm slot_term, Eterm *ret); -static int db_select_tree(Process *p, DbTable *tbl, +static int db_select_tree(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, int reversed, Eterm *ret); -static int db_select_count_tree(Process *p, DbTable *tbl, +static int db_select_count_tree(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret); -static int db_select_chunk_tree(Process *p, DbTable *tbl, +static int db_select_chunk_tree(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Sint chunk_size, int reversed, Eterm *ret); static int db_select_continue_tree(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); static int db_select_count_continue_tree(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); -static int db_select_delete_tree(Process *p, DbTable *tbl, +static int db_select_delete_tree(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret); static int db_select_delete_continue_tree(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); @@ -1058,7 +1058,7 @@ static int db_select_continue_tree(Process *p, } -static int db_select_tree(Process *p, DbTable *tbl, +static int db_select_tree(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, int reverse, Eterm *ret) { /* Strategy: Traverse backwards to build resulting list from tail to head */ @@ -1151,7 +1151,7 @@ static int db_select_tree(Process *p, DbTable *tbl, continuation = TUPLE8 (hp, - tb->common.id, + tid, key, sc.end_condition, /* From the match program, needn't be copied */ make_small(0), /* Chunk size of zero means not chunked to the @@ -1263,7 +1263,7 @@ static int db_select_count_continue_tree(Process *p, } -static int db_select_count_tree(Process *p, DbTable *tbl, +static int db_select_count_tree(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret) { DbTableTree *tb = &tbl->tree; @@ -1349,7 +1349,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, continuation = TUPLE5 (hp, - tb->common.id, + tid, key, sc.end_condition, /* From the match program, needn't be copied */ mpb, @@ -1363,7 +1363,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, } -static int db_select_chunk_tree(Process *p, DbTable *tbl, +static int db_select_chunk_tree(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Sint chunk_size, int reverse, Eterm *ret) @@ -1474,7 +1474,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, continuation = TUPLE8 (hp, - tb->common.id, + tid, key, sc.end_condition, /* From the match program, needn't be copied */ @@ -1499,7 +1499,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp); continuation = TUPLE8 (hp, - tb->common.id, + tid, key, sc.end_condition, /* From the match program, needn't be copied */ make_small(chunk_size), @@ -1605,7 +1605,7 @@ static int db_select_delete_continue_tree(Process *p, #undef RET_TO_BIF } -static int db_select_delete_tree(Process *p, DbTable *tbl, +static int db_select_delete_tree(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret) { DbTableTree *tb = &tbl->tree; @@ -1691,7 +1691,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, continuation = TUPLE5 (hp, - tb->common.id, + tid, key, sc.end_condition, /* From the match program, needn't be copied */ mpb, diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 673ec80604..958b504a60 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -135,30 +135,34 @@ typedef struct db_table_method Eterm slot, Eterm* ret); int (*db_select_chunk)(Process* p, - DbTable* tb, /* [in out] */ + DbTable* tb, /* [in out] */ + Eterm tid, Eterm pattern, Sint chunk_size, int reverse, Eterm* ret); int (*db_select)(Process* p, - DbTable* tb, /* [in out] */ + DbTable* tb, /* [in out] */ + Eterm tid, Eterm pattern, int reverse, Eterm* ret); int (*db_select_delete)(Process* p, - DbTable* tb, /* [in out] */ + DbTable* tb, /* [in out] */ + Eterm tid, Eterm pattern, Eterm* ret); int (*db_select_continue)(Process* p, - DbTable* tb, /* [in out] */ + DbTable* tb, /* [in out] */ Eterm continuation, Eterm* ret); int (*db_select_delete_continue)(Process* p, - DbTable* tb, /* [in out] */ + DbTable* tb, /* [in out] */ Eterm continuation, Eterm* ret); int (*db_select_count)(Process* p, - DbTable* tb, /* [in out] */ + DbTable* tb, /* [in out] */ + Eterm tid, Eterm pattern, Eterm* ret); int (*db_select_count_continue)(Process* p, -- cgit v1.2.3 From 14982307b07e899e888e5bfb93b3a138d4948459 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 24 Feb 2017 20:43:54 +0100 Subject: erts: Remove meta_main_tab \o/ O / \ Also removed the body for CHECK_TABLES enabled by HARDDEBUG. Removed quite useless check for hash, but kept dead check code for tree. --- erts/emulator/beam/erl_db.c | 201 ++---------------------------------- erts/emulator/beam/erl_db_hash.c | 28 ----- erts/emulator/beam/erl_db_tree.c | 9 +- erts/emulator/beam/erl_db_util.h | 11 -- erts/emulator/beam/erl_lock_check.c | 2 - 5 files changed, 11 insertions(+), 240 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 3f92a69df1..11936ed9dc 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -179,45 +179,12 @@ static int fixed_tabs_find(DbFixation* first, DbFixation* fix) #include "erl_rbtree.h" - - -/* -** The main meta table, containing all ets tables. -*/ -#ifdef ERTS_SMP - -#define ERTS_META_MAIN_TAB_LOCK_TAB_BITS 8 -#define ERTS_META_MAIN_TAB_LOCK_TAB_SIZE (1 << ERTS_META_MAIN_TAB_LOCK_TAB_BITS) -#define ERTS_META_MAIN_TAB_LOCK_TAB_MASK (ERTS_META_MAIN_TAB_LOCK_TAB_SIZE - 1) - -typedef union { - erts_smp_rwmtx_t rwmtx; - byte cache_line_align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE( - sizeof(erts_smp_rwmtx_t))]; -} erts_meta_main_tab_lock_t; - -static erts_meta_main_tab_lock_t *meta_main_tab_locks; - +#ifdef HARDDEBUG +# error Do something useful with CHECK_TABLES maybe +#else +# define CHECK_TABLES() #endif -static struct { - union { - DbTable *tb; /* Only directly readable if slot is ALIVE */ - UWord next_free; /* (index<<2)|1 if slot is FREE */ - }u; -} *meta_main_tab; -/* A slot in meta_main_tab can have three states: - * FREE : Free to use for new table. Part of linked free-list. - * ALIVE: Contains a table - * DEAD : Contains a table that is being removed. - */ -#define IS_SLOT_FREE(i) (meta_main_tab[(i)].u.next_free & 1) -#define IS_SLOT_DEAD(i) (meta_main_tab[(i)].u.next_free & 2) -#define IS_SLOT_ALIVE(i) (!(meta_main_tab[(i)].u.next_free & (1|2))) -#define GET_NEXT_FREE_SLOT(i) (meta_main_tab[(i)].u.next_free >> 2) -#define SET_NEXT_FREE_SLOT(i,next) (meta_main_tab[(i)].u.next_free = ((next)<<2)|1) -#define MARK_SLOT_DEAD(i) (meta_main_tab[(i)].u.next_free |= 2) -#define GET_ANY_SLOT_TAB(i) ((DbTable*)(meta_main_tab[(i)].u.next_free & ~(1|2))) /* dead or alive */ static void send_ets_transfer_message(Process *c_p, Process *proc, @@ -315,23 +282,6 @@ make_tid(Process *c_p, DbTable *tb) return erts_mk_magic_ref(&hp, &c_p->off_heap, tb->common.btid); } -static ERTS_INLINE erts_smp_rwmtx_t * -get_meta_main_tab_lock(unsigned slot) -{ -#ifdef ERTS_SMP - return &meta_main_tab_locks[slot & ERTS_META_MAIN_TAB_LOCK_TAB_MASK].rwmtx; -#else - return NULL; -#endif -} - -static erts_smp_spinlock_t meta_main_tab_main_lock; -static Uint meta_main_tab_first_free; /* Index of first free slot */ -static int meta_main_tab_cnt; /* Number of active tables */ -static int meta_main_tab_top; /* Highest ever used slot + 1 */ -static Uint meta_main_tab_slot_mask; /* The slot index part of an unnamed table id */ -static Uint meta_main_tab_seq_incr; -static Uint meta_main_tab_seq_cnt = 0; /* To give unique(-ish) table identifiers */ /* ** The meta hash table of all NAMED ets tables @@ -713,22 +663,6 @@ DbTable* db_get_table_aux(Process *p, tb = tid2tab(id); if (tb) id = tb->common.id; - else if (is_small(id)) { - Uint slot = unsigned_val(id) & meta_main_tab_slot_mask; - if (!meta_already_locked) { - mtl = get_meta_main_tab_lock(slot); - erts_smp_rwmtx_rlock(mtl); - } -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) - else { - erts_smp_rwmtx_t *test_mtl = get_meta_main_tab_lock(slot); - ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(test_mtl) - || erts_lc_rwmtx_is_rwlocked(test_mtl)); - } -#endif - if (slot < db_max_tabs && IS_SLOT_ALIVE(slot)) - tb = meta_main_tab[slot].u.tb; - } else if (is_atom(id)) { struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&mtl); if (!meta_already_locked) @@ -779,18 +713,6 @@ DbTable* db_get_table(Process *p, return db_get_table_aux(p, id, what, kind, 0); } -/* Requires meta_main_tab_locks[slot] locked. -*/ -static ERTS_INLINE void free_slot(int slot) -{ - ASSERT(!IS_SLOT_FREE(slot)); - erts_smp_spin_lock(&meta_main_tab_main_lock); - SET_NEXT_FREE_SLOT(slot,meta_main_tab_first_free); - meta_main_tab_first_free = slot; - meta_main_tab_cnt--; - erts_smp_spin_unlock(&meta_main_tab_main_lock); -} - static int insert_named_tab(Eterm name_atom, DbTable* tb, int have_lock) { int ret = 0; @@ -1602,7 +1524,6 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) } } else { - Uint slot; tb = tid2tab(BIF_ARG_1); if (!tb) BIF_ERROR(BIF_P, BADARG); @@ -1611,8 +1532,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) old_name = tb->common.id; goto named_tab; } - slot = unsigned_val(tb->common.id) & meta_main_tab_slot_mask; - lck2 = get_meta_main_tab_lock(slot); + lck2 = NULL; } } @@ -1665,7 +1585,6 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) BIF_RETTYPE ets_new_2(BIF_ALIST_2) { DbTable* tb = NULL; - int slot; Eterm list; Eterm val; Eterm ret; @@ -1681,7 +1600,6 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) int cret; #endif DbTableMethod* meth; - erts_smp_rwmtx_t *mmtl; if (is_not_atom(BIF_ARG_1)) { BIF_ERROR(BIF_P, BADARG); @@ -1847,57 +1765,21 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) make_btid(tb); - erts_smp_spin_lock(&meta_main_tab_main_lock); - - if (meta_main_tab_cnt >= db_max_tabs) { - erts_smp_spin_unlock(&meta_main_tab_main_lock); - erts_send_error_to_logger_str(BIF_P->group_leader, - "** Too many db tables **\n"); - free_heir_data(tb); - tb->common.meth->db_free_table(tb); - free_dbtable((void *) tb); - BIF_ERROR(BIF_P, SYSTEM_LIMIT); - } - - slot = meta_main_tab_first_free; - ASSERT(slot>=0 && slot= meta_main_tab_top) { - ASSERT(slot == meta_main_tab_top); - meta_main_tab_top = slot + 1; - } - if (is_named) { ret = BIF_ARG_1; + tb->common.id = ret; } else { - ret = make_small(slot | meta_main_tab_seq_cnt); - meta_main_tab_seq_cnt += meta_main_tab_seq_incr; - ASSERT((unsigned_val(ret) & meta_main_tab_slot_mask) == slot); + ret = make_tid(BIF_P, tb); + tb->common.id = THE_NON_VALUE; /* or what? */ } - erts_smp_spin_unlock(&meta_main_tab_main_lock); - - tb->common.id = ret; - tb->common.slot = slot; /* store slot for erase */ if (!is_named) ret = make_tid(BIF_P, tb); save_sched_table(BIF_P, tb); - mmtl = get_meta_main_tab_lock(slot); - erts_smp_rwmtx_rwlock(mmtl); - meta_main_tab[slot].u.tb = tb; - ASSERT(IS_SLOT_ALIVE(slot)); - erts_smp_rwmtx_rwunlock(mmtl); - if (is_named && !insert_named_tab(BIF_ARG_1, tb, 0)) { - mmtl = get_meta_main_tab_lock(slot); - erts_smp_rwmtx_rwlock(mmtl); - free_slot(slot); - erts_smp_rwmtx_rwunlock(mmtl); - tid_clear(BIF_P, tb); db_lock(tb,LCK_WRITE); @@ -2024,7 +1906,6 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) { int trap; DbTable* tb; - erts_smp_rwmtx_t *mmtl; #ifdef HARDDEBUG erts_fprintf(stderr, @@ -2070,21 +1951,6 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) tid_clear(BIF_P, tb); - mmtl = get_meta_main_tab_lock(tb->common.slot); -#ifdef ERTS_SMP - if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) { - /* - * We keep our increased refc over this op in order to - * prevent the table from disapearing. - */ - db_unlock(tb, LCK_WRITE); - erts_smp_rwmtx_rwlock(mmtl); - db_lock(tb, LCK_WRITE); - } -#endif - /* We must keep the slot, to be found by db_proc_dead() if process dies */ - MARK_SLOT_DEAD(tb->common.slot); - erts_smp_rwmtx_rwunlock(mmtl); if (is_atom(tb->common.id)) remove_named_tab(tb, 0); @@ -3425,16 +3291,6 @@ void init_db(ErtsDbSpinCount db_spin_count) if (erts_ets_rwmtx_spin_count >= 0) rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count; - meta_main_tab_locks = - erts_alloc_permanent_cache_aligned(ERTS_ALC_T_DB_TABLES, - sizeof(erts_meta_main_tab_lock_t) - * ERTS_META_MAIN_TAB_LOCK_TAB_SIZE); - - for (i = 0; i < ERTS_META_MAIN_TAB_LOCK_TAB_SIZE; i++) { - erts_smp_rwmtx_init_opt_x(&meta_main_tab_locks[i].rwmtx, &rwmtx_opt, - "meta_main_tab_slot", make_small(i)); - } - erts_smp_spinlock_init(&meta_main_tab_main_lock, "meta_main_tab_main"); for (i=0; icommon.meth->db_free_table_continue(tb); @@ -4078,13 +3919,6 @@ static int free_table_cont(Process *p, DbTable *tb) tb->common.id); #endif /* Completely done - we will not get called again. */ - mmtl = get_meta_main_tab_lock(tb->common.slot); -#ifdef ERTS_SMP - erts_smp_rwmtx_rwlock(mmtl); -#endif - free_slot(tb->common.slot); - erts_smp_rwmtx_rwunlock(mmtl); - delete_owned_table(p, tb); table_dec_refc(tb, 0); BUMP_REDS(p, 100); @@ -4393,22 +4227,3 @@ erts_ets_colliding_names(Process* p, Eterm name, Uint cnt) return list; } -#ifdef HARDDEBUG /* Here comes some debug functions */ - -void db_check_tables(void) -{ -#ifdef ERTS_SMP - return; -#else - int i; - - for (i = 0; i < db_max_tabs; i++) { - if (IS_SLOT_ALIVE(i)) { - DbTable* tb = meta_main_tab[i].t; - tb->common.meth->db_check_table(tb); - } - } -#endif -} - -#endif /* HARDDEBUG */ diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index adb2376c34..781886a822 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -529,11 +529,6 @@ DbTableMethod db_hash = db_free_table_continue_hash, db_print_hash, db_foreach_offheap_hash, -#ifdef HARDDEBUG - db_check_table_hash, -#else - NULL, -#endif db_lookup_dbterm_hash, db_finalize_dbterm_hash }; @@ -846,7 +841,6 @@ Lnew: grow(tb, nitems); } } - CHECK_TABLES(); return DB_ERROR_NONE; Ldone: @@ -871,7 +865,6 @@ get_term_list(Process *p, DbTableHash *tb, Eterm key, HashValue hval, } } copy = build_term_list(p, b1, b2, sz, tb); - CHECK_TABLES(); if (bend) { *bend = b2; } @@ -2898,24 +2891,3 @@ Eterm erts_ets_hash_sizeof_ext_segtab(void) return make_small(((SIZEOF_EXT_SEGTAB(0)-1) / sizeof(UWord)) + 1); } -#ifdef HARDDEBUG - -void db_check_table_hash(DbTable *tbl) -{ - DbTableHash *tb = &tbl->hash; - HashDbTerm* list; - int j; - - for (j = 0; j < NACTIVE(tb); j++) { - if ((list = BUCKET(tb,j)) != 0) { - while (list != 0) { - if (!is_tuple(make_tuple(list->dbterm.tpl))) { - erts_exit(ERTS_ERROR_EXIT, "Bad term in slot %d of ets table", j); - } - list = list->next; - } - } - } -} - -#endif diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 3c2dd65e61..3dd06fd7bb 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -180,7 +180,6 @@ static ERTS_INLINE TreeDbTerm* replace_dbterm(DbTableTree *tb, TreeDbTerm* old, static TreeDbTerm *traverse_until(TreeDbTerm *t, int *current, int to); static void check_slot_pos(DbTableTree *tb); static void check_saved_stack(DbTableTree *tb); -static int check_table_tree(DbTableTree* tb, TreeDbTerm *t); #define TREE_DEBUG #endif @@ -442,11 +441,6 @@ DbTableMethod db_tree = db_free_table_continue_tree, db_print_tree, db_foreach_offheap_tree, -#ifdef HARDDEBUG - db_check_table_tree, -#else - NULL, -#endif db_lookup_dbterm_tree, db_finalize_dbterm_tree @@ -3130,6 +3124,9 @@ static void do_dump_tree2(DbTableTree* tb, int to, void *to_arg, int show, #ifdef HARDDEBUG +/* + * No called, but kept as it might come to use + */ void db_check_table_tree(DbTable *tbl) { DbTableTree *tb = &tbl->tree; diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 958b504a60..e22113b72f 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -185,7 +185,6 @@ typedef struct db_table_method void (*db_foreach_offheap)(DbTable* db, /* [in out] */ void (*func)(ErlOffHeap *, void *), void *arg); - void (*db_check_table)(DbTable* tb); /* Lookup a dbterm for updating. Return false if not found. */ int (*db_lookup_dbterm)(Process *, DbTable *, Eterm key, Eterm obj, @@ -257,7 +256,6 @@ typedef struct db_table_common { "local" fixations not included. */ /* All 32-bit fields */ Uint32 status; /* bit masks defined below */ - int slot; /* slot index in meta_main_tab */ int keypos; /* defaults to 1 */ int compress; } DbTableCommon; @@ -527,14 +525,5 @@ erts_db_get_match_prog_binary(Eterm term) #define Binary2MatchProg(BP) \ (ASSERT(IsMatchProgBinary((BP))), \ ((MatchProg *) ERTS_MAGIC_BIN_DATA((BP)))) -/* -** Debugging -*/ -#ifdef HARDDEBUG -void db_check_tables(void); /* in db.c */ -#define CHECK_TABLES() db_check_tables() -#else -#define CHECK_TABLES() -#endif #endif /* _DB_UTIL_H */ diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 7f7c06da8e..da73469516 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -101,13 +101,11 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "code_write_permission", NULL }, { "purge_state", NULL }, { "meta_name_tab", "address" }, - { "meta_main_tab_slot", "address" }, { "db_tab", "address" }, { "proc_status", "pid" }, { "proc_trace", "pid" }, { "ports_snapshot", NULL }, { "db_tab_fix", "address" }, - { "meta_main_tab_main", NULL }, { "db_hash_slot", "address" }, { "node_table", NULL }, { "dist_table", NULL }, -- cgit v1.2.3 From ba23c44add22306495b66d7258f835db00f12868 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 27 Feb 2017 16:18:05 +0100 Subject: erts: Remove now redundant 'id' from DbTableCommon 'the_name' keeps name of all tables. 'type' & DB_NAMED_TABLE mark tables as named. table ref id is built from magic bin when needed. --- erts/emulator/beam/erl_db.c | 94 ++++++++++++++++++++-------------------- erts/emulator/beam/erl_db_util.h | 5 ++- 2 files changed, 50 insertions(+), 49 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 11936ed9dc..5d14530d84 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -261,6 +261,17 @@ is_table_alive(DbTable *tb) return !!rtb; } +static ERTS_INLINE int +is_table_named(DbTable *tb) +{ +#ifdef ERTS_SMP + return tb->common.type & DB_NAMED_TABLE; +#else + return tb->common.status & DB_NAMED_TABLE; +#endif +} + + static ERTS_INLINE void tid_clear(Process *c_p, DbTable *tb) { @@ -660,10 +671,7 @@ DbTable* db_get_table_aux(Process *p, */ ASSERT(erts_get_scheduler_data()); - tb = tid2tab(id); - if (tb) - id = tb->common.id; - else if (is_atom(id)) { + if (is_atom(id)) { struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&mtl); if (!meta_already_locked) erts_smp_rwmtx_rlock(mtl); @@ -672,7 +680,7 @@ DbTable* db_get_table_aux(Process *p, || erts_lc_rwmtx_is_rwlocked(mtl)); mtl = NULL; } - + tb = NULL; if (bucket->pu.tb != NULL) { if (is_atom(bucket->u.name_atom)) { /* single */ if (bucket->u.name_atom == id) @@ -690,11 +698,13 @@ DbTable* db_get_table_aux(Process *p, } } } + else + tb = tid2tab(id); + if (tb) { db_lock(tb, kind); - if (tb->common.id != id - || ((tb->common.status & what) == 0 - && p->common.id != tb->common.owner)) { + if ((tb->common.status & what) == 0 + && p->common.id != tb->common.owner) { db_unlock(tb, kind); tb = NULL; } @@ -777,9 +787,10 @@ static int remove_named_tab(DbTable *tb, int have_lock) { int ret = 0; erts_smp_rwmtx_t* rwlock; - Eterm name_atom = tb->common.id; + Eterm name_atom = tb->common.the_name; struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom, &rwlock); + ASSERT(is_table_named(tb)); #ifdef ERTS_SMP if (!have_lock && erts_smp_rwmtx_tryrwlock(rwlock) == EBUSY) { db_unlock(tb, LCK_WRITE); @@ -1528,8 +1539,8 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) if (!tb) BIF_ERROR(BIF_P, BADARG); else { - if (is_atom(tb->common.id)) { - old_name = tb->common.id; + if (is_table_named(tb)) { + old_name = tb->common.the_name; goto named_tab; } lck2 = NULL; @@ -1544,24 +1555,19 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) if (!tb) goto badarg; - if (is_not_atom(tb->common.id)) { /* Not a named table */ - tb->common.the_name = BIF_ARG_2; - goto done; - } + if (is_table_named(tb)) { + if (!insert_named_tab(BIF_ARG_2, tb, 1)) + goto badarg; - if (!insert_named_tab(BIF_ARG_2, tb, 1)) - goto badarg; - - if (!remove_named_tab(tb, 1)) - erts_exit(ERTS_ERROR_EXIT,"Could not find named tab %s", tb->common.id); - - tb->common.id = tb->common.the_name = BIF_ARG_2; - - done: - if (is_atom(tb->common.id)) - ret = tb->common.id; - else + if (!remove_named_tab(tb, 1)) + erts_exit(ERTS_ERROR_EXIT,"Could not find named tab %s", tb->common.the_name); + ret = BIF_ARG_2; + } + else { /* Not a named table */ ret = BIF_ARG_1; + } + tb->common.the_name = BIF_ARG_2; + db_unlock(tb, LCK_WRITE); erts_smp_rwmtx_rwunlock(lck1); if (lck2) @@ -1691,6 +1697,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) } else if (val == am_named_table) { is_named = 1; + status |= DB_NAMED_TABLE; } else if (val == am_compressed) { is_compressed = 1; @@ -1765,16 +1772,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) make_btid(tb); - if (is_named) { - ret = BIF_ARG_1; - tb->common.id = ret; - } - else { - ret = make_tid(BIF_P, tb); - tb->common.id = THE_NON_VALUE; /* or what? */ - } - - if (!is_named) + if (is_named) + ret = BIF_ARG_1; + else ret = make_tid(BIF_P, tb); save_sched_table(BIF_P, tb); @@ -1951,7 +1951,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) tid_clear(BIF_P, tb); - if (is_atom(tb->common.id)) + if (is_table_named(tb)) remove_named_tab(tb, 0); /* disable inheritance */ @@ -2385,8 +2385,8 @@ ets_all_reply(ErtsSchedulerData *esdp, ErtsEtsAllReq **reqpp, while (1) { if (is_table_alive(tb)) { Eterm tid; - if (is_atom(tb->common.id)) - tid = tb->common.id; + if (is_table_named(tb)) + tid = tb->common.the_name; else tid = erts_mk_magic_ref(&hp, ohp, tb->common.btid); list = CONS(hp, tid, list); @@ -3446,7 +3446,7 @@ send_ets_transfer_message(Process *c_p, Process *proc, Eterm tid, hd_copy, msg, sender; hsz = 5; - if (is_not_atom(tb->common.id)) + if (!is_table_named(tb)) hsz += ERTS_MAGIC_REF_THING_SIZE; if (is_immed(heir_data)) hd_sz = 0; @@ -3456,8 +3456,8 @@ send_ets_transfer_message(Process *c_p, Process *proc, } mp = erts_alloc_message_heap(proc, locks, hsz, &hp, &ohp); - if (is_atom(tb->common.id)) - tid = tb->common.id; + if (is_table_named(tb)) + tid = tb->common.the_name; else tid = erts_mk_magic_ref(&hp, ohp, tb->common.btid); if (!hd_sz) @@ -3584,7 +3584,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) tb->common.status &= ~(DB_PROTECTED | DB_PUBLIC | DB_PRIVATE); tb->common.status |= DB_DELETE; - if (is_atom(tb->common.id)) + if (is_table_named(tb)) remove_named_tab(tb, 0); free_heir_data(tb); @@ -3990,7 +3990,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) } else if (What == am_node) { ret = erts_this_dist_entry->sysname; } else if (What == am_named_table) { - ret = is_atom(tb->common.id) ? am_true : am_false; + ret = is_table_named(tb) ? am_true : am_false; } else if (What == am_compressed) { ret = tb->common.compress ? am_true : am_false; } @@ -4097,8 +4097,8 @@ static void print_table(fmtfn_t to, void *to_arg, int show, DbTable* tb) Eterm tid; Eterm heap[ERTS_MAGIC_REF_THING_SIZE]; - if (is_atom(tb->common.id)) { - tid = tb->common.id; + if (is_table_named(tb)) { + tid = tb->common.the_name; } else { ErlOffHeap oh; ERTS_INIT_OFF_HEAP(&oh); diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index e22113b72f..198f339751 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -243,7 +243,6 @@ typedef struct db_table_common { UWord heir_data; /* To send in ETS-TRANSFER (is_immed or (DbTerm*) */ Uint64 heir_started_interval; /* To further identify the heir */ Eterm the_name; /* an atom */ - Eterm id; /* atom | integer */ Binary *btid; DbTableMethod* meth; /* table methods */ erts_smp_atomic_t nitems; /* Total number of items in table */ @@ -273,8 +272,10 @@ typedef struct db_table_common { #define DB_ORDERED_SET (1 << 9) #define DB_DELETE (1 << 10) /* table is being deleted */ #define DB_FREQ_READ (1 << 11) +#define DB_NAMED_TABLE (1 << 12) -#define ERTS_ETS_TABLE_TYPES (DB_BAG|DB_SET|DB_DUPLICATE_BAG|DB_ORDERED_SET|DB_FINE_LOCKED|DB_FREQ_READ) +#define ERTS_ETS_TABLE_TYPES (DB_BAG|DB_SET|DB_DUPLICATE_BAG|DB_ORDERED_SET\ + |DB_FINE_LOCKED|DB_FREQ_READ|DB_NAMED_TABLE) #define IS_HASH_TABLE(Status) (!!((Status) & \ (DB_BAG | DB_SET | DB_DUPLICATE_BAG))) -- cgit v1.2.3 From dc7048b07adc0bf7f6dc841bbb210d950e7a6ba9 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 28 Feb 2017 18:29:22 +0100 Subject: erts: Cleanup table status bits --- erts/emulator/beam/erl_db.c | 2 +- erts/emulator/beam/erl_db_util.h | 24 +++++++++++------------- 2 files changed, 12 insertions(+), 14 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 5d14530d84..9c597afabf 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1614,7 +1614,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); } - status = DB_NORMAL | DB_SET | DB_PROTECTED; + status = DB_SET | DB_PROTECTED; keypos = 1; is_named = 0; #ifdef ERTS_SMP diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 198f339751..4fb285186e 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -260,19 +260,17 @@ typedef struct db_table_common { } DbTableCommon; /* These are status bit patterns */ -#define DB_NORMAL (1 << 0) -#define DB_PRIVATE (1 << 1) -#define DB_PROTECTED (1 << 2) -#define DB_PUBLIC (1 << 3) -#define DB_BAG (1 << 4) -#define DB_SET (1 << 5) -/*#define DB_LHASH (1 << 6)*/ -#define DB_FINE_LOCKED (1 << 7) /* fine grained locking enabled */ -#define DB_DUPLICATE_BAG (1 << 8) -#define DB_ORDERED_SET (1 << 9) -#define DB_DELETE (1 << 10) /* table is being deleted */ -#define DB_FREQ_READ (1 << 11) -#define DB_NAMED_TABLE (1 << 12) +#define DB_PRIVATE (1 << 0) +#define DB_PROTECTED (1 << 1) +#define DB_PUBLIC (1 << 2) +#define DB_DELETE (1 << 3) /* table is being deleted */ +#define DB_SET (1 << 4) +#define DB_BAG (1 << 5) +#define DB_DUPLICATE_BAG (1 << 6) +#define DB_ORDERED_SET (1 << 7) +#define DB_FINE_LOCKED (1 << 8) /* write_concurrency */ +#define DB_FREQ_READ (1 << 9) /* read_concurrency */ +#define DB_NAMED_TABLE (1 << 10) #define ERTS_ETS_TABLE_TYPES (DB_BAG|DB_SET|DB_DUPLICATE_BAG|DB_ORDERED_SET\ |DB_FINE_LOCKED|DB_FREQ_READ|DB_NAMED_TABLE) -- cgit v1.2.3 From b9240f72148c536b038c0d26b1ef83da7c951429 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 2 Mar 2017 19:58:04 +0100 Subject: erts: Improve reduction count during table cleanup --- erts/emulator/beam/erl_db.c | 76 ++++++++++++++++++++++------------------ erts/emulator/beam/erl_db_hash.c | 30 ++++++++-------- erts/emulator/beam/erl_db_hash.h | 2 +- erts/emulator/beam/erl_db_tree.c | 28 +++++++-------- erts/emulator/beam/erl_db_util.h | 2 +- 5 files changed, 73 insertions(+), 65 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 9c597afabf..2f188b5391 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -356,9 +356,9 @@ static void fix_table_locked(Process* p, DbTable* tb); static void unfix_table_locked(Process* p, DbTable* tb, db_lock_kind_t* kind); static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data); static void free_heir_data(DbTable*); -static void free_fixations_locked(Process* p, DbTable *tb); +static SWord free_fixations_locked(Process* p, DbTable *tb); -static int free_table_cont(Process *p, DbTable *tb); +static SWord free_table_continue(Process *p, DbTable *tb, SWord reds); static void print_table(fmtfn_t to, void *to_arg, int show, DbTable* tb); static BIF_RETTYPE ets_select_delete_1(BIF_ALIST_1); static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1); @@ -393,8 +393,6 @@ free_dbtable(void *vtb) erts_smp_atomic_read_nob(&tb->common.memory_size)-sizeof(DbTable), tb->common.fixations); } - erts_fprintf(stderr, "ets: free_dbtable(%T) deleted!!!\r\n", - tb->common.id); #endif #ifdef ERTS_SMP erts_smp_rwmtx_destroy(&tb->common.rwlock); @@ -1904,7 +1902,8 @@ BIF_RETTYPE ets_lookup_element_3(BIF_ALIST_3) */ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) { - int trap; + SWord initial_reds = ERTS_BIF_REDS_LEFT(BIF_P); + SWord reds = initial_reds; DbTable* tb; #ifdef HARDDEBUG @@ -1958,11 +1957,10 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) free_heir_data(tb); tb->common.heir = am_none; - free_fixations_locked(BIF_P, tb); + reds -= free_fixations_locked(BIF_P, tb); db_unlock(tb, LCK_WRITE); - trap = free_table_cont(BIF_P, tb); - if (trap) { + if (free_table_continue(BIF_P, tb, reds) < 0) { /* * Package the DbTable* pointer into a bignum so that it can be safely * passed through a trap. We used to pass the DbTable* pointer directly @@ -1972,9 +1970,11 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) Eterm *hp = HAlloc(BIF_P, 2); hp[0] = make_pos_bignum_header(1); hp[1] = (Eterm) tb; + BUMP_ALL_REDS(BIF_P); BIF_TRAP1(&ets_delete_continue_exp, BIF_P, make_big(hp)); } else { + BUMP_REDS(BIF_P, (initial_reds - reds)); BIF_RET(am_true); } } @@ -3471,9 +3471,10 @@ send_ets_transfer_message(Process *c_p, Process *proc, /* Auto-release fixation from exiting process */ -static void proc_cleanup_fixed_table(Process* p, DbFixation* fix) +static SWord proc_cleanup_fixed_table(Process* p, DbFixation* fix) { DbTable* tb = btid2tab(fix->tabs.btid); + SWord work = 0; ASSERT(fix->procs.p == p); (void)p; if (tb) { @@ -3496,7 +3497,7 @@ static void proc_cleanup_fixed_table(Process* p, DbFixation* fix) erts_smp_mtx_unlock(&tb->common.fixlock); #endif if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status)) { - db_unfix_table_hash(&(tb->hash)); + work += db_unfix_table_hash(&(tb->hash)); } ASSERT(sizeof(DbFixation) == ERTS_ALC_DBG_BLK_SZ(fix)); @@ -3516,6 +3517,9 @@ static void proc_cleanup_fixed_table(Process* p, DbFixation* fix) } erts_free(ERTS_ALC_T_DB_FIXATION, fix); ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation)); + ++work; + + return work; } @@ -3543,6 +3547,8 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) CleanupState *state = (CleanupState *) c_p->u.terminate; Eterm pid = c_p->common.id; CleanupState default_state; + SWord initial_reds = ERTS_BIF_REDS_LEFT(c_p); + SWord reds = initial_reds; if (!state) { state = &default_state; @@ -3588,17 +3594,17 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) remove_named_tab(tb, 0); free_heir_data(tb); - free_fixations_locked(c_p, tb); + reds -= free_fixations_locked(c_p, tb); db_unlock(tb, LCK_WRITE); state->op = FREE_OWNED_TABLE; - /* fall through */ + break; } case FREE_OWNED_TABLE: - if (free_table_cont(c_p, state->tb)) + reds = free_table_continue(c_p, state->tb, reds); + if (reds < 0) goto yield; state->op = GET_OWNED_TABLE; - break; case UNFIX_TABLES: { @@ -3612,20 +3618,21 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) if (state != &default_state) erts_free(ERTS_ALC_T_DB_PROC_CLEANUP, state); c_p->u.terminate = NULL; + + BUMP_REDS(c_p, (initial_reds - reds)); return 0; } fixed_tabs_delete(c_p, fix); - proc_cleanup_fixed_table(c_p, fix); + reds -= proc_cleanup_fixed_table(c_p, fix); - BUMP_REDS(c_p, 10); break; } default: ERTS_DB_INTERNAL_ERROR("Bad internal state"); } - } while (ERTS_BIF_REDS_LEFT(c_p) > 0); + } while (reds > 0); yield: @@ -3740,6 +3747,7 @@ struct free_fixations_ctx { Process* p; DbTable* tb; + SWord cnt; }; static void free_fixations_op(DbFixation* fix, void* vctx) @@ -3785,6 +3793,7 @@ static void free_fixations_op(DbFixation* fix, void* vctx) ctx->tb, (void *) fix, sizeof(DbFixation)); ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation)); } + ctx->cnt++; } #ifdef ERTS_SMP @@ -3802,7 +3811,7 @@ int erts_db_execute_free_fixation(Process* p, DbFixation* fix) } #endif -static void free_fixations_locked(Process* p, DbTable *tb) +static SWord free_fixations_locked(Process* p, DbTable *tb) { struct free_fixations_ctx ctx; @@ -3810,9 +3819,11 @@ static void free_fixations_locked(Process* p, DbTable *tb) ctx.p = p; ctx.tb = tb; + ctx.cnt = 0; fixing_procs_rbt_foreach_destroy(&tb->common.fixing_procs, free_fixations_op, &ctx); tb->common.fixing_procs = NULL; + return ctx.cnt; } static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data) @@ -3876,42 +3887,40 @@ static void free_heir_data(DbTable* tb) static BIF_RETTYPE ets_delete_trap(BIF_ALIST_1) { - Process *p = BIF_P; + SWord initial_reds = ERTS_BIF_REDS_LEFT(BIF_P); + SWord reds = initial_reds; Eterm cont = BIF_ARG_1; - int trap; Eterm* ptr = big_val(cont); DbTable *tb = *((DbTable **) (UWord) (ptr + 1)); ASSERT(*ptr == make_pos_bignum_header(1)); - trap = free_table_cont(p, tb); - - if (trap) { - BIF_TRAP1(&ets_delete_continue_exp, p, cont); + if (free_table_continue(BIF_P, tb, reds) < 0) { + BUMP_ALL_REDS(BIF_P); + BIF_TRAP1(&ets_delete_continue_exp, BIF_P, cont); } else { + BUMP_REDS(BIF_P, (initial_reds - reds)); BIF_RET(am_true); } } /* - * free_table_cont() returns 0 when done and !0 when more work is needed. + * free_table_continue() returns reductions left + * done if >= 0 + * yield if < 0 */ -static int free_table_cont(Process *p, DbTable *tb) +static SWord free_table_continue(Process *p, DbTable *tb, SWord reds) { - Eterm result; - - result = tb->common.meth->db_free_table_continue(tb); + reds = tb->common.meth->db_free_table_continue(tb, reds); - if (result == 0) { + if (reds < 0) { #ifdef HARDDEBUG erts_fprintf(stderr,"ets: free_table_cont %T (continue begin)\r\n", tb->common.id); #endif /* More work to be done. Let other processes work and call us again. */ - BUMP_ALL_REDS(p); - return !0; } else { #ifdef HARDDEBUG @@ -3921,9 +3930,8 @@ static int free_table_cont(Process *p, DbTable *tb) /* Completely done - we will not get called again. */ delete_owned_table(p, tb); table_dec_refc(tb, 0); - BUMP_REDS(p, 100); - return 0; } + return reds; } struct fixing_procs_info_ctx diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 781886a822..18405342da 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -418,7 +418,7 @@ static void db_print_hash(fmtfn_t to, DbTable *tbl); static int db_free_table_hash(DbTable *tbl); -static int db_free_table_continue_hash(DbTable *tbl); +static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds); static void db_foreach_offheap_hash(DbTable *, @@ -576,9 +576,10 @@ static void restore_fixdel(DbTableHash* tb, FixedDeletion* fixdel) ** Table interface routines ie what's called by the bif's */ -void db_unfix_table_hash(DbTableHash *tb) +SWord db_unfix_table_hash(DbTableHash *tb) { FixedDeletion* fixdel; + SWord work = 0; ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rwlocked(&tb->common.rwlock) || (erts_smp_lc_rwmtx_is_rlocked(&tb->common.rwlock) @@ -599,7 +600,7 @@ restart: if (!IS_FIXED(tb)) { goto restart; /* unfixed again! */ } - return; + return work; } if (ix < NACTIVE(tb)) { bp = &BUCKET(tb, ix); @@ -609,6 +610,7 @@ restart: if (b->hvalue == INVALID_HASH) { *bp = b->next; free_term(tb, b); + work++; b = *bp; } else { bp = &b->next; @@ -624,9 +626,12 @@ restart: (void *) fx, sizeof(FixedDeletion)); ERTS_ETS_MISC_MEM_ADD(-sizeof(FixedDeletion)); + work++; } /* ToDo: Maybe try grow/shrink the table as well */ + + return work; } int db_create_hash(Process *p, DbTable *tbl) @@ -2080,19 +2085,17 @@ static void db_print_hash(fmtfn_t to, void *to_arg, int show, DbTable *tbl) /* release all memory occupied by a single table */ static int db_free_table_hash(DbTable *tbl) { - while (!db_free_table_continue_hash(tbl)) + while (db_free_table_continue_hash(tbl, ERTS_SWORD_MAX) < 0) ; return 0; } -static int db_free_table_continue_hash(DbTable *tbl) +static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds) { DbTableHash *tb = &tbl->hash; - int done; FixedDeletion* fixdel = (FixedDeletion*) erts_smp_atomic_read_acqb(&tb->fixdel); ERTS_SMP_LC_ASSERT(IS_TAB_WLOCKED(tb) || (tb->common.status & DB_DELETE)); - done = 0; while (fixdel != NULL) { FixedDeletion *fx = fixdel; @@ -2102,22 +2105,21 @@ static int db_free_table_continue_hash(DbTable *tbl) (void *) fx, sizeof(FixedDeletion)); ERTS_ETS_MISC_MEM_ADD(-sizeof(FixedDeletion)); - if (++done >= 2*DELETE_RECORD_LIMIT) { + if (--reds < 0) { erts_smp_atomic_set_relb(&tb->fixdel, (erts_aint_t)fixdel); - return 0; /* Not done */ + return reds; /* Not done */ } } erts_smp_atomic_set_relb(&tb->fixdel, (erts_aint_t)NULL); - done /= 2; while(tb->nslots != 0) { - done += 1 + EXT_SEGSZ/64 + free_seg(tb, 1); + reds -= EXT_SEGSZ/64 + free_seg(tb, 1); /* * If we have done enough work, get out here. */ - if (done >= DELETE_RECORD_LIMIT) { - return 0; /* Not done */ + if (reds < 0) { + return reds; /* Not done */ } } #ifdef ERTS_SMP @@ -2132,7 +2134,7 @@ static int db_free_table_continue_hash(DbTable *tbl) } #endif ASSERT(erts_smp_atomic_read_nob(&tb->common.memory_size) == sizeof(DbTable)); - return 1; /* Done */ + return reds; /* Done */ } diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index cd02418370..c340c72311 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -75,7 +75,7 @@ typedef struct db_table_hash { ** table types. The process is always an [in out] parameter. */ void db_initialize_hash(void); -void db_unfix_table_hash(DbTableHash *tb /* [in out] */); +SWord db_unfix_table_hash(DbTableHash *tb); Uint db_kept_items_hash(DbTableHash *tb); /* Interface for meta pid table */ diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 3dd06fd7bb..14a7451267 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -282,7 +282,7 @@ struct select_delete_context { static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key); static TreeDbTerm *linkout_object_tree(DbTableTree *tb, Eterm object); -static int do_free_tree_cont(DbTableTree *tb, int num_left); +static SWord do_free_tree_continue(DbTableTree *tb, SWord reds); static void free_term(DbTableTree *tb, TreeDbTerm* p); static int balance_left(TreeDbTerm **this); static int balance_right(TreeDbTerm **this); @@ -388,7 +388,7 @@ static void db_print_tree(fmtfn_t to, void *to_arg, int show, DbTable *tbl); static int db_free_table_tree(DbTable *tbl); -static int db_free_table_continue_tree(DbTable *tbl); +static SWord db_free_table_continue_tree(DbTable *tbl, SWord); static void db_foreach_offheap_tree(DbTable *, void (*)(ErlOffHeap *, void *), @@ -1751,23 +1751,22 @@ static void db_print_tree(fmtfn_t to, void *to_arg, /* release all memory occupied by a single table */ static int db_free_table_tree(DbTable *tbl) { - while (!db_free_table_continue_tree(tbl)) + while (db_free_table_continue_tree(tbl, ERTS_SWORD_MAX) < 0) ; return 1; } -static int db_free_table_continue_tree(DbTable *tbl) +static SWord db_free_table_continue_tree(DbTable *tbl, SWord reds) { DbTableTree *tb = &tbl->tree; - int result; if (!tb->deletion) { tb->static_stack.pos = 0; tb->deletion = 1; PUSH_NODE(&tb->static_stack, tb->root); } - result = do_free_tree_cont(tb, DELETE_RECORD_LIMIT); - if (result) { /* Completely done. */ + reds = do_free_tree_continue(tb, reds); + if (reds >= 0) { /* Completely done. */ erts_db_free(ERTS_ALC_T_DB_STK, (DbTable *) tb, (void *) tb->static_stack.array, @@ -1775,7 +1774,7 @@ static int db_free_table_continue_tree(DbTable *tbl) ASSERT(erts_smp_atomic_read_nob(&tb->common.memory_size) == sizeof(DbTable)); } - return result; + return reds; } static int db_delete_all_objects_tree(Process* p, DbTable* tbl) @@ -2058,7 +2057,7 @@ static int analyze_pattern(DbTableTree *tb, Eterm pattern, return DB_ERROR_NONE; } -static int do_free_tree_cont(DbTableTree *tb, int num_left) +static SWord do_free_tree_continue(DbTableTree *tb, SWord reds) { TreeDbTerm *root; TreeDbTerm *p; @@ -2077,15 +2076,14 @@ static int do_free_tree_cont(DbTableTree *tb, int num_left) root = p; } else { free_term(tb, root); - if (--num_left > 0) { - break; - } else { - return 0; /* Done enough for now */ - } + if (--reds < 0) { + return reds; /* Done enough for now */ + } + break; } } } - return 1; + return reds; } /* diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 4fb285186e..72298842d7 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -175,7 +175,7 @@ typedef struct db_table_method DbTable* db /* [in out] */ ); int (*db_free_table)(DbTable* db /* [in out] */ ); - int (*db_free_table_continue)(DbTable* db); /* [in out] */ + SWord (*db_free_table_continue)(DbTable* db, SWord reds); void (*db_print)(fmtfn_t to, void* to_arg, -- cgit v1.2.3 From 3fcf58b142e22adb754b312576570c904655f877 Mon Sep 17 00:00:00 2001 From: Deepak Goel Date: Wed, 15 Feb 2017 15:13:38 -0800 Subject: This patch fixes the issue in which erlang fails to start if the hostname is 64 characters on a linux system. --- erts/emulator/beam/erl_mtrace.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c index e275867928..bb6f8660f1 100644 --- a/erts/emulator/beam/erl_mtrace.c +++ b/erts/emulator/beam/erl_mtrace.c @@ -572,7 +572,7 @@ void erts_mtrace_pre_init(void) void erts_mtrace_init(char *receiver, char *nodename) { - char hostname[MAXHOSTNAMELEN]; + char hostname[MAXHOSTNAMELEN + 1]; char pid[21]; /* enough for a 64 bit number */ socket_desc = ERTS_SOCK_INVALID_SOCKET; @@ -613,9 +613,10 @@ void erts_mtrace_init(char *receiver, char *nodename) } tracep = trace_buffer; endp = trace_buffer + TRACE_BUF_SZ; - if (erts_sock_gethostname(hostname, MAXHOSTNAMELEN) != 0) + /* gethostname requires that the len is max(hostname) + 1 */ + if (erts_sock_gethostname(hostname, MAXHOSTNAMELEN + 1) != 0) hostname[0] = '\0'; - hostname[MAXHOSTNAMELEN-1] = '\0'; + hostname[MAXHOSTNAMELEN] = '\0'; sys_get_pid(pid, sizeof(pid)); write_trace_header(nodename ? nodename : "", pid, hostname); erts_mtrace_update_heap_size(); -- cgit v1.2.3 From 5b7e50500f6cb3e680b277f871392ac706c5c1d7 Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Fri, 3 Jun 2016 00:39:01 +0100 Subject: ETS: Allow for matchspec-based replacement --- erts/emulator/beam/bif.tab | 1 + erts/emulator/beam/erl_db.c | 96 +++++++++ erts/emulator/beam/erl_db.h | 1 + erts/emulator/beam/erl_db_hash.c | 318 ++++++++++++++++++++++++++++++ erts/emulator/beam/erl_db_tree.c | 407 +++++++++++++++++++++++++++++++++++++-- erts/emulator/beam/erl_db_util.h | 8 + 6 files changed, 817 insertions(+), 14 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 6f50297fc5..4140938210 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -361,6 +361,7 @@ bif ets:select_reverse/1 bif ets:select_reverse/2 bif ets:select_reverse/3 bif ets:select_delete/2 +bif ets:select_replace/2 bif ets:match_spec_compile/1 bif ets:match_spec_run_r/3 diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 2f188b5391..81fe79a92e 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -362,6 +362,7 @@ static SWord free_table_continue(Process *p, DbTable *tb, SWord reds); static void print_table(fmtfn_t to, void *to_arg, int show, DbTable* tb); static BIF_RETTYPE ets_select_delete_1(BIF_ALIST_1); static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1); +static BIF_RETTYPE ets_select_replace_1(BIF_ALIST_1); static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1); static BIF_RETTYPE ets_delete_trap(BIF_ALIST_1); static Eterm table_info(Process* p, DbTable* tb, Eterm What); @@ -376,6 +377,7 @@ static BIF_RETTYPE ets_select3(Process* p, Eterm arg1, Eterm arg2, Eterm arg3); */ Export ets_select_delete_continue_exp; Export ets_select_count_continue_exp; +Export ets_select_replace_continue_exp; Export ets_select_continue_exp; /* @@ -2922,6 +2924,95 @@ BIF_RETTYPE ets_select_count_2(BIF_ALIST_2) return result; } +/* + ** This is for trapping, cannot be called directly. + */ +static BIF_RETTYPE ets_select_replace_1(BIF_ALIST_1) +{ + Process *p = BIF_P; + Eterm a1 = BIF_ARG_1; + BIF_RETTYPE result; + DbTable* tb; + int cret; + Eterm ret; + Eterm *tptr; + db_lock_kind_t kind = LCK_WRITE_REC; + + CHECK_TABLES(); + ASSERT(is_tuple(a1)); + tptr = tuple_val(a1); + ASSERT(arityval(*tptr) >= 1); + + if ((tb = db_get_table(p, tptr[1], DB_WRITE, kind)) == NULL) { + BIF_ERROR(p,BADARG); + } + + cret = tb->common.meth->db_select_replace_continue(p,tb,a1,&ret); + + if(!DID_TRAP(p,ret) && ITERATION_SAFETY(p,tb) != ITER_SAFE) { + unfix_table_locked(p, tb, &kind); + } + + db_unlock(tb, kind); + + switch (cret) { + case DB_ERROR_NONE: + ERTS_BIF_PREP_RET(result, ret); + break; + default: + ERTS_BIF_PREP_ERROR(result, p, BADARG); + break; + } + erts_match_set_release_result(p); + + return result; +} + + +BIF_RETTYPE ets_select_replace_2(BIF_ALIST_2) +{ + BIF_RETTYPE result; + DbTable* tb; + int cret; + Eterm ret; + enum DbIterSafety safety; + + CHECK_TABLES(); + + if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) { + BIF_ERROR(BIF_P, BADARG); + } + safety = ITERATION_SAFETY(BIF_P,tb); + if (safety == ITER_UNSAFE) { + local_fix_table(tb); + } + cret = tb->common.meth->db_select_replace(BIF_P, tb, BIF_ARG_2, &ret); + + if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) { + fix_table_locked(BIF_P,tb); + } + if (safety == ITER_UNSAFE) { + local_unfix_table(tb); + } + db_unlock(tb, LCK_WRITE_REC); + + switch (cret) { + case DB_ERROR_NONE: + ERTS_BIF_PREP_RET(result, ret); + break; + case DB_ERROR_SYSRES: + ERTS_BIF_PREP_ERROR(result, BIF_P, SYSTEM_LIMIT); + break; + default: + ERTS_BIF_PREP_ERROR(result, BIF_P, BADARG); + break; + } + + erts_match_set_release_result(BIF_P); + + return result; +} + BIF_RETTYPE ets_select_reverse_3(BIF_ALIST_3) { @@ -3334,6 +3425,11 @@ void init_db(ErtsDbSpinCount db_spin_count) am_ets, am_atom_put("count_trap",11), 1, &ets_select_count_1); + /* Non visual BIF to trap to. */ + erts_init_trap_export(&ets_select_replace_continue_exp, + am_ets, am_atom_put("replace_trap",11), 1, + &ets_select_replace_1); + /* Non visual BIF to trap to. */ erts_init_trap_export(&ets_select_continue_exp, am_ets, am_atom_put("select_trap",11), 1, diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index cbf4b9e007..4ff9f224e8 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -122,6 +122,7 @@ extern int erts_ets_realloc_always_moves; /* set in erl_init */ extern int erts_ets_always_compress; /* set in erl_init */ extern Export ets_select_delete_continue_exp; extern Export ets_select_count_continue_exp; +extern Export ets_select_replace_continue_exp; extern Export ets_select_continue_exp; extern erts_smp_atomic_t erts_ets_misc_mem_size; diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 18405342da..a4ca6dce77 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -288,6 +288,9 @@ static ERTS_INLINE Sint next_slot_w(DbTableHash* tb, Uint ix, #endif } +#ifndef MIN +#define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) +#endif /* * Some special binary flags @@ -403,6 +406,9 @@ static int db_select_count_hash(Process *p, DbTable *tbl, Eterm tid, static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret); +static int db_select_replace_hash(Process *p, DbTable *tbl, + Eterm pattern, Eterm *ret); + static int db_select_continue_hash(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); @@ -411,6 +417,10 @@ static int db_select_count_continue_hash(Process *p, DbTable *tbl, static int db_select_delete_continue_hash(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); + +static int db_select_replace_continue_hash(Process *p, DbTable *tbl, + Eterm continuation, Eterm *ret); + static int db_take_hash(Process *, DbTable *, Eterm, Eterm *); static void db_print_hash(fmtfn_t to, void *to_arg, @@ -523,6 +533,8 @@ DbTableMethod db_hash = db_select_delete_continue_hash, db_select_count_hash, db_select_count_continue_hash, + db_select_replace_hash, + db_select_replace_continue_hash, db_take_hash, db_delete_all_objects_hash, db_free_table_hash, @@ -1989,6 +2001,312 @@ static int db_take_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) return DB_ERROR_NONE; } +static int db_select_replace_hash(Process *p, + DbTable *tbl, + Eterm pattern, + Eterm *ret) +{ + DbTableHash *tb = &tbl->hash; + struct mp_info mpi; + Uint slot_ix = 0; + HashDbTerm **current = NULL, **initial = NULL, **iter = NULL; + HashDbTerm *new = NULL, *next = NULL; + HashValue hval = INVALID_HASH; + unsigned current_list_pos = 0; + Eterm key; + Eterm match_res; + Eterm *hp; + int num_left = 1000; + Uint got = 0; + Eterm continuation; + int errcode; + Eterm mpb; + Eterm egot; + erts_smp_rwmtx_t* lck; + +#define RET_TO_BIF(Term,RetVal) do { \ + if (mpi.mp != NULL) { \ + erts_bin_free(mpi.mp); \ + } \ + if (mpi.lists != mpi.dlists) { \ + erts_free(ERTS_ALC_T_DB_SEL_LIST, \ + (void *) mpi.lists); \ + } \ + *ret = (Term); \ + return RetVal; \ + } while(0) + + + if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) { + RET_TO_BIF(NIL,errcode); + } + + if (!mpi.something_can_match) { + RET_TO_BIF(make_small(0), DB_ERROR_NONE); + /* can't possibly match anything */ + } + + if (!mpi.key_given) { + /* Run this code if pattern is variable or GETKEY(pattern) */ + /* is a variable */ + lck = WLOCK_HASH(tb,slot_ix); + current = &BUCKET(tb,slot_ix); + } else { + /* We have at least one */ + slot_ix = mpi.lists[current_list_pos].ix; + lck = WLOCK_HASH(tb, slot_ix); + current = mpi.lists[current_list_pos++].bucket; + ASSERT(*current == BUCKET(tb,slot_ix)); + } + + + initial = current; + for(;;) { + if ((*current) == NULL) { + if (mpi.key_given) { /* Key is bound */ + WUNLOCK_HASH(lck); + if (current_list_pos == mpi.num_lists) { + goto done; + } else { + slot_ix = mpi.lists[current_list_pos].ix; + lck = WLOCK_HASH(tb, slot_ix); + current = mpi.lists[current_list_pos].bucket; + ASSERT(mpi.lists[current_list_pos].bucket == &BUCKET(tb,slot_ix)); + ++current_list_pos; + } + } else { + if ((slot_ix=next_slot_w(tb,slot_ix,&lck)) == 0) { + goto done; + } + if (num_left <= 0) { + WUNLOCK_HASH(lck); + goto trap; + } + current = &BUCKET(tb,slot_ix); + } + initial = current; + } + else if ((*current)->hvalue == INVALID_HASH) { + current = &((*current)->next); + } + else if (tb->common.status & DB_BAG) { + match_res = db_match_dbterm(&tb->common, p, mpi.mp, 0, + &(*current)->dbterm, NULL, 0); + if (is_value(match_res) && + is_value(key = db_getkey(tb->common.keypos, match_res)) && + eq(key, GETKEY(tb, (*current)->dbterm.tpl))) + { + // we need to make sure we don't end up with duplicate objects; + // it's quite inefficient + int object_exists = 0; + for (iter = initial; *iter != NULL; iter = &((*iter)->next)) + if (((*iter)->hvalue != INVALID_HASH) + && (*iter != *current) + && db_eq(&tb->common, match_res, &(*iter)->dbterm)) + { + object_exists = 1; + break; + } + + if (!object_exists) { + next = (*current)->next; + hval = (*current)->hvalue; + new = replace_dbterm(tb, *current, match_res); + new->next = next; + new->hvalue = hval; + *current = new; + ++got; + } + } + --num_left; + current = &((*current)->next); + } + else { + match_res = db_match_dbterm(&tb->common, p, mpi.mp, 0, + &(*current)->dbterm, NULL, 0); + if (is_value(match_res) && + is_value(key = db_getkey(tb->common.keypos, match_res)) && + eq(key, GETKEY(tb, (*current)->dbterm.tpl))) + { + next = (*current)->next; + hval = (*current)->hvalue; + new = replace_dbterm(tb, *current, match_res); + new->next = next; + new->hvalue = hval; + *current = new; + ++got; + } + --num_left; + current = &((*current)->next); + } + } +done: + // the more objects we've replaced, the more reductions we've consumed + BUMP_REDS(p, MIN(2000, (1000 - num_left) + (int)got)); + RET_TO_BIF(erts_make_integer(got,p),DB_ERROR_NONE); +trap: + BUMP_ALL_REDS(p); + if (IS_USMALL(0, got)) { + hp = HAlloc(p, PROC_BIN_SIZE + 5); + egot = make_small(got); + } + else { + hp = HAlloc(p, BIG_UINT_HEAP_SIZE + PROC_BIN_SIZE + 5); + egot = uint_to_big(got, hp); + hp += BIG_UINT_HEAP_SIZE; + } + mpb = db_make_mp_binary(p,mpi.mp,&hp); + continuation = TUPLE4(hp, tb->common.id, make_small(slot_ix), + mpb, + egot); + mpi.mp = NULL; /*otherwise the return macro will destroy it */ + RET_TO_BIF(bif_trap1(&ets_select_replace_continue_exp, p, + continuation), + DB_ERROR_NONE); + +#undef RET_TO_BIF + +} + +/* +** This is called when select_replace traps +*/ +static int db_select_replace_continue_hash(Process *p, + DbTable *tbl, + Eterm continuation, + Eterm *ret) +{ + DbTableHash *tb = &tbl->hash; + Uint slot_ix; + HashDbTerm **current = NULL, **initial = NULL, **iter = NULL; + HashDbTerm *new = NULL, *next = NULL; + HashValue hval = INVALID_HASH; + Eterm key; + Eterm match_res; + Eterm *hp; + int num_left = 1000; + Uint got, prev_got; + Eterm *tptr; + Binary *mp; + Eterm egot; + erts_smp_rwmtx_t* lck; + +#define RET_TO_BIF(Term,RetVal) do { \ + *ret = (Term); \ + return RetVal; \ + } while(0) + + + tptr = tuple_val(continuation); + slot_ix = unsigned_val(tptr[2]); + mp = ((ProcBin *) binary_val(tptr[3]))->val; + if (is_big(tptr[4])) { + got = big_to_uint32(tptr[4]); + } else { + got = unsigned_val(tptr[4]); + } + prev_got = got; + + lck = WLOCK_HASH(tb,slot_ix); + if (slot_ix >= NACTIVE(tb)) { + WUNLOCK_HASH(lck); + goto done; + } + current = &BUCKET(tb,slot_ix); + initial = current; + + for(;;) { + if ((*current) == NULL) { + if ((slot_ix=next_slot_w(tb,slot_ix,&lck)) == 0) { + goto done; + } + if (num_left <= 0) { + WUNLOCK_HASH(lck); + goto trap; + } + current = &BUCKET(tb,slot_ix); + initial = current; + } + else if ((*current)->hvalue == INVALID_HASH) { + current = &((*current)->next); + } + else if (tb->common.status & DB_BAG) { + match_res = db_match_dbterm(&tb->common, p, mp, 0, + &(*current)->dbterm, NULL, 0); + if (is_value(match_res) && + is_value(key = db_getkey(tb->common.keypos, match_res)) && + eq(key, GETKEY(tb, (*current)->dbterm.tpl))) + { + // we need to make sure we don't end up with duplicate objects; + // it's quite inefficient + int object_exists = 0; + for (iter = initial; *iter != NULL; iter = &((*iter)->next)) + if (((*iter)->hvalue != INVALID_HASH) + && (*iter != *current) + && db_eq(&tb->common, match_res, &(*iter)->dbterm)) + { + object_exists = 1; + break; + } + + if (!object_exists) { + next = (*current)->next; + hval = (*current)->hvalue; + new = replace_dbterm(tb, *current, match_res); + new->next = next; + new->hvalue = hval; + *current = new; + ++got; + } + } + --num_left; + current = &((*current)->next); + } + else { + match_res = db_match_dbterm(&tb->common, p, mp, 0, + &(*current)->dbterm, NULL, 0); + if (is_value(match_res) && + is_value(key = db_getkey(tb->common.keypos, match_res)) && + eq(key, GETKEY(tb, (*current)->dbterm.tpl))) + { + next = (*current)->next; + hval = (*current)->hvalue; + new = replace_dbterm(tb, *current, match_res); + new->next = next; + new->hvalue = hval; + *current = new; + ++got; + } + --num_left; + current = &((*current)->next); + } + } +done: + // the more objects we've replaced, the more reductions we've consumed + BUMP_REDS(p, MIN(2000, (1000 - num_left) + (int)(got - prev_got))); + RET_TO_BIF(erts_make_integer(got,p),DB_ERROR_NONE); +trap: + BUMP_ALL_REDS(p); + if (IS_USMALL(0, got)) { + hp = HAlloc(p, 5); + egot = make_small(got); + } + else { + hp = HAlloc(p, BIG_UINT_HEAP_SIZE + 5); + egot = uint_to_big(got, hp); + hp += BIG_UINT_HEAP_SIZE; + } + continuation = TUPLE4(hp, tb->common.id, make_small(slot_ix), + tptr[3], + egot); + RET_TO_BIF(bif_trap1(&ets_select_replace_continue_exp, p, + continuation), + DB_ERROR_NONE); + +#undef RET_TO_BIF + +} + /* ** Other interface routines (not directly coupled to one bif) */ diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 14a7451267..af1296367a 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -76,9 +76,18 @@ ((Dtt->pos) ? \ (Dtt)->array[(Dtt)->pos - 1] : NULL) -#define EMPTY_NODE(Dtt) (TOP_NODE(Dtt) == NULL) +#define TOPN_NODE(Dtt, Pos) \ + (((Pos) < Dtt->pos) ? \ + (Dtt)->array[(Dtt)->pos - ((Pos) + 1)] : NULL) + +#define REPLACE_TOP_NODE(Dtt, Node) \ + if ((Dtt)->pos) (Dtt)->array[(Dtt)->pos - 1] = (Node) +#define EMPTY_NODE(Dtt) (TOP_NODE(Dtt) == NULL) +#ifndef MIN +#define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) +#endif /* Obtain table static stack if available. NULL if not. ** Must be released with release_stack() @@ -225,9 +234,9 @@ struct mp_info { Eterm most; /* The highest matching key (possibly * partially bound expression) */ - TreeDbTerm *save_term; /* If the key is completely bound, this - * will be the Tree node we're searching - * for, otherwise it will be useless */ + TreeDbTerm **save_term; /* If the key is completely bound, this + * will be the Tree node we're searching + * for, otherwise it will be useless */ Binary *mp; /* The compiled match program */ }; @@ -276,6 +285,21 @@ struct select_delete_context { int keypos; }; +/* + * Used by doit_select_replace + */ +struct select_replace_context { + Process *p; + DbTableTree *tb; + Binary *mp; + Eterm end_condition; + Eterm *lastobj; + Sint32 max; + int keypos; + int all_objects; + Sint replaced; +}; + /* ** Forward declarations */ @@ -290,6 +314,7 @@ static int delsub(TreeDbTerm **this); static TreeDbTerm *slot_search(Process *p, DbTableTree *tb, Sint slot); static TreeDbTerm *find_node(DbTableTree *tb, Eterm key); static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key); +static TreeDbTerm **find_ptr(DbTableTree *tb, DbTreeStack*, TreeDbTerm *this); static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack*, Eterm key); static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack*, Eterm key); static TreeDbTerm *find_next_from_pb_key(DbTableTree *tb, DbTreeStack*, @@ -311,8 +336,16 @@ static void traverse_forward(DbTableTree *tb, TreeDbTerm *, void *, int), - void *context); -static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret, + void *context); +static void traverse_update_backwards(DbTableTree *tb, + DbTreeStack*, + Eterm lastkey, + int (*doit)(DbTableTree *tb, + TreeDbTerm **, // out + void *, + int), + void *context); +static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm ***ret, Eterm *partly_bound_key); static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key); static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done); @@ -335,6 +368,10 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr, int forward); +static int doit_select_replace(DbTableTree *tb, + TreeDbTerm **this_ptr, + void *ptr, + int forward); static int partly_bound_can_match_lesser(Eterm partly_bound_1, Eterm partly_bound_2); @@ -383,6 +420,10 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret); static int db_select_delete_continue_tree(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); +static int db_select_replace_tree(Process *p, DbTable *tbl, + Eterm pattern, Eterm *ret); +static int db_select_replace_continue_tree(Process *p, DbTable *tbl, + Eterm continuation, Eterm *ret); static int db_take_tree(Process *, DbTable *, Eterm, Eterm *); static void db_print_tree(fmtfn_t to, void *to_arg, int show, DbTable *tbl); @@ -435,6 +476,8 @@ DbTableMethod db_tree = db_select_delete_continue_tree, db_select_count_tree, db_select_count_continue_tree, + db_select_replace_tree, + db_select_replace_continue_tree, db_take_tree, db_delete_all_objects_tree, db_free_table_tree, @@ -1103,7 +1146,7 @@ static int db_select_tree(Process *p, DbTable *tbl, Eterm tid, if (!mpi.got_partial && mpi.some_limitation && CMP_EQ(mpi.least,mpi.most)) { - doit_select(tb,mpi.save_term,&sc,0 /* direction doesn't matter */); + doit_select(tb,*(mpi.save_term),&sc,0 /* direction doesn't matter */); RET_TO_BIF(sc.accum,DB_ERROR_NONE); } @@ -1306,7 +1349,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, Eterm tid, if (!mpi.got_partial && mpi.some_limitation && CMP_EQ(mpi.least,mpi.most)) { - doit_select_count(tb,mpi.save_term,&sc,0 /* dummy */); + doit_select_count(tb,*(mpi.save_term),&sc,0 /* dummy */); RET_TO_BIF(erts_make_integer(sc.got,p),DB_ERROR_NONE); } @@ -1409,7 +1452,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, Eterm tid, if (!mpi.got_partial && mpi.some_limitation && CMP_EQ(mpi.least,mpi.most)) { - doit_select(tb,mpi.save_term,&sc, 0 /* direction doesn't matter */); + doit_select(tb,*(mpi.save_term),&sc, 0 /* direction doesn't matter */); if (sc.accum != NIL) { hp=HAlloc(p, 3); RET_TO_BIF(TUPLE2(hp,sc.accum,am_EOT),DB_ERROR_NONE); @@ -1650,7 +1693,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, Eterm tid, if (!mpi.got_partial && mpi.some_limitation && CMP_EQ(mpi.least,mpi.most)) { - doit_select_delete(tb,mpi.save_term,&sc, 0 /* direction doesn't + doit_select_delete(tb,*(mpi.save_term),&sc, 0 /* direction doesn't matter */); RET_TO_BIF(erts_make_integer(sc.accum,p),DB_ERROR_NONE); } @@ -1702,6 +1745,212 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, Eterm tid, } +static int db_select_replace_continue_tree(Process *p, + DbTable *tbl, + Eterm continuation, + Eterm *ret) +{ + DbTableTree *tb = &tbl->tree; + DbTreeStack* stack; + struct select_replace_context sc; + unsigned sz; + Eterm *hp; + Eterm lastkey; + Eterm end_condition; + Binary *mp; + Eterm key; + Eterm *tptr; + Eterm ereplaced; + Sint prev_replaced; + + +#define RET_TO_BIF(Term, State) do { *ret = (Term); return State; } while(0); + + /* Decode continuation. We know it's a tuple and everything else as + this is only called by ourselves */ + + /* continuation: + {Table, Lastkey, EndCondition, MatchProgBin, HowManyReplaced}*/ + + tptr = tuple_val(continuation); + + if (arityval(*tptr) != 5) + erts_exit(ERTS_ERROR_EXIT,"Internal error in ets:select_replace/1"); + + lastkey = tptr[2]; + end_condition = tptr[3]; + if (!(thing_subtag(*binary_val(tptr[4])) == REFC_BINARY_SUBTAG)) + RET_TO_BIF(NIL,DB_ERROR_BADPARAM); + mp = ((ProcBin *) binary_val(tptr[4]))->val; + if (!IsMatchProgBinary(mp)) + RET_TO_BIF(NIL,DB_ERROR_BADPARAM); + + sc.p = p; + sc.mp = mp; + sc.end_condition = NIL; + sc.lastobj = NULL; + sc.max = 1000; + sc.keypos = tb->common.keypos; + if (is_big(tptr[5])) { + sc.replaced = big_to_uint32(tptr[5]); + } else { + sc.replaced = unsigned_val(tptr[5]); + } + prev_replaced = sc.replaced; + + stack = get_any_stack(tb); + traverse_update_backwards(tb, stack, lastkey, &doit_select_replace, &sc); + release_stack(tb,stack); + + // the more objects we've replaced, the more reductions we've consumed + BUMP_REDS(p, MIN(2000, (1000 - sc.max) + (sc.replaced - prev_replaced))); + + if (sc.max > 0) { + RET_TO_BIF(erts_make_integer(sc.replaced,p), DB_ERROR_NONE); + } + key = GETKEY(tb, sc.lastobj); + if (end_condition != NIL && + (cmp_partly_bound(end_condition,key) > 0)) { + /* done anyway */ + RET_TO_BIF(make_small(sc.replaced),DB_ERROR_NONE); + } + /* Not done yet, let's trap. */ + sz = size_object(key); + if (IS_USMALL(0, sc.replaced)) { + hp = HAlloc(p, sz + 6); + ereplaced = make_small(sc.replaced); + } + else { + hp = HAlloc(p, BIG_UINT_HEAP_SIZE + sz + 6); + ereplaced = uint_to_big(sc.replaced, hp); + hp += BIG_UINT_HEAP_SIZE; + } + key = copy_struct(key, sz, &hp, &MSO(p)); + continuation = TUPLE5 + (hp, + tptr[1], + key, + tptr[3], + tptr[4], + ereplaced); + RET_TO_BIF(bif_trap1(&ets_select_replace_continue_exp, p, continuation), + DB_ERROR_NONE); + +#undef RET_TO_BIF +} + +static int db_select_replace_tree(Process *p, DbTable *tbl, + Eterm pattern, Eterm *ret) +{ + DbTableTree *tb = &tbl->tree; + DbTreeStack* stack; + struct select_replace_context sc; + struct mp_info mpi; + Eterm lastkey = THE_NON_VALUE; + Eterm key; + Eterm continuation; + unsigned sz; + Eterm *hp; + TreeDbTerm *this; + int errcode; + Eterm ereplaced; + Eterm mpb; + + +#define RET_TO_BIF(Term,RetVal) do { \ + if (mpi.mp != NULL) { \ + erts_bin_free(mpi.mp); \ + } \ + *ret = (Term); \ + return RetVal; \ + } while(0) + + mpi.mp = NULL; + + sc.lastobj = NULL; + sc.p = p; + sc.tb = tb; + sc.max = 1000; + sc.end_condition = NIL; + sc.keypos = tb->common.keypos; + sc.replaced = 0; + + if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) { + RET_TO_BIF(NIL,errcode); + } + + if (!mpi.something_can_match) { + RET_TO_BIF(make_small(0),DB_ERROR_NONE); + /* can't possibly match anything */ + } + + sc.mp = mpi.mp; + sc.all_objects = mpi.all_objects; + + stack = get_static_stack(tb); + if (!mpi.got_partial && mpi.some_limitation && + CMP_EQ(mpi.least,mpi.most)) { + TreeDbTerm* term = *(mpi.save_term); + doit_select_replace(tb,mpi.save_term,&sc,0 /* dummy */); + if (stack != NULL) { + if (TOP_NODE(stack) == term) + // throw away potentially invalid reference + REPLACE_TOP_NODE(stack, *(mpi.save_term)); + release_stack(tb, stack); + } + RET_TO_BIF(erts_make_integer(sc.replaced,p),DB_ERROR_NONE); + } + + if (stack == NULL) + stack = get_any_stack(tb); + + if (mpi.some_limitation) { + if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) { + lastkey = GETKEY(tb, this->dbterm.tpl); + } + sc.end_condition = mpi.least; + } + + traverse_update_backwards(tb, stack, lastkey, &doit_select_replace, &sc); + release_stack(tb,stack); + // the more objects we've replaced, the more reductions we've consumed + BUMP_REDS(p, MIN(2000, (1000 - sc.max) + sc.replaced)); + if (sc.max > 0) { + RET_TO_BIF(erts_make_integer(sc.replaced,p),DB_ERROR_NONE); + } + + key = GETKEY(tb, sc.lastobj); + sz = size_object(key); + if (IS_USMALL(0, sc.replaced)) { + hp = HAlloc(p, sz + PROC_BIN_SIZE + 6); + ereplaced = make_small(sc.replaced); + } + else { + hp = HAlloc(p, BIG_UINT_HEAP_SIZE + sz + PROC_BIN_SIZE + 6); + ereplaced = uint_to_big(sc.replaced, hp); + hp += BIG_UINT_HEAP_SIZE; + } + key = copy_struct(key, sz, &hp, &MSO(p)); + if (mpi.all_objects) + (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; + mpb = db_make_mp_binary(p,mpi.mp,&hp); + + continuation = TUPLE5 + (hp, + tb->common.id, + key, + sc.end_condition, /* From the match program, needn't be copied */ + mpb, + ereplaced); + + /* Don't free mpi.mp, so don't use macro */ + *ret = bif_trap1(&ets_select_replace_continue_exp, p, continuation); + return DB_ERROR_NONE; + +#undef RET_TO_BIF + +} + static int db_take_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) { DbTableTree *tb = &tbl->tree; @@ -2011,7 +2260,7 @@ static int analyze_pattern(DbTableTree *tb, Eterm pattern, ++i; partly_bound = NIL; - res = key_given(tb, tpl, &mpi->save_term, &partly_bound); + res = key_given(tb, tpl, &(mpi->save_term), &partly_bound); if ( res >= 0 ) { /* Can match something */ key = 0; mpi->something_can_match = 1; @@ -2514,6 +2763,58 @@ static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key) return this; } +/* + * Find node and return the address of the node pointer (NULL if not found) + * Tries to reuse the existing stack for performance. + */ + +static TreeDbTerm **find_ptr(DbTableTree *tb, DbTreeStack *stack, TreeDbTerm *this) { + Eterm key = GETKEY(tb, this->dbterm.tpl); + TreeDbTerm *tmp; + TreeDbTerm *parent; + Sint c; + + if(( tmp = TOP_NODE(stack)) != NULL) { + if (!cmp_key_eq(tb,key,tmp)) { + /* Start from the beginning */ + stack->pos = stack->slot = 0; + } + } + if (EMPTY_NODE(stack)) { /* Have to rebuild the stack */ + if (( tmp = tb->root ) == NULL) + return NULL; + for (;;) { + PUSH_NODE(stack, tmp); + if (( c = cmp_key(tb,key,tmp) ) < 0) { + if (tmp->left == NULL) /* We are at the next + and the element does + not exist */ + break; + else + tmp = tmp->left; + } else if (c > 0) { + if (tmp->right == NULL) /* Done */ + return NULL; + else + tmp = tmp->right; + } else + break; + } + } + + if (TOP_NODE(stack) != this) + return NULL; + + parent = TOPN_NODE(stack, 1); + if (parent == NULL) + return ((this != tb->root) ? NULL : &(tb->root)); + if (parent->left == this) + return &(parent->left); + if (parent->right == this) + return &(parent->right); + return NULL; +} + static int db_lookup_dbterm_tree(Process *p, DbTable *tbl, Eterm key, Eterm obj, DbUpdateHandle* handle) @@ -2654,14 +2955,61 @@ static void traverse_forward(DbTableTree *tb, } } +/* + * Traverse the tree with an update callback function, used by db_select_replace + */ +static void traverse_update_backwards(DbTableTree *tb, + DbTreeStack* stack, + Eterm lastkey, + int (*doit)(DbTableTree*, + TreeDbTerm**, + void*, + int), + void* context) +{ + int res; + TreeDbTerm *this, *next, **this_ptr; + + if (lastkey == THE_NON_VALUE) { + stack->pos = stack->slot = 0; + if (( this = tb->root ) == NULL) { + return; + } + while (this != NULL) { + PUSH_NODE(stack, this); + this = this->right; + } + this = TOP_NODE(stack); + this_ptr = find_ptr(tb, stack, this); + ASSERT(this_ptr != NULL); + res = (*doit)(tb, this_ptr, context, 0); + REPLACE_TOP_NODE(stack, *this_ptr); + next = find_prev(tb, stack, GETKEY(tb, (*this_ptr)->dbterm.tpl)); + if (!res) + return; + } else { + next = find_prev(tb, stack, lastkey); + } + + while ((this = next) != NULL) { + this_ptr = find_ptr(tb, stack, this); + ASSERT(this_ptr != NULL); + res = (*doit)(tb, this_ptr, context, 0); + REPLACE_TOP_NODE(stack, *this_ptr); + next = find_prev(tb, stack, GETKEY(tb, (*this_ptr)->dbterm.tpl)); + if (!res) + return; + } +} + /* * Returns 0 if not given 1 if given and -1 on no possible match * if key is given; *ret is set to point to the object concerned. */ -static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret, +static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm ***ret, Eterm *partly_bound) { - TreeDbTerm *this; + TreeDbTerm **this; Eterm key; ASSERT(ret != NULL); @@ -2671,7 +3019,7 @@ static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret, if (is_non_value(key)) return -1; /* can't possibly match anything */ if (!db_has_variable(key)) { /* Bound key */ - if (( this = find_node(tb, key) ) == NULL) { + if (( this = find_node2(tb, key) ) == NULL) { return -1; } *ret = this; @@ -3094,6 +3442,37 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr, return 1; } +static int doit_select_replace(DbTableTree *tb, TreeDbTerm **this, void *ptr, + int forward) +{ + struct select_replace_context *sc = (struct select_replace_context *) ptr; + Eterm ret, key; + + sc->lastobj = (*this)->dbterm.tpl; + + /* Always backwards traversing */ + if (sc->end_condition != NIL && + (cmp_partly_bound(sc->end_condition, + GETKEY_WITH_POS(sc->keypos, (*this)->dbterm.tpl)) > 0)) { + return 0; + } + ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0, + &(*this)->dbterm, NULL, 0); + + if (is_value(ret) && + is_value(key = db_getkey(tb->common.keypos, ret)) && + (cmp_key(tb, key, *this) == 0)) + { + *this = replace_dbterm(tb, *this, ret); + sc->lastobj = (*this)->dbterm.tpl; + ++(sc->replaced); + } + if (--(sc->max) <= 0) { + return 0; + } + return 1; +} + #ifdef TREE_DEBUG static void do_dump_tree2(DbTableTree* tb, int to, void *to_arg, int show, TreeDbTerm *t, int offset) diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 72298842d7..89ef79d2dc 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -169,6 +169,14 @@ typedef struct db_table_method DbTable* tb, /* [in out] */ Eterm continuation, Eterm* ret); + int (*db_select_replace)(Process* p, + DbTable* tb, /* [in out] */ + Eterm pattern, + Eterm* ret); + int (*db_select_replace_continue)(Process* p, + DbTable* tb, /* [in out] */ + Eterm continuation, + Eterm* ret); int (*db_take)(Process *, DbTable *, Eterm, Eterm *); int (*db_delete_all_objects)(Process* p, -- cgit v1.2.3 From e15319fcdb5c99514cd63d7a02d04c97587e8853 Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Mon, 27 Jun 2016 22:09:37 +0100 Subject: Disable ets:select_replace/2 for bags The existing implementation presented both semantic inconsistencies and performance issues. --- erts/emulator/beam/erl_db.c | 8 +++++ erts/emulator/beam/erl_db_hash.c | 74 +++------------------------------------- 2 files changed, 12 insertions(+), 70 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 81fe79a92e..d77c585628 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -2982,6 +2982,14 @@ BIF_RETTYPE ets_select_replace_2(BIF_ALIST_2) if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) { BIF_ERROR(BIF_P, BADARG); } + + if (tb->common.status & DB_BAG) { + /* Bag implementation presented both semantic consistency + and performance issues */ + db_unlock(tb, LCK_WRITE_REC); + BIF_ERROR(BIF_P, BADARG); + } + safety = ITERATION_SAFETY(BIF_P,tb); if (safety == ITER_UNSAFE) { local_fix_table(tb); diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index a4ca6dce77..96d74bb050 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -2009,7 +2009,7 @@ static int db_select_replace_hash(Process *p, DbTableHash *tb = &tbl->hash; struct mp_info mpi; Uint slot_ix = 0; - HashDbTerm **current = NULL, **initial = NULL, **iter = NULL; + HashDbTerm **current = NULL; HashDbTerm *new = NULL, *next = NULL; HashValue hval = INVALID_HASH; unsigned current_list_pos = 0; @@ -2036,6 +2036,8 @@ static int db_select_replace_hash(Process *p, return RetVal; \ } while(0) + /* Bag implementation presented both semantic consistency and performance issues */ + ASSERT(!(tb->common.status & DB_BAG)); if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) { RET_TO_BIF(NIL,errcode); @@ -2060,7 +2062,6 @@ static int db_select_replace_hash(Process *p, } - initial = current; for(;;) { if ((*current) == NULL) { if (mpi.key_given) { /* Key is bound */ @@ -2084,43 +2085,10 @@ static int db_select_replace_hash(Process *p, } current = &BUCKET(tb,slot_ix); } - initial = current; } else if ((*current)->hvalue == INVALID_HASH) { current = &((*current)->next); } - else if (tb->common.status & DB_BAG) { - match_res = db_match_dbterm(&tb->common, p, mpi.mp, 0, - &(*current)->dbterm, NULL, 0); - if (is_value(match_res) && - is_value(key = db_getkey(tb->common.keypos, match_res)) && - eq(key, GETKEY(tb, (*current)->dbterm.tpl))) - { - // we need to make sure we don't end up with duplicate objects; - // it's quite inefficient - int object_exists = 0; - for (iter = initial; *iter != NULL; iter = &((*iter)->next)) - if (((*iter)->hvalue != INVALID_HASH) - && (*iter != *current) - && db_eq(&tb->common, match_res, &(*iter)->dbterm)) - { - object_exists = 1; - break; - } - - if (!object_exists) { - next = (*current)->next; - hval = (*current)->hvalue; - new = replace_dbterm(tb, *current, match_res); - new->next = next; - new->hvalue = hval; - *current = new; - ++got; - } - } - --num_left; - current = &((*current)->next); - } else { match_res = db_match_dbterm(&tb->common, p, mpi.mp, 0, &(*current)->dbterm, NULL, 0); @@ -2178,7 +2146,7 @@ static int db_select_replace_continue_hash(Process *p, { DbTableHash *tb = &tbl->hash; Uint slot_ix; - HashDbTerm **current = NULL, **initial = NULL, **iter = NULL; + HashDbTerm **current = NULL; HashDbTerm *new = NULL, *next = NULL; HashValue hval = INVALID_HASH; Eterm key; @@ -2213,7 +2181,6 @@ static int db_select_replace_continue_hash(Process *p, goto done; } current = &BUCKET(tb,slot_ix); - initial = current; for(;;) { if ((*current) == NULL) { @@ -2225,43 +2192,10 @@ static int db_select_replace_continue_hash(Process *p, goto trap; } current = &BUCKET(tb,slot_ix); - initial = current; } else if ((*current)->hvalue == INVALID_HASH) { current = &((*current)->next); } - else if (tb->common.status & DB_BAG) { - match_res = db_match_dbterm(&tb->common, p, mp, 0, - &(*current)->dbterm, NULL, 0); - if (is_value(match_res) && - is_value(key = db_getkey(tb->common.keypos, match_res)) && - eq(key, GETKEY(tb, (*current)->dbterm.tpl))) - { - // we need to make sure we don't end up with duplicate objects; - // it's quite inefficient - int object_exists = 0; - for (iter = initial; *iter != NULL; iter = &((*iter)->next)) - if (((*iter)->hvalue != INVALID_HASH) - && (*iter != *current) - && db_eq(&tb->common, match_res, &(*iter)->dbterm)) - { - object_exists = 1; - break; - } - - if (!object_exists) { - next = (*current)->next; - hval = (*current)->hvalue; - new = replace_dbterm(tb, *current, match_res); - new->next = next; - new->hvalue = hval; - *current = new; - ++got; - } - } - --num_left; - current = &((*current)->next); - } else { match_res = db_match_dbterm(&tb->common, p, mp, 0, &(*current)->dbterm, NULL, 0); -- cgit v1.2.3 From 3323f01156a698e2d21c2bb1c51b58cf47444aa0 Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Sun, 29 Jan 2017 20:20:34 +0000 Subject: Deduplicate select* code on ETS hash tables Refactor existing solution into a common iteration loop parameterized using stateful callbacks. --- erts/emulator/beam/erl_db_hash.c | 1992 +++++++++++++++++++------------------- erts/emulator/beam/erl_db_tree.c | 3 +- 2 files changed, 1015 insertions(+), 980 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 96d74bb050..1b5c8fe422 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -355,8 +355,6 @@ static ERTS_INLINE void SET_SEGTAB(DbTableHash* tb, erts_smp_atomic_set_nob(&tb->segtab, (erts_aint_t) segtab); } - - /* ** Forward decl's (static functions) */ @@ -401,23 +399,21 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl, Eterm tid, int reverse, Eterm *ret); static int db_select_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, int reverse, Eterm *ret); -static int db_select_count_hash(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, Eterm *ret); -static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, Eterm *ret); - -static int db_select_replace_hash(Process *p, DbTable *tbl, - Eterm pattern, Eterm *ret); - -static int db_select_continue_hash(Process *p, DbTable *tbl, +static int db_select_continue_hash(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); +static int db_select_count_hash(Process *p, DbTable *tbl, Eterm tid, + Eterm pattern, Eterm *ret); static int db_select_count_continue_hash(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); +static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid, + Eterm pattern, Eterm *ret); static int db_select_delete_continue_hash(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); +static int db_select_replace_hash(Process *p, DbTable *tbl, + Eterm pattern, Eterm *ret); static int db_select_replace_continue_hash(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); @@ -1151,816 +1147,1093 @@ static BIF_RETTYPE bif_trap1(Export *bif, { BIF_TRAP1(bif, p, p1); } - -/* - * Continue collecting select matches, this may happen either due to a trap - * or when the user calls ets:select/1 - */ -static int db_select_continue_hash(Process *p, - DbTable *tbl, - Eterm continuation, - Eterm *ret) -{ - DbTableHash *tb = &tbl->hash; - Sint slot_ix; - Sint save_slot_ix; - Sint chunk_size; - int all_objects; - Binary *mp; - int num_left = 1000; - HashDbTerm *current = 0; - Eterm match_list; - Eterm *hp; - Eterm match_res; - Sint got; - Eterm *tptr; - erts_smp_rwmtx_t* lck; - -#define RET_TO_BIF(Term, State) do { *ret = (Term); return State; } while(0); - - /* Decode continuation. We know it's a tuple but not the arity or anything else */ - - tptr = tuple_val(continuation); - - if (arityval(*tptr) != 6) - RET_TO_BIF(NIL,DB_ERROR_BADPARAM); - - if (!is_small(tptr[2]) || !is_small(tptr[3]) || - !(is_list(tptr[5]) || tptr[5] == NIL) || !is_small(tptr[6])) - RET_TO_BIF(NIL,DB_ERROR_BADPARAM); - if ((chunk_size = signed_val(tptr[3])) < 0) - RET_TO_BIF(NIL,DB_ERROR_BADPARAM); - mp = erts_db_get_match_prog_binary(tptr[4]); - if (!mp) - RET_TO_BIF(NIL,DB_ERROR_BADPARAM); - all_objects = mp->flags & BIN_FLAG_ALL_OBJECTS; - match_list = tptr[5]; - if ((got = signed_val(tptr[6])) < 0) - RET_TO_BIF(NIL,DB_ERROR_BADPARAM); - - slot_ix = signed_val(tptr[2]); - if (slot_ix < 0 /* EOT */ - || (chunk_size && got >= chunk_size)) { - goto done; /* Already got all or enough in the match_list */ - } - - lck = RLOCK_HASH(tb,slot_ix); - if (slot_ix >= NACTIVE(tb)) { - RUNLOCK_HASH(lck); - RET_TO_BIF(NIL,DB_ERROR_BADPARAM); - } - - while ((current = BUCKET(tb,slot_ix)) == NULL) { - slot_ix = next_slot(tb, slot_ix, &lck); - if (slot_ix == 0) { - slot_ix = -1; /* EOT */ - goto done; - } - } - for(;;) { - if (current->hvalue != INVALID_HASH && - (match_res = db_match_dbterm(&tb->common, p, mp, all_objects, - ¤t->dbterm, &hp, 2), - is_value(match_res))) { - - match_list = CONS(hp, match_res, match_list); - ++got; - } - - --num_left; - save_slot_ix = slot_ix; - if ((current = next(tb, (Uint*)&slot_ix, &lck, current)) == NULL) { - slot_ix = -1; /* EOT */ - break; - } - if (slot_ix != save_slot_ix) { - if (chunk_size && got >= chunk_size) { - RUNLOCK_HASH(lck); - break; - } - if (num_left <= 0 || MBUF(p)) { - /* - * We have either reached our limit, or just created some heap fragments. - * Since many heap fragments will make the GC slower, trap and GC now. - */ - RUNLOCK_HASH(lck); - goto trap; - } - } - } -done: - BUMP_REDS(p, 1000 - num_left); - if (chunk_size) { - Eterm continuation; - Eterm rest = NIL; - Sint rest_size = 0; - - if (got > chunk_size) { /* Cannot write destructively here, - the list may have - been in user space */ - rest = NIL; - hp = HAlloc(p, (got - chunk_size) * 2); - while (got-- > chunk_size) { - rest = CONS(hp, CAR(list_val(match_list)), rest); - hp += 2; - match_list = CDR(list_val(match_list)); - ++rest_size; - } - } - if (rest != NIL || slot_ix >= 0) { - hp = HAlloc(p,3+7); - continuation = TUPLE6(hp, tptr[1], make_small(slot_ix), - tptr[3], tptr[4], rest, - make_small(rest_size)); - hp += 7; - RET_TO_BIF(TUPLE2(hp, match_list, continuation),DB_ERROR_NONE); - } else { - if (match_list != NIL) { - hp = HAlloc(p, 3); - RET_TO_BIF(TUPLE2(hp, match_list, am_EOT),DB_ERROR_NONE); - } else { - RET_TO_BIF(am_EOT, DB_ERROR_NONE); - } - } - } - RET_TO_BIF(match_list,DB_ERROR_NONE); - -trap: - BUMP_ALL_REDS(p); - - hp = HAlloc(p,7); - continuation = TUPLE6(hp, tptr[1], make_small(slot_ix), tptr[3], - tptr[4], match_list, make_small(got)); - RET_TO_BIF(bif_trap1(&ets_select_continue_exp, p, - continuation), - DB_ERROR_NONE); -#undef RET_TO_BIF -} +/* + * Match traversal callbacks + */ -static int db_select_hash(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, int reverse, - Eterm *ret) -{ - return db_select_chunk_hash(p, tbl, tid, pattern, 0, reverse, ret); -} +/* Called when no match is possible. + * context_ptr: Pointer to context + * ret: Pointer to traversal function term return. + * + * Both the direct return value and 'ret' are used as the traversal function return values. + */ +typedef int (*mtraversal_on_nothing_can_match_t)(void* context_ptr, Eterm* ret); + +/* Called for each match result. + * context_ptr: Pointer to context + * slot_ix: Current slot index + * current_ptr_ptr: Triple pointer to either the bucket or the 'next' pointer in the previous element; + * can be (carefully) used to adjust iteration when deleting or replacing elements. + * match_res: The result of running the match program against the current term. + * + * Should return 1 for successful match, 0 otherwise. + */ +typedef int (*mtraversal_on_match_res_t)(void* context_ptr, Sint slot_ix, HashDbTerm*** current_ptr_ptr, + Eterm match_res); + +/* Called when either we've matched enough elements in this cycle or EOT was reached. + * context_ptr: Pointer to context + * slot_ix: Current slot index + * got: How many elements have been matched so far + * iterations_left: Number of intended iterations (down from an initial max.) left in this traversal cycle + * mpp: Double pointer to the compiled match program + * ret: Pointer to traversal function term return. + * + * Both the direct return value and 'ret' are used as the traversal function return values. + * If *mpp is set to NULL, it won't be deallocated (useful for trapping.) + */ +typedef int (*mtraversal_on_loop_ended_t)(void* context_ptr, Sint slot_ix, Sint got, + Sint iterations_left, Binary** mpp, Eterm* ret); + +/* Called when it's time to trap + * context_ptr: Pointer to context + * slot_ix: Current slot index + * got: How many elements have been matched so far + * mpp: Double pointer to the compiled match program + * ret: Pointer to traversal function term return. + * + * Both the direct return value and 'ret' are used as the traversal function return values. + * If *mpp is set to NULL, it won't be deallocated (useful for trapping.) + */ +typedef int (*mtraversal_on_trap_t)(void* context_ptr, Sint slot_ix, Sint got, Binary** mpp, Eterm* ret); -static int db_select_chunk_hash(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, Sint chunk_size, - int reverse, /* not used */ - Eterm *ret) +/* + * Begin hash table match traversal + */ +static int match_traverse(Process* p, DbTableHash* tb, Eterm pattern, + Sint chunk_size, /* If 0, no chunking */ + Sint iterations_left, /* Nr. of iterations left */ + Eterm** hpp, /* Heap */ + int lock_for_write, /* Set to 1 if we're going to delete or + modify existing terms */ + mtraversal_on_nothing_can_match_t on_nothing_can_match, + mtraversal_on_match_res_t on_match_res, + mtraversal_on_loop_ended_t on_loop_ended, + mtraversal_on_trap_t on_trap, + void* context_ptr, /* State for callbacks above */ + Eterm* ret) { - DbTableHash *tb = &tbl->hash; + Sint slot_ix = -1; /* Slot index */ + HashDbTerm** current_ptr = NULL; /* Refers to either the bucket pointer or + * the 'next' pointer in the previous term + */ + HashDbTerm* saved_current = NULL; /* Helper to avoid double skip on match */ struct mp_info mpi; - Sint slot_ix; - HashDbTerm *current = 0; - unsigned current_list_pos = 0; - Eterm match_list; + unsigned current_list_pos = 0; /* Prefound buckets list index */ Eterm match_res; - Eterm *hp; - int num_left = 1000; - Uint got = 0; - Eterm continuation; - int errcode; - Eterm mpb; - erts_smp_rwmtx_t* lck; - - -#define RET_TO_BIF(Term,RetVal) do { \ - if (mpi.mp != NULL) { \ - erts_bin_free(mpi.mp); \ - } \ - if (mpi.lists != mpi.dlists) { \ - erts_free(ERTS_ALC_T_DB_SEL_LIST, \ - (void *) mpi.lists); \ - } \ - *ret = (Term); \ - return RetVal; \ - } while(0) + Sint got = 0; /* Matched terms counter */ + erts_smp_rwmtx_t* lck = NULL; /* Slot lock */ + int ret_value = DB_ERROR_NONE; +#ifdef ERTS_SMP + erts_smp_rwmtx_t* (*lock_hash_function)(DbTableHash*, HashValue) + = (lock_for_write ? WLOCK_HASH : RLOCK_HASH); + void (*unlock_hash_function)(erts_smp_rwmtx_t*) + = (lock_for_write ? WUNLOCK_HASH : RUNLOCK_HASH); +#else + #define lock_hash_function(tb, hval) NULL + #define unlock_hash_function(lck) ((void)lck) +#endif + Sint (*next_slot_function)(DbTableHash*, Uint, erts_smp_rwmtx_t**) + = (lock_for_write ? next_slot_w : next_slot); + *ret = NIL; - if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) { - RET_TO_BIF(NIL,errcode); + if ((ret_value = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) { + goto done; } if (!mpi.something_can_match) { - if (chunk_size) { - RET_TO_BIF(am_EOT, DB_ERROR_NONE); /* We're done */ - } - RET_TO_BIF(NIL, DB_ERROR_NONE); - /* can't possibly match anything */ + /* Can't possibly match anything */ + ret_value = on_nothing_can_match(context_ptr, ret); + goto done; + } + + if (mpi.all_objects) { + mpi.mp->flags |= BIN_FLAG_ALL_OBJECTS; } + /* + * Look for initial slot / bucket + */ if (!mpi.key_given) { - /* Run this code if pattern is variable or GETKEY(pattern) */ - /* is a variable */ - slot_ix = 0; - lck = RLOCK_HASH(tb,slot_ix); - for (;;) { - ASSERT(slot_ix < NACTIVE(tb)); - if ((current = BUCKET(tb,slot_ix)) != NULL) { - break; - } - slot_ix = next_slot(tb,slot_ix,&lck); - if (slot_ix == 0) { - if (chunk_size) { - RET_TO_BIF(am_EOT, DB_ERROR_NONE); /* We're done */ - } - RET_TO_BIF(NIL,DB_ERROR_NONE); - } - } + /* Run this code if pattern is variable or GETKEY(pattern) */ + /* is a variable */ + slot_ix = 0; + lck = lock_hash_function(tb,slot_ix); + for (;;) { + ASSERT(slot_ix < NACTIVE(tb)); + if (*(current_ptr = &BUCKET(tb,slot_ix)) != NULL) { + break; + } + slot_ix = next_slot_function(tb,slot_ix,&lck); + if (slot_ix == 0) { + ret_value = on_loop_ended(context_ptr, slot_ix, got, iterations_left, &mpi.mp, ret); + goto done; + } + } } else { - /* We have at least one */ - slot_ix = mpi.lists[current_list_pos].ix; - lck = RLOCK_HASH(tb, slot_ix); - current = *(mpi.lists[current_list_pos].bucket); - ASSERT(current == BUCKET(tb,slot_ix)); - ++current_list_pos; + /* We have at least one */ + slot_ix = mpi.lists[current_list_pos].ix; + lck = lock_hash_function(tb, slot_ix); + current_ptr = mpi.lists[current_list_pos].bucket; + ASSERT(*current_ptr == BUCKET(tb,slot_ix)); + ++current_list_pos; } - match_list = NIL; - + /* + * Execute traversal cycle + */ for(;;) { - if (current != NULL) { - if (current->hvalue != INVALID_HASH) { - match_res = db_match_dbterm(&tb->common, p, mpi.mp, 0, - ¤t->dbterm, &hp, 2); - if (is_value(match_res)) { - match_list = CONS(hp, match_res, match_list); - ++got; - } - --num_left; - } - current = current->next; - } - else if (mpi.key_given) { /* Key is bound */ - RUNLOCK_HASH(lck); - if (current_list_pos == mpi.num_lists) { - slot_ix = -1; /* EOT */ - goto done; - } else { - slot_ix = mpi.lists[current_list_pos].ix; - lck = RLOCK_HASH(tb, slot_ix); - current = *(mpi.lists[current_list_pos].bucket); - ASSERT(mpi.lists[current_list_pos].bucket == &BUCKET(tb,slot_ix)); - ++current_list_pos; - } - } - else { /* Key is variable */ - - if ((slot_ix=next_slot(tb,slot_ix,&lck)) == 0) { - slot_ix = -1; - break; - } - if (chunk_size && got >= chunk_size) { - RUNLOCK_HASH(lck); - break; - } - if (num_left <= 0 || MBUF(p)) { - /* - * We have either reached our limit, or just created some heap fragments. - * Since many heap fragments will make the GC slower, trap and GC now. - */ - RUNLOCK_HASH(lck); - goto trap; - } - current = BUCKET(tb,slot_ix); + if (*current_ptr != NULL) { + if ((*current_ptr)->hvalue != INVALID_HASH) { + match_res = db_match_dbterm(&tb->common, p, mpi.mp, 0, + &(*current_ptr)->dbterm, hpp, 2); + saved_current = *current_ptr; + if (on_match_res(context_ptr, slot_ix, ¤t_ptr, match_res)) { + ++got; + } + --iterations_left; + if (*current_ptr != saved_current) { + /* Don't advance to next, the callback did it already */ + continue; + } + } + current_ptr = &((*current_ptr)->next); + } + else if (mpi.key_given) { /* Key is bound */ + unlock_hash_function(lck); + if (current_list_pos == mpi.num_lists) { + ret_value = on_loop_ended(context_ptr, slot_ix, got, iterations_left, &mpi.mp, ret); + goto done; + } else { + slot_ix = mpi.lists[current_list_pos].ix; + lck = lock_hash_function(tb, slot_ix); + current_ptr = mpi.lists[current_list_pos].bucket; + ASSERT(mpi.lists[current_list_pos].bucket == &BUCKET(tb,slot_ix)); + ++current_list_pos; + } + } + else { /* Key is variable */ + if ((slot_ix = next_slot_function(tb,slot_ix,&lck)) == 0) { + slot_ix = -1; + break; + } + if (chunk_size && got >= chunk_size) { + unlock_hash_function(lck); + break; + } + if (iterations_left <= 0 || MBUF(p)) { + /* + * We have either reached our limit, or just created some heap fragments. + * Since many heap fragments will make the GC slower, trap and GC now. + */ + unlock_hash_function(lck); + ret_value = on_trap(context_ptr, slot_ix, got, &mpi.mp, ret); + goto done; + } + current_ptr = &BUCKET(tb,slot_ix); } - } -done: - BUMP_REDS(p, 1000 - num_left); - if (chunk_size) { - Eterm continuation; - Eterm rest = NIL; - Sint rest_size = 0; - - if (mpi.all_objects) - (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; - if (got > chunk_size) { /* Split list in return value and 'rest' */ - Eterm tmp = match_list; - rest = match_list; - while (got-- > chunk_size + 1) { - tmp = CDR(list_val(tmp)); - ++rest_size; - } - ++rest_size; - match_list = CDR(list_val(tmp)); - CDR(list_val(tmp)) = NIL; /* Destructive, the list has never - been in 'user space' */ - } - if (rest != NIL || slot_ix >= 0) { /* Need more calls */ - hp = HAlloc(p,3+7+ERTS_MAGIC_REF_THING_SIZE); - mpb = erts_db_make_match_prog_ref(p,(mpi.mp),&hp); - if (mpi.all_objects) - (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; - continuation = TUPLE6(hp, tid, make_small(slot_ix), - make_small(chunk_size), - mpb, rest, - make_small(rest_size)); - mpi.mp = NULL; /*otherwise the return macro will destroy it */ - hp += 7; - RET_TO_BIF(TUPLE2(hp, match_list, continuation),DB_ERROR_NONE); - } else { /* All data is exhausted */ - if (match_list != NIL) { /* No more data to search but still a - result to return to the caller */ - hp = HAlloc(p, 3); - RET_TO_BIF(TUPLE2(hp, match_list, am_EOT),DB_ERROR_NONE); - } else { /* Reached the end of the ttable with no data to return */ - RET_TO_BIF(am_EOT, DB_ERROR_NONE); - } - } - } - RET_TO_BIF(match_list,DB_ERROR_NONE); -trap: - BUMP_ALL_REDS(p); - if (mpi.all_objects) - (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; - hp = HAlloc(p,7+ERTS_MAGIC_REF_THING_SIZE); - mpb = erts_db_make_match_prog_ref(p,(mpi.mp),&hp); - continuation = TUPLE6(hp, tid, make_small(slot_ix), - make_small(chunk_size), - mpb, match_list, - make_small(got)); - mpi.mp = NULL; /*otherwise the return macro will destroy it */ - RET_TO_BIF(bif_trap1(&ets_select_continue_exp, p, - continuation), - DB_ERROR_NONE); - -#undef RET_TO_BIF - -} - -static int db_select_count_hash(Process *p, - DbTable *tbl, - Eterm tid, - Eterm pattern, - Eterm *ret) -{ - DbTableHash *tb = &tbl->hash; - struct mp_info mpi; - Uint slot_ix = 0; - HashDbTerm* current = NULL; - unsigned current_list_pos = 0; - Eterm *hp; - int num_left = 1000; - Uint got = 0; - Eterm continuation; - int errcode; - Eterm egot; - Eterm mpb; - erts_smp_rwmtx_t* lck; - -#define RET_TO_BIF(Term,RetVal) do { \ - if (mpi.mp != NULL) { \ - erts_bin_free(mpi.mp); \ - } \ - if (mpi.lists != mpi.dlists) { \ - erts_free(ERTS_ALC_T_DB_SEL_LIST, \ - (void *) mpi.lists); \ - } \ - *ret = (Term); \ - return RetVal; \ - } while(0) - - - if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) { - RET_TO_BIF(NIL,errcode); - } - - if (!mpi.something_can_match) { - RET_TO_BIF(make_small(0), DB_ERROR_NONE); - /* can't possibly match anything */ } - if (!mpi.key_given) { - /* Run this code if pattern is variable or GETKEY(pattern) */ - /* is a variable */ - slot_ix = 0; - lck = RLOCK_HASH(tb,slot_ix); - current = BUCKET(tb,slot_ix); - } else { - /* We have at least one */ - slot_ix = mpi.lists[current_list_pos].ix; - lck = RLOCK_HASH(tb, slot_ix); - current = *(mpi.lists[current_list_pos].bucket); - ASSERT(current == BUCKET(tb,slot_ix)); - ++current_list_pos; - } + ret_value = on_loop_ended(context_ptr, slot_ix, got, iterations_left, &mpi.mp, ret); - for(;;) { - if (current != NULL) { - if (current->hvalue != INVALID_HASH) { - if (db_match_dbterm(&tb->common, p, mpi.mp, 0, - ¤t->dbterm, NULL,0) == am_true) { - ++got; - } - --num_left; - } - current = current->next; - } - else { /* next bucket */ - if (mpi.key_given) { /* Key is bound */ - RUNLOCK_HASH(lck); - if (current_list_pos == mpi.num_lists) { - goto done; - } else { - slot_ix = mpi.lists[current_list_pos].ix; - lck = RLOCK_HASH(tb, slot_ix); - current = *(mpi.lists[current_list_pos].bucket); - ASSERT(mpi.lists[current_list_pos].bucket == &BUCKET(tb,slot_ix)); - ++current_list_pos; - } - } - else { - if ((slot_ix=next_slot(tb,slot_ix,&lck)) == 0) { - goto done; - } - if (num_left <= 0) { - RUNLOCK_HASH(lck); - goto trap; - } - current = BUCKET(tb,slot_ix); - } - } - } done: - BUMP_REDS(p, 1000 - num_left); - RET_TO_BIF(erts_make_integer(got,p),DB_ERROR_NONE); -trap: - BUMP_ALL_REDS(p); - if (IS_USMALL(0, got)) { - hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE + 5); - egot = make_small(got); + /* We should only jump directly to this label if + * we've already called on_nothing_can_match / on_loop_ended / on_trap + */ + if (mpi.mp != NULL) { + erts_bin_free(mpi.mp); } - else { - hp = HAlloc(p, BIG_UINT_HEAP_SIZE + ERTS_MAGIC_REF_THING_SIZE + 5); - egot = uint_to_big(got, hp); - hp += BIG_UINT_HEAP_SIZE; + if (mpi.lists != mpi.dlists) { + erts_free(ERTS_ALC_T_DB_SEL_LIST, + (void *) mpi.lists); } - mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp); - continuation = TUPLE4(hp, tid, make_small(slot_ix), - mpb, - egot); - mpi.mp = NULL; /*otherwise the return macro will destroy it */ - RET_TO_BIF(bif_trap1(&ets_select_count_continue_exp, p, - continuation), - DB_ERROR_NONE); + return ret_value; -#undef RET_TO_BIF +#ifndef SMP +#undef lock_hash_function +#undef unlock_hash_function +#endif } -static int db_select_delete_hash(Process *p, - DbTable *tbl, - Eterm tid, - Eterm pattern, - Eterm *ret) +/* + * Continue hash table match traversal + */ +static int match_traverse_continue(Process* p, DbTableHash* tb, + Sint chunk_size, /* If 0, no chunking */ + Sint iterations_left, /* Nr. of iterations left */ + Eterm** hpp, /* Heap */ + Sint slot_ix, /* Slot index to resume traversal from */ + Sint got, /* Matched terms counter */ + Binary** mpp, /* Existing match program */ + int lock_for_write, /* Set to 1 if we're going to delete or + modify existing terms */ + mtraversal_on_match_res_t on_match_res, + mtraversal_on_loop_ended_t on_loop_ended, + mtraversal_on_trap_t on_trap, + void* context_ptr, /* For callbacks */ + Eterm* ret) { - DbTableHash *tb = &tbl->hash; - struct mp_info mpi; - Uint slot_ix = 0; - HashDbTerm **current = NULL; - unsigned current_list_pos = 0; - Eterm *hp; - int num_left = 1000; - Uint got = 0; - Eterm continuation; - int errcode; - Uint last_pseudo_delete = (Uint)-1; - Eterm mpb; - Eterm egot; + int all_objects = (*mpp)->flags & BIN_FLAG_ALL_OBJECTS; + HashDbTerm** current_ptr = NULL; /* Refers to either the bucket pointer or + * the 'next' pointer in the previous term + */ + HashDbTerm* saved_current = NULL; /* Helper to avoid double skip on match */ + Eterm match_res = NIL; + erts_smp_rwmtx_t* lck = NULL; + int ret_value = DB_ERROR_NONE; #ifdef ERTS_SMP - erts_aint_t fixated_by_me = tb->common.is_thread_safe ? 0 : 1; /* ToDo: something nicer */ + erts_smp_rwmtx_t* (*lock_hash_function)(DbTableHash*, HashValue) + = (lock_for_write ? WLOCK_HASH : RLOCK_HASH); + void (*unlock_hash_function)(erts_smp_rwmtx_t*) + = (lock_for_write ? WUNLOCK_HASH : RUNLOCK_HASH); #else - erts_aint_t fixated_by_me = 0; + #define lock_hash_function(tb, hval) NULL + #define unlock_hash_function(lck) ((void)lck) #endif - erts_smp_rwmtx_t* lck; + Sint (*next_slot_function)(DbTableHash* tb, Uint ix, erts_smp_rwmtx_t** lck_ptr) + = (lock_for_write ? next_slot_w : next_slot); -#define RET_TO_BIF(Term,RetVal) do { \ - if (mpi.mp != NULL) { \ - erts_bin_free(mpi.mp); \ - } \ - if (mpi.lists != mpi.dlists) { \ - erts_free(ERTS_ALC_T_DB_SEL_LIST, \ - (void *) mpi.lists); \ - } \ - *ret = (Term); \ - return RetVal; \ - } while(0) + *ret = NIL; + if (got < 0) + return DB_ERROR_BADPARAM; - if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) { - RET_TO_BIF(NIL,errcode); + if (slot_ix < 0 /* EOT */ + || (chunk_size && got >= chunk_size)) + { + /* Already got all or enough in the match_list */ + ret_value = on_loop_ended(context_ptr, slot_ix, got, iterations_left, mpp, ret); + goto done; } - if (!mpi.something_can_match) { - RET_TO_BIF(make_small(0), DB_ERROR_NONE); - /* can't possibly match anything */ + lck = lock_hash_function(tb, slot_ix); + if (slot_ix >= NACTIVE(tb)) { /* Is this possible? */ + unlock_hash_function(lck); + ret_value = DB_ERROR_BADPARAM; + goto done; } - if (!mpi.key_given) { - /* Run this code if pattern is variable or GETKEY(pattern) */ - /* is a variable */ - lck = WLOCK_HASH(tb,slot_ix); - current = &BUCKET(tb,slot_ix); - } else { - /* We have at least one */ - slot_ix = mpi.lists[current_list_pos].ix; - lck = WLOCK_HASH(tb, slot_ix); - current = mpi.lists[current_list_pos++].bucket; - ASSERT(*current == BUCKET(tb,slot_ix)); + /* + * Resume traversal cycle from where we left + */ + current_ptr = &BUCKET(tb,slot_ix); + for(;;) { + if (*current_ptr != NULL) { + if ((*current_ptr)->hvalue != INVALID_HASH) { + match_res = db_match_dbterm(&tb->common, p, *mpp, all_objects, + &(*current_ptr)->dbterm, hpp, 2); + saved_current = *current_ptr; + if (on_match_res(context_ptr, slot_ix, ¤t_ptr, match_res)) { + ++got; + } + --iterations_left; + if (*current_ptr != saved_current) { + /* Don't advance to next, the callback did it already */ + continue; + } + } + current_ptr = &((*current_ptr)->next); + } + else { + if ((slot_ix=next_slot_function(tb,slot_ix,&lck)) == 0) { + slot_ix = -1; + break; + } + if (chunk_size && got >= chunk_size) { + unlock_hash_function(lck); + break; + } + if (iterations_left <= 0 || MBUF(p)) { + /* + * We have either reached our limit, or just created some heap fragments. + * Since many heap fragments will make the GC slower, trap and GC now. + */ + unlock_hash_function(lck); + ret_value = on_trap(context_ptr, slot_ix, got, mpp, ret); + goto done; + } + current_ptr = &BUCKET(tb,slot_ix); + } } + ret_value = on_loop_ended(context_ptr, slot_ix, got, iterations_left, mpp, ret); - for(;;) { - if ((*current) == NULL) { - if (mpi.key_given) { /* Key is bound */ - WUNLOCK_HASH(lck); - if (current_list_pos == mpi.num_lists) { - goto done; - } else { - slot_ix = mpi.lists[current_list_pos].ix; - lck = WLOCK_HASH(tb, slot_ix); - current = mpi.lists[current_list_pos].bucket; - ASSERT(mpi.lists[current_list_pos].bucket == &BUCKET(tb,slot_ix)); - ++current_list_pos; - } - } else { - if ((slot_ix=next_slot_w(tb,slot_ix,&lck)) == 0) { - goto done; - } - if (num_left <= 0) { - WUNLOCK_HASH(lck); - goto trap; - } - current = &BUCKET(tb,slot_ix); - } - } - else if ((*current)->hvalue == INVALID_HASH) { - current = &((*current)->next); - } - else { - int did_erase = 0; - if (db_match_dbterm(&tb->common, p, mpi.mp, 0, - &(*current)->dbterm, NULL, 0) == am_true) { - HashDbTerm *del; - if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */ - if (slot_ix != last_pseudo_delete) { - if (!add_fixed_deletion(tb, slot_ix, fixated_by_me)) - goto do_erase; - last_pseudo_delete = slot_ix; - } - (*current)->hvalue = INVALID_HASH; - } else { - do_erase: - del = *current; - *current = (*current)->next; - free_term(tb, del); - did_erase = 1; - } - erts_smp_atomic_dec_nob(&tb->common.nitems); - ++got; - } - --num_left; - if (!did_erase) { - current = &((*current)->next); - } - } - } done: - BUMP_REDS(p, 1000 - num_left); - if (got) { - try_shrink(tb); - } - RET_TO_BIF(erts_make_integer(got,p),DB_ERROR_NONE); -trap: + /* We should only jump directly to this label if + * we've already called on_loop_ended / on_trap + */ + return ret_value; + +#ifndef SMP +#undef lock_hash_function +#undef unlock_hash_function +#endif +} + + +/* + * Common traversal trapping/continuation code; + * used by select_count, select_delete and select_replace, + * as well as their continuation-handling counterparts. + */ + +static ERTS_INLINE int on_mtraversal_simple_trap(Export* trap_function, + Process* p, + DbTableHash* tb, + Eterm tid, + Eterm* prev_continuation_tptr, + Sint slot_ix, + Sint got, + Binary** mpp, + Eterm* ret) +{ + Eterm* hp = NULL; + Eterm egot = NIL; + Eterm mpb = NIL; + Eterm continuation = NIL; + int is_first_trap = (prev_continuation_tptr == NULL); + size_t base_halloc_sz = (is_first_trap ? PROC_BIN_SIZE : 0); + BUMP_ALL_REDS(p); if (IS_USMALL(0, got)) { - hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE + 5); + hp = HAlloc(p, base_halloc_sz + 5); egot = make_small(got); } else { - hp = HAlloc(p, BIG_UINT_HEAP_SIZE + ERTS_MAGIC_REF_THING_SIZE + 5); + hp = HAlloc(p, base_halloc_sz + BIG_UINT_HEAP_SIZE + 5); egot = uint_to_big(got, hp); hp += BIG_UINT_HEAP_SIZE; } - mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp); - continuation = TUPLE4(hp, tid, make_small(slot_ix), - mpb, - egot); - mpi.mp = NULL; /*otherwise the return macro will destroy it */ - RET_TO_BIF(bif_trap1(&ets_select_delete_continue_exp, p, - continuation), - DB_ERROR_NONE); -#undef RET_TO_BIF + if (is_first_trap) { + mpb = db_make_mp_binary(p, *mpp, &hp); + *mpp = NULL; /* otherwise the caller will destroy it */ + } + else { + mpb = prev_continuation_tptr[3]; + } + continuation = TUPLE4( + hp, + tid, + make_small(slot_ix), + mpb, + egot); + *ret = bif_trap1(trap_function, p, continuation); + return DB_ERROR_NONE; } -/* -** This is called when select_delete traps -*/ -static int db_select_delete_continue_hash(Process *p, - DbTable *tbl, - Eterm continuation, - Eterm *ret) + +static ERTS_INLINE int unpack_simple_mtraversal_continuation(Eterm continuation, + Eterm** tptr_ptr, + Eterm* tid_ptr, + Sint* slot_ix_p, + Binary** mpp, + Sint* got_p) { - DbTableHash *tb = &tbl->hash; - Uint slot_ix; - Uint last_pseudo_delete = (Uint)-1; - HashDbTerm **current = NULL; - Eterm *hp; - int num_left = 1000; - Uint got; - Eterm *tptr; - Binary *mp; - Eterm egot; - int fixated_by_me = ONLY_WRITER(p,tb) ? 0 : 1; /* ToDo: something nicer */ - erts_smp_rwmtx_t* lck; + Eterm* tptr = NULL; + ASSERT(is_tuple(continuation)); + tptr = tuple_val(continuation); + if (arityval(*tptr) != 4) + return 1; -#define RET_TO_BIF(Term,RetVal) do { \ - *ret = (Term); \ - return RetVal; \ - } while(0) + if (! is_small(tptr[2]) + || !(is_binary(tptr[3]) && thing_subtag(*binary_val(tptr[3])) == REFC_BINARY_SUBTAG) + || !(is_big(tptr[4]) || is_small(tptr[4]))) + { + return 1; + } - - tptr = tuple_val(continuation); - slot_ix = unsigned_val(tptr[2]); - mp = erts_db_get_match_prog_binary_unchecked(tptr[3]); + *tptr_ptr = tptr; + *tid_ptr = tptr[1]; + *slot_ix_p = unsigned_val(tptr[2]); + *mpp = ((ProcBin *) binary_val(tptr[3]))->val; if (is_big(tptr[4])) { - got = big_to_uint32(tptr[4]); - } else { - got = unsigned_val(tptr[4]); + *got_p = big_to_uint32(tptr[4]); } - - lck = WLOCK_HASH(tb,slot_ix); - if (slot_ix >= NACTIVE(tb)) { - WUNLOCK_HASH(lck); - goto done; - } - current = &BUCKET(tb,slot_ix); - - for(;;) { - if ((*current) == NULL) { - if ((slot_ix=next_slot_w(tb,slot_ix,&lck)) == 0) { - goto done; - } - if (num_left <= 0) { - WUNLOCK_HASH(lck); - goto trap; - } - current = &BUCKET(tb,slot_ix); - } - else if ((*current)->hvalue == INVALID_HASH) { - current = &((*current)->next); - } - else { - int did_erase = 0; - if (db_match_dbterm(&tb->common, p, mp, 0, - &(*current)->dbterm, NULL, 0) == am_true) { - HashDbTerm *del; - if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */ - if (slot_ix != last_pseudo_delete) { - if (!add_fixed_deletion(tb, slot_ix, fixated_by_me)) - goto do_erase; - last_pseudo_delete = slot_ix; - } - (*current)->hvalue = INVALID_HASH; - } else { - do_erase: - del = *current; - *current = (*current)->next; - free_term(tb, del); - did_erase = 1; - } - erts_smp_atomic_dec_nob(&tb->common.nitems); - ++got; - } - - --num_left; - if (!did_erase) { - current = &((*current)->next); - } - } + else { + *got_p = unsigned_val(tptr[4]); } -done: - BUMP_REDS(p, 1000 - num_left); - if (got) { - try_shrink(tb); + return 0; +} + + +/* + * + * select / select_chunk match traversal + * + */ + +#define MAX_SELECT_CHUNK_ITERATIONS 1000 + +typedef struct { + Process* p; + DbTableHash* tb; + Eterm tid; + Eterm* hp; + Sint chunk_size; + Eterm match_list; + Eterm* prev_continuation_tptr; +} mtraversal_select_chunk_context_t; + +static int mtraversal_select_chunk_on_nothing_can_match(void* context_ptr, Eterm* ret) { + mtraversal_select_chunk_context_t* sc_context_ptr = (mtraversal_select_chunk_context_t*) context_ptr; + *ret = (sc_context_ptr->chunk_size > 0 ? am_EOT : NIL); + return DB_ERROR_NONE; +} + +static int mtraversal_select_chunk_on_match_res(void* context_ptr, Sint slot_ix, + HashDbTerm*** current_ptr_ptr, + Eterm match_res) +{ + mtraversal_select_chunk_context_t* sc_context_ptr = (mtraversal_select_chunk_context_t*) context_ptr; + if (is_value(match_res)) { + sc_context_ptr->match_list = CONS(sc_context_ptr->hp, match_res, sc_context_ptr->match_list); + return 1; } - RET_TO_BIF(erts_make_integer(got,p),DB_ERROR_NONE); -trap: - BUMP_ALL_REDS(p); - if (IS_USMALL(0, got)) { - hp = HAlloc(p, 5); - egot = make_small(got); + return 0; +} + +static int mtraversal_select_chunk_on_loop_ended(void* context_ptr, Sint slot_ix, Sint got, + Sint iterations_left, Binary** mpp, Eterm* ret) +{ + mtraversal_select_chunk_context_t* sc_context_ptr = (mtraversal_select_chunk_context_t*) context_ptr; + Eterm mpb = NIL; + + if (iterations_left == MAX_SELECT_CHUNK_ITERATIONS) { + /* We didn't get to iterate a single time, which means EOT */ + ASSERT(sc_context_ptr->match_list == NIL); + *ret = (sc_context_ptr->chunk_size > 0 ? am_EOT : NIL); + return DB_ERROR_NONE; } else { - hp = HAlloc(p, BIG_UINT_HEAP_SIZE + 5); - egot = uint_to_big(got, hp); - hp += BIG_UINT_HEAP_SIZE; + ASSERT(iterations_left < MAX_SELECT_CHUNK_ITERATIONS); + BUMP_REDS(sc_context_ptr->p, MAX_SELECT_CHUNK_ITERATIONS - iterations_left); + if (sc_context_ptr->chunk_size) { + Eterm continuation; + Eterm rest = NIL; + Sint rest_size = 0; + + if (got > sc_context_ptr->chunk_size) { /* Split list in return value and 'rest' */ + Eterm tmp = sc_context_ptr->match_list; + rest = sc_context_ptr->match_list; + while (got-- > sc_context_ptr->chunk_size + 1) { + tmp = CDR(list_val(tmp)); + ++rest_size; + } + ++rest_size; + sc_context_ptr->match_list = CDR(list_val(tmp)); + CDR(list_val(tmp)) = NIL; /* Destructive, the list has never + been in 'user space' */ + } + if (rest != NIL || slot_ix >= 0) { /* Need more calls */ + sc_context_ptr->hp = HAlloc(sc_context_ptr->p, 3 + 7 + PROC_BIN_SIZE); + mpb = db_make_mp_binary(sc_context_ptr->p, *mpp, &sc_context_ptr->hp); + continuation = TUPLE6( + sc_context_ptr->hp, + sc_context_ptr->tid, + make_small(slot_ix), + make_small(sc_context_ptr->chunk_size), + mpb, rest, + make_small(rest_size)); + *mpp = NULL; /* Otherwise the caller will destroy it */ + sc_context_ptr->hp += 7; + *ret = TUPLE2(sc_context_ptr->hp, sc_context_ptr->match_list, continuation); + return DB_ERROR_NONE; + } else { /* All data is exhausted */ + if (sc_context_ptr->match_list != NIL) { /* No more data to search but still a + result to return to the caller */ + sc_context_ptr->hp = HAlloc(sc_context_ptr->p, 3); + *ret = TUPLE2(sc_context_ptr->hp, sc_context_ptr->match_list, am_EOT); + return DB_ERROR_NONE; + } else { /* Reached the end of the ttable with no data to return */ + *ret = am_EOT; + return DB_ERROR_NONE; + } + } + } + *ret = sc_context_ptr->match_list; + return DB_ERROR_NONE; + } +} + +static int mtraversal_select_chunk_on_trap(void* context_ptr, Sint slot_ix, Sint got, + Binary** mpp, Eterm* ret) +{ + mtraversal_select_chunk_context_t* sc_context_ptr = (mtraversal_select_chunk_context_t*) context_ptr; + Eterm mpb = NIL; + Eterm continuation = NIL; + Eterm* hp = NULL; + + BUMP_ALL_REDS(sc_context_ptr->p); + + if (sc_context_ptr->prev_continuation_tptr == NULL) { + /* First time we're trapping */ + hp = HAlloc(sc_context_ptr->p, 7 + PROC_BIN_SIZE); + mpb = db_make_mp_binary(sc_context_ptr->p, *mpp, &hp); + continuation = TUPLE6( + hp, + sc_context_ptr->tid, + make_small(slot_ix), + make_small(sc_context_ptr->chunk_size), + mpb, + sc_context_ptr->match_list, + make_small(got)); + *mpp = NULL; /* otherwise the caller will destroy it */ } - continuation = TUPLE4(hp, tptr[1], make_small(slot_ix), - tptr[3], - egot); - RET_TO_BIF(bif_trap1(&ets_select_delete_continue_exp, p, - continuation), - DB_ERROR_NONE); + else { + /* Not the first time we're trapping; reuse continuation terms */ + hp = HAlloc(sc_context_ptr->p, 7); + continuation = TUPLE6( + hp, + sc_context_ptr->prev_continuation_tptr[1], + make_small(slot_ix), + sc_context_ptr->prev_continuation_tptr[3], + sc_context_ptr->prev_continuation_tptr[4], + sc_context_ptr->match_list, + make_small(got)); + } + *ret = bif_trap1(&ets_select_continue_exp, sc_context_ptr->p, continuation); + return DB_ERROR_NONE; +} -#undef RET_TO_BIF +static int db_select_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, int reverse, Eterm *ret) { + return db_select_chunk_hash(p, tbl, tid, pattern, 0, reverse, ret); +} +static int db_select_chunk_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Sint chunk_size, + int reverse, Eterm *ret) +{ + mtraversal_select_chunk_context_t sc_context = {0}; + sc_context.p = p; + sc_context.tb = &tbl->hash; + sc_context.tid = tid; + sc_context.hp = NULL; + sc_context.chunk_size = chunk_size; + sc_context.match_list = NIL; + sc_context.prev_continuation_tptr = NULL; + + return match_traverse( + sc_context.p, sc_context.tb, pattern, sc_context.chunk_size, + MAX_SELECT_CHUNK_ITERATIONS, + &sc_context.hp, 0, + mtraversal_select_chunk_on_nothing_can_match, + mtraversal_select_chunk_on_match_res, + mtraversal_select_chunk_on_loop_ended, + mtraversal_select_chunk_on_trap, + &sc_context, ret); } - + /* -** This is called when select_count traps -*/ -static int db_select_count_continue_hash(Process *p, - DbTable *tbl, - Eterm continuation, - Eterm *ret) + * + * select_continue match traversal + * + */ + +static int mtraversal_select_chunk_continue_on_loop_ended(void* context_ptr, Sint slot_ix, Sint got, + Sint iterations_left, Binary** mpp, Eterm* ret) { - DbTableHash *tb = &tbl->hash; - Uint slot_ix; - HashDbTerm* current; - Eterm *hp; - int num_left = 1000; - Uint got; - Eterm *tptr; - Binary *mp; - Eterm egot; - erts_smp_rwmtx_t* lck; + mtraversal_select_chunk_context_t* sc_context_ptr = (mtraversal_select_chunk_context_t*) context_ptr; + Eterm continuation = NIL; + Eterm rest = NIL; + Eterm* hp = NULL; + + ASSERT(iterations_left <= MAX_SELECT_CHUNK_ITERATIONS); + BUMP_REDS(sc_context_ptr->p, MAX_SELECT_CHUNK_ITERATIONS - iterations_left); + if (sc_context_ptr->chunk_size) { + Sint rest_size = 0; + if (got > sc_context_ptr->chunk_size) { + /* Cannot write destructively here, + the list may have + been in user space */ + rest = NIL; + hp = HAlloc(sc_context_ptr->p, (got - sc_context_ptr->chunk_size) * 2); + while (got-- > sc_context_ptr->chunk_size) { + rest = CONS(hp, CAR(list_val(sc_context_ptr->match_list)), rest); + hp += 2; + sc_context_ptr->match_list = CDR(list_val(sc_context_ptr->match_list)); + ++rest_size; + } + } + if (rest != NIL || slot_ix >= 0) { + hp = HAlloc(sc_context_ptr->p, 3 + 7); + continuation = TUPLE6( + hp, + sc_context_ptr->prev_continuation_tptr[1], + make_small(slot_ix), + sc_context_ptr->prev_continuation_tptr[3], + sc_context_ptr->prev_continuation_tptr[4], + rest, + make_small(rest_size)); + hp += 7; + *ret = TUPLE2(hp, sc_context_ptr->match_list, continuation); + return DB_ERROR_NONE; + } else { + if (sc_context_ptr->match_list != NIL) { + hp = HAlloc(sc_context_ptr->p, 3); + *ret = TUPLE2(hp, sc_context_ptr->match_list, am_EOT); + return DB_ERROR_NONE; + } else { + *ret = am_EOT; + return DB_ERROR_NONE; + } + } + } + *ret = sc_context_ptr->match_list; + return DB_ERROR_NONE; +} -#define RET_TO_BIF(Term,RetVal) do { \ - *ret = (Term); \ - return RetVal; \ - } while(0) +/* + * This is called when select traps + */ +static int db_select_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret) { + mtraversal_select_chunk_context_t sc_context = {0}; + Eterm* tptr = NULL; + Eterm tid = NIL; + Binary* mp = NULL; + Sint got = 0; + Sint slot_ix = 0; + Sint chunk_size = 0; + Eterm match_list = NIL; + Sint iterations_left = MAX_SELECT_CHUNK_ITERATIONS; + *ret = NIL; - + /* Decode continuation. We know it's a tuple but not the arity or anything else */ + ASSERT(is_tuple(continuation)); tptr = tuple_val(continuation); - slot_ix = unsigned_val(tptr[2]); - mp = erts_db_get_match_prog_binary_unchecked(tptr[3]); - if (is_big(tptr[4])) { - got = big_to_uint32(tptr[4]); - } else { - got = unsigned_val(tptr[4]); - } - - lck = RLOCK_HASH(tb, slot_ix); - if (slot_ix >= NACTIVE(tb)) { /* Is this posible? */ - RUNLOCK_HASH(lck); - goto done; - } - current = BUCKET(tb,slot_ix); - - for(;;) { - if (current != NULL) { - if (current->hvalue == INVALID_HASH) { - current = current->next; - continue; - } - if (db_match_dbterm(&tb->common, p, mp, 0, ¤t->dbterm, - NULL, 0) == am_true) { - ++got; - } - --num_left; - current = current->next; - } - else { /* next bucket */ - if ((slot_ix = next_slot(tb,slot_ix,&lck)) == 0) { - goto done; - } - if (num_left <= 0) { - RUNLOCK_HASH(lck); - goto trap; - } - current = BUCKET(tb,slot_ix); - } + if (arityval(*tptr) != 6) + return DB_ERROR_BADPARAM; + + if (!is_small(tptr[2]) || !is_small(tptr[3]) || !is_binary(tptr[4]) || + !(is_list(tptr[5]) || tptr[5] == NIL) || !is_small(tptr[6])) + return DB_ERROR_BADPARAM; + if ((chunk_size = signed_val(tptr[3])) < 0) + return DB_ERROR_BADPARAM; + if (!(thing_subtag(*binary_val(tptr[4])) == REFC_BINARY_SUBTAG)) + return DB_ERROR_BADPARAM; + + mp = ((ProcBin *) binary_val(tptr[4]))->val; + if (!IsMatchProgBinary(mp)) + return DB_ERROR_BADPARAM; + + if ((got = signed_val(tptr[6])) < 0) + return DB_ERROR_BADPARAM; + + tid = tptr[1]; + slot_ix = signed_val(tptr[2]); + match_list = tptr[5]; + + /* Proceed */ + sc_context.p = p; + sc_context.tb = &tbl->hash; + sc_context.tid = tid; + sc_context.hp = NULL; + sc_context.chunk_size = chunk_size; + sc_context.match_list = match_list; + sc_context.prev_continuation_tptr = tptr; + + return match_traverse_continue( + sc_context.p, sc_context.tb, sc_context.chunk_size, + iterations_left, &sc_context.hp, slot_ix, got, &mp, 0, + mtraversal_select_chunk_on_match_res, /* Reuse callback */ + mtraversal_select_chunk_continue_on_loop_ended, + mtraversal_select_chunk_on_trap, /* Reuse callback */ + &sc_context, ret); +} + +#undef MAX_SELECT_CHUNK_ITERATIONS + + +/* + * + * select_count match traversal + * + */ + +#define MAX_SELECT_COUNT_ITERATIONS 1000 + +typedef struct { + Process* p; + DbTableHash* tb; + Eterm tid; + Eterm* hp; + Eterm* prev_continuation_tptr; +} mtraversal_select_count_context_t; + +static int mtraversal_select_count_on_nothing_can_match(void* context_ptr, Eterm* ret) { + *ret = make_small(0); + return DB_ERROR_NONE; +} + +static int mtraversal_select_count_on_match_res(void* context_ptr, Sint slot_ix, + HashDbTerm*** current_ptr_ptr, + Eterm match_res) +{ + return (match_res == am_true); +} + +static int mtraversal_select_count_on_loop_ended(void* context_ptr, Sint slot_ix, Sint got, + Sint iterations_left, Binary** mpp, Eterm* ret) +{ + mtraversal_select_count_context_t* scnt_context_ptr = (mtraversal_select_count_context_t*) context_ptr; + ASSERT(iterations_left <= MAX_SELECT_COUNT_ITERATIONS); + BUMP_REDS(scnt_context_ptr->p, MAX_SELECT_COUNT_ITERATIONS - iterations_left); + *ret = erts_make_integer(got, scnt_context_ptr->p); + return DB_ERROR_NONE; +} + +static int mtraversal_select_count_on_trap(void* context_ptr, Sint slot_ix, Sint got, + Binary** mpp, Eterm* ret) +{ + mtraversal_select_count_context_t* scnt_context_ptr = (mtraversal_select_count_context_t*) context_ptr; + return on_mtraversal_simple_trap( + &ets_select_count_continue_exp, + scnt_context_ptr->p, + scnt_context_ptr->tb, + scnt_context_ptr->tid, + scnt_context_ptr->prev_continuation_tptr, + slot_ix, got, mpp, ret); +} + +static int db_select_count_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret) { + mtraversal_select_count_context_t scnt_context = {0}; + Sint iterations_left = MAX_SELECT_COUNT_ITERATIONS; + Sint chunk_size = 0; + + scnt_context.p = p; + scnt_context.tb = &tbl->hash; + scnt_context.tid = tid; + scnt_context.hp = NULL; + scnt_context.prev_continuation_tptr = NULL; + + return match_traverse( + scnt_context.p, scnt_context.tb, + pattern, chunk_size, + iterations_left, NULL, 0, + mtraversal_select_count_on_nothing_can_match, + mtraversal_select_count_on_match_res, + mtraversal_select_count_on_loop_ended, + mtraversal_select_count_on_trap, + &scnt_context, ret); +} + +/* + * This is called when select_count traps + */ +static int db_select_count_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret) { + mtraversal_select_count_context_t scnt_context = {0}; + Eterm* tptr = NULL; + Eterm tid = NIL; + Binary* mp = NULL; + Sint got = 0; + Sint slot_ix = 0; + Sint chunk_size = 0; + *ret = NIL; + + if (unpack_simple_mtraversal_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) { + return DB_ERROR_BADPARAM; } -done: - BUMP_REDS(p, 1000 - num_left); - RET_TO_BIF(erts_make_integer(got,p),DB_ERROR_NONE); -trap: - BUMP_ALL_REDS(p); - if (IS_USMALL(0, got)) { - hp = HAlloc(p, 5); - egot = make_small(got); + + scnt_context.p = p; + scnt_context.tb = &tbl->hash; + scnt_context.tid = tid; + scnt_context.hp = NULL; + scnt_context.prev_continuation_tptr = tptr; + + return match_traverse_continue( + scnt_context.p, scnt_context.tb, chunk_size, + MAX_SELECT_COUNT_ITERATIONS, + NULL, slot_ix, got, &mp, 0, + mtraversal_select_count_on_match_res, /* Reuse callback */ + mtraversal_select_count_on_loop_ended, /* Reuse callback */ + mtraversal_select_count_on_trap, /* Reuse callback */ + &scnt_context, ret); +} + +#undef MAX_SELECT_COUNT_ITERATIONS + + +/* + * + * select_delete match traversal + * + */ + +#define MAX_SELECT_DELETE_ITERATIONS 1000 + +typedef struct { + Process* p; + DbTableHash* tb; + Eterm tid; + Eterm* hp; + Eterm* prev_continuation_tptr; + erts_aint_t fixated_by_me; + Uint last_pseudo_delete; +} mtraversal_select_delete_context_t; + +static int mtraversal_select_delete_on_nothing_can_match(void* context_ptr, Eterm* ret) { + *ret = make_small(0); + return DB_ERROR_NONE; +} + +static int mtraversal_select_delete_on_match_res(void* context_ptr, Sint slot_ix, + HashDbTerm*** current_ptr_ptr, + Eterm match_res) +{ + HashDbTerm** current_ptr = *current_ptr_ptr; + mtraversal_select_delete_context_t* sd_context_ptr = (mtraversal_select_delete_context_t*) context_ptr; + HashDbTerm* del = NULL; + if (match_res != am_true) + return 0; + + if (NFIXED(sd_context_ptr->tb) > sd_context_ptr->fixated_by_me) { /* fixated by others? */ + if (slot_ix != sd_context_ptr->last_pseudo_delete) { + if (!add_fixed_deletion(sd_context_ptr->tb, slot_ix, sd_context_ptr->fixated_by_me)) + goto do_erase; + sd_context_ptr->last_pseudo_delete = slot_ix; + } + (*current_ptr)->hvalue = INVALID_HASH; } else { - hp = HAlloc(p, BIG_UINT_HEAP_SIZE + 5); - egot = uint_to_big(got, hp); - hp += BIG_UINT_HEAP_SIZE; + do_erase: + del = *current_ptr; + *current_ptr = (*current_ptr)->next; // replace pointer to term using next + free_term(sd_context_ptr->tb, del); } - continuation = TUPLE4(hp, tptr[1], make_small(slot_ix), - tptr[3], - egot); - RET_TO_BIF(bif_trap1(&ets_select_count_continue_exp, p, - continuation), - DB_ERROR_NONE); + erts_smp_atomic_dec_nob(&sd_context_ptr->tb->common.nitems); -#undef RET_TO_BIF + return 1; +} +static int mtraversal_select_delete_on_loop_ended(void* context_ptr, Sint slot_ix, Sint got, + Sint iterations_left, Binary** mpp, Eterm* ret) +{ + mtraversal_select_delete_context_t* sd_context_ptr = (mtraversal_select_delete_context_t*) context_ptr; + ASSERT(iterations_left <= MAX_SELECT_DELETE_ITERATIONS); + BUMP_REDS(sd_context_ptr->p, MAX_SELECT_DELETE_ITERATIONS - iterations_left); + if (got) { + try_shrink(sd_context_ptr->tb); + } + *ret = erts_make_integer(got, sd_context_ptr->p); + return DB_ERROR_NONE; } - + +static int mtraversal_select_delete_on_trap(void* context_ptr, Sint slot_ix, Sint got, + Binary** mpp, Eterm* ret) +{ + mtraversal_select_delete_context_t* sd_context_ptr = (mtraversal_select_delete_context_t*) context_ptr; + return on_mtraversal_simple_trap( + &ets_select_delete_continue_exp, + sd_context_ptr->p, + sd_context_ptr->tb, + sd_context_ptr->tid, + sd_context_ptr->prev_continuation_tptr, + slot_ix, got, mpp, ret); +} + +static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret) { + mtraversal_select_delete_context_t sd_context = {0}; + Sint chunk_size = 0; + + sd_context.p = p; + sd_context.tb = &tbl->hash; + sd_context.tid = tid; + sd_context.hp = NULL; + sd_context.prev_continuation_tptr = NULL; +#ifdef ERTS_SMP + sd_context.fixated_by_me = sd_context.tb->common.is_thread_safe ? 0 : 1; /* TODO: something nicer */ +#else + sd_context.fixated_by_me = 0; +#endif + sd_context.last_pseudo_delete = (Uint) -1; + + return match_traverse( + sd_context.p, sd_context.tb, pattern, chunk_size, + MAX_SELECT_DELETE_ITERATIONS, NULL, 1, + mtraversal_select_delete_on_nothing_can_match, + mtraversal_select_delete_on_match_res, + mtraversal_select_delete_on_loop_ended, + mtraversal_select_delete_on_trap, + &sd_context, ret); +} + +/* + * This is called when select_delete traps + */ +static int db_select_delete_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret) { + mtraversal_select_delete_context_t sd_context = {0}; + Eterm* tptr = NULL; + Eterm tid = NIL; + Binary* mp = NULL; + Sint got = 0; + Sint slot_ix = 0; + Sint chunk_size = 0; + *ret = NIL; + + if (unpack_simple_mtraversal_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) { + return DB_ERROR_BADPARAM; + } + + sd_context.p = p; + sd_context.tb = &tbl->hash; + sd_context.tid = tid; + sd_context.hp = NULL; + sd_context.prev_continuation_tptr = tptr; + sd_context.fixated_by_me = ONLY_WRITER(p, sd_context.tb) ? 0 : 1; /* TODO: something nicer */ + sd_context.last_pseudo_delete = (Uint) -1; + + return match_traverse_continue( + sd_context.p, sd_context.tb, chunk_size, + MAX_SELECT_DELETE_ITERATIONS, + NULL, slot_ix, got, &mp, 1, + mtraversal_select_delete_on_match_res, /* Reuse callback */ + mtraversal_select_delete_on_loop_ended, /* Reuse callback */ + mtraversal_select_delete_on_trap, /* Reuse callback */ + &sd_context, ret); +} + +#undef MAX_SELECT_DELETE_ITERATIONS + + +/* + * + * select_replace match traversal + * + */ + +#define MAX_SELECT_REPLACE_ITERATIONS 1000 + +typedef struct { + Process* p; + DbTableHash* tb; + Eterm tid; + Eterm* hp; + Eterm* prev_continuation_tptr; +} mtraversal_select_replace_context_t; + +static int mtraversal_select_replace_on_nothing_can_match(void* context_ptr, Eterm* ret) { + *ret = make_small(0); + return DB_ERROR_NONE; +} + +static int mtraversal_select_replace_on_match_res(void* context_ptr, Sint slot_ix, + HashDbTerm*** current_ptr_ptr, + Eterm match_res) +{ + mtraversal_select_replace_context_t* sr_context_ptr = (mtraversal_select_replace_context_t*) context_ptr; + DbTableHash* tb = sr_context_ptr->tb; + Eterm key = NIL; + HashDbTerm* new = NULL; + HashDbTerm* next = NULL; + HashValue hval = INVALID_HASH; + + if (is_value(match_res) && + is_value(key = db_getkey(tb->common.keypos, match_res)) && + eq(key, GETKEY(tb, (**current_ptr_ptr)->dbterm.tpl))) + { + next = (**current_ptr_ptr)->next; + hval = (**current_ptr_ptr)->hvalue; + new = replace_dbterm(tb, **current_ptr_ptr, match_res); + new->next = next; + new->hvalue = hval; + **current_ptr_ptr = new; /* replace 'next' pointer in previous object */ + *current_ptr_ptr = &((**current_ptr_ptr)->next); /* advance to next object */ + return 1; + } + return 0; +} + +static int mtraversal_select_replace_on_loop_ended(void* context_ptr, Sint slot_ix, Sint got, + Sint iterations_left, Binary** mpp, Eterm* ret) +{ + mtraversal_select_replace_context_t* sr_context_ptr = (mtraversal_select_replace_context_t*) context_ptr; + ASSERT(iterations_left <= MAX_SELECT_REPLACE_ITERATIONS); + /* the more objects we've replaced, the more reductions we've consumed */ + BUMP_REDS(sr_context_ptr->p, + MIN(MAX_SELECT_REPLACE_ITERATIONS * 2, + (MAX_SELECT_REPLACE_ITERATIONS - iterations_left) + (int)got)); + *ret = erts_make_integer(got, sr_context_ptr->p); + return DB_ERROR_NONE; +} + +static int mtraversal_select_replace_on_trap(void* context_ptr, Sint slot_ix, Sint got, + Binary** mpp, Eterm* ret) +{ + mtraversal_select_replace_context_t* sr_context_ptr = (mtraversal_select_replace_context_t*) context_ptr; + return on_mtraversal_simple_trap( + &ets_select_replace_continue_exp, + sr_context_ptr->p, + sr_context_ptr->tb, + sr_context_ptr->tid, + sr_context_ptr->prev_continuation_tptr, + slot_ix, got, mpp, ret); +} + +static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm pattern, Eterm *ret) +{ + mtraversal_select_replace_context_t sr_context = {0}; + Sint chunk_size = 0; + + /* Bag implementation presented both semantic consistency and performance issues, + * unsupported for now + */ + ASSERT(!(tb->common.status & DB_BAG)); + + sr_context.p = p; + sr_context.tb = &tbl->hash; + sr_context.tid = NIL; // TODO + sr_context.hp = NULL; + sr_context.prev_continuation_tptr = NULL; + + return match_traverse( + sr_context.p, sr_context.tb, pattern, chunk_size, + MAX_SELECT_REPLACE_ITERATIONS, NULL, 1, + mtraversal_select_replace_on_nothing_can_match, + mtraversal_select_replace_on_match_res, + mtraversal_select_replace_on_loop_ended, + mtraversal_select_replace_on_trap, + &sr_context, ret); +} + +/* + * This is called when select_replace traps + */ +static int db_select_replace_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret) +{ + mtraversal_select_replace_context_t sr_context = {0}; + Eterm* tptr = NULL; + Eterm tid = NIL; + Binary* mp = NULL; + Sint got = 0; + Sint slot_ix = 0; + Sint chunk_size = 0; + *ret = NIL; + + if (unpack_simple_mtraversal_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) { + return DB_ERROR_BADPARAM; + } + + /* Proceed */ + sr_context.p = p; + sr_context.tb = &tbl->hash; + sr_context.tid = tid; + sr_context.hp = NULL; + sr_context.prev_continuation_tptr = tptr; + + return match_traverse_continue( + sr_context.p, sr_context.tb, chunk_size, + MAX_SELECT_REPLACE_ITERATIONS, + NULL, slot_ix, got, &mp, 1, + mtraversal_select_replace_on_match_res, /* Reuse callback */ + mtraversal_select_replace_on_loop_ended, /* Reuse callback */ + mtraversal_select_replace_on_trap, /* Reuse callback */ + &sr_context, ret); +} + + static int db_take_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) { DbTableHash *tb = &tbl->hash; @@ -2001,245 +2274,6 @@ static int db_take_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) return DB_ERROR_NONE; } -static int db_select_replace_hash(Process *p, - DbTable *tbl, - Eterm pattern, - Eterm *ret) -{ - DbTableHash *tb = &tbl->hash; - struct mp_info mpi; - Uint slot_ix = 0; - HashDbTerm **current = NULL; - HashDbTerm *new = NULL, *next = NULL; - HashValue hval = INVALID_HASH; - unsigned current_list_pos = 0; - Eterm key; - Eterm match_res; - Eterm *hp; - int num_left = 1000; - Uint got = 0; - Eterm continuation; - int errcode; - Eterm mpb; - Eterm egot; - erts_smp_rwmtx_t* lck; - -#define RET_TO_BIF(Term,RetVal) do { \ - if (mpi.mp != NULL) { \ - erts_bin_free(mpi.mp); \ - } \ - if (mpi.lists != mpi.dlists) { \ - erts_free(ERTS_ALC_T_DB_SEL_LIST, \ - (void *) mpi.lists); \ - } \ - *ret = (Term); \ - return RetVal; \ - } while(0) - - /* Bag implementation presented both semantic consistency and performance issues */ - ASSERT(!(tb->common.status & DB_BAG)); - - if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) { - RET_TO_BIF(NIL,errcode); - } - - if (!mpi.something_can_match) { - RET_TO_BIF(make_small(0), DB_ERROR_NONE); - /* can't possibly match anything */ - } - - if (!mpi.key_given) { - /* Run this code if pattern is variable or GETKEY(pattern) */ - /* is a variable */ - lck = WLOCK_HASH(tb,slot_ix); - current = &BUCKET(tb,slot_ix); - } else { - /* We have at least one */ - slot_ix = mpi.lists[current_list_pos].ix; - lck = WLOCK_HASH(tb, slot_ix); - current = mpi.lists[current_list_pos++].bucket; - ASSERT(*current == BUCKET(tb,slot_ix)); - } - - - for(;;) { - if ((*current) == NULL) { - if (mpi.key_given) { /* Key is bound */ - WUNLOCK_HASH(lck); - if (current_list_pos == mpi.num_lists) { - goto done; - } else { - slot_ix = mpi.lists[current_list_pos].ix; - lck = WLOCK_HASH(tb, slot_ix); - current = mpi.lists[current_list_pos].bucket; - ASSERT(mpi.lists[current_list_pos].bucket == &BUCKET(tb,slot_ix)); - ++current_list_pos; - } - } else { - if ((slot_ix=next_slot_w(tb,slot_ix,&lck)) == 0) { - goto done; - } - if (num_left <= 0) { - WUNLOCK_HASH(lck); - goto trap; - } - current = &BUCKET(tb,slot_ix); - } - } - else if ((*current)->hvalue == INVALID_HASH) { - current = &((*current)->next); - } - else { - match_res = db_match_dbterm(&tb->common, p, mpi.mp, 0, - &(*current)->dbterm, NULL, 0); - if (is_value(match_res) && - is_value(key = db_getkey(tb->common.keypos, match_res)) && - eq(key, GETKEY(tb, (*current)->dbterm.tpl))) - { - next = (*current)->next; - hval = (*current)->hvalue; - new = replace_dbterm(tb, *current, match_res); - new->next = next; - new->hvalue = hval; - *current = new; - ++got; - } - --num_left; - current = &((*current)->next); - } - } -done: - // the more objects we've replaced, the more reductions we've consumed - BUMP_REDS(p, MIN(2000, (1000 - num_left) + (int)got)); - RET_TO_BIF(erts_make_integer(got,p),DB_ERROR_NONE); -trap: - BUMP_ALL_REDS(p); - if (IS_USMALL(0, got)) { - hp = HAlloc(p, PROC_BIN_SIZE + 5); - egot = make_small(got); - } - else { - hp = HAlloc(p, BIG_UINT_HEAP_SIZE + PROC_BIN_SIZE + 5); - egot = uint_to_big(got, hp); - hp += BIG_UINT_HEAP_SIZE; - } - mpb = db_make_mp_binary(p,mpi.mp,&hp); - continuation = TUPLE4(hp, tb->common.id, make_small(slot_ix), - mpb, - egot); - mpi.mp = NULL; /*otherwise the return macro will destroy it */ - RET_TO_BIF(bif_trap1(&ets_select_replace_continue_exp, p, - continuation), - DB_ERROR_NONE); - -#undef RET_TO_BIF - -} - -/* -** This is called when select_replace traps -*/ -static int db_select_replace_continue_hash(Process *p, - DbTable *tbl, - Eterm continuation, - Eterm *ret) -{ - DbTableHash *tb = &tbl->hash; - Uint slot_ix; - HashDbTerm **current = NULL; - HashDbTerm *new = NULL, *next = NULL; - HashValue hval = INVALID_HASH; - Eterm key; - Eterm match_res; - Eterm *hp; - int num_left = 1000; - Uint got, prev_got; - Eterm *tptr; - Binary *mp; - Eterm egot; - erts_smp_rwmtx_t* lck; - -#define RET_TO_BIF(Term,RetVal) do { \ - *ret = (Term); \ - return RetVal; \ - } while(0) - - - tptr = tuple_val(continuation); - slot_ix = unsigned_val(tptr[2]); - mp = ((ProcBin *) binary_val(tptr[3]))->val; - if (is_big(tptr[4])) { - got = big_to_uint32(tptr[4]); - } else { - got = unsigned_val(tptr[4]); - } - prev_got = got; - - lck = WLOCK_HASH(tb,slot_ix); - if (slot_ix >= NACTIVE(tb)) { - WUNLOCK_HASH(lck); - goto done; - } - current = &BUCKET(tb,slot_ix); - - for(;;) { - if ((*current) == NULL) { - if ((slot_ix=next_slot_w(tb,slot_ix,&lck)) == 0) { - goto done; - } - if (num_left <= 0) { - WUNLOCK_HASH(lck); - goto trap; - } - current = &BUCKET(tb,slot_ix); - } - else if ((*current)->hvalue == INVALID_HASH) { - current = &((*current)->next); - } - else { - match_res = db_match_dbterm(&tb->common, p, mp, 0, - &(*current)->dbterm, NULL, 0); - if (is_value(match_res) && - is_value(key = db_getkey(tb->common.keypos, match_res)) && - eq(key, GETKEY(tb, (*current)->dbterm.tpl))) - { - next = (*current)->next; - hval = (*current)->hvalue; - new = replace_dbterm(tb, *current, match_res); - new->next = next; - new->hvalue = hval; - *current = new; - ++got; - } - --num_left; - current = &((*current)->next); - } - } -done: - // the more objects we've replaced, the more reductions we've consumed - BUMP_REDS(p, MIN(2000, (1000 - num_left) + (int)(got - prev_got))); - RET_TO_BIF(erts_make_integer(got,p),DB_ERROR_NONE); -trap: - BUMP_ALL_REDS(p); - if (IS_USMALL(0, got)) { - hp = HAlloc(p, 5); - egot = make_small(got); - } - else { - hp = HAlloc(p, BIG_UINT_HEAP_SIZE + 5); - egot = uint_to_big(got, hp); - hp += BIG_UINT_HEAP_SIZE; - } - continuation = TUPLE4(hp, tb->common.id, make_small(slot_ix), - tptr[3], - egot); - RET_TO_BIF(bif_trap1(&ets_select_replace_continue_exp, p, - continuation), - DB_ERROR_NONE); - -#undef RET_TO_BIF - -} /* ** Other interface routines (not directly coupled to one bif) diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index af1296367a..83cbf059c5 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -1843,6 +1843,7 @@ static int db_select_replace_tree(Process *p, DbTable *tbl, Eterm pattern, Eterm *ret) { DbTableTree *tb = &tbl->tree; + Eterm tid = NIL; // TODO DbTreeStack* stack; struct select_replace_context sc; struct mp_info mpi; @@ -1937,7 +1938,7 @@ static int db_select_replace_tree(Process *p, DbTable *tbl, continuation = TUPLE5 (hp, - tb->common.id, + tid, key, sc.end_condition, /* From the match program, needn't be copied */ mpb, -- cgit v1.2.3 From 36d93952f6ca64192f05e0482fa55270103c8d97 Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Tue, 14 Feb 2017 22:31:31 +0000 Subject: Use magic refs on revamped ETS code --- erts/emulator/beam/erl_db_hash.c | 27 +++++++++++---------------- erts/emulator/beam/erl_db_tree.c | 12 ++++-------- 2 files changed, 15 insertions(+), 24 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 1b5c8fe422..44d11dd5e7 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -1495,7 +1495,7 @@ static ERTS_INLINE int on_mtraversal_simple_trap(Export* trap_function, Eterm mpb = NIL; Eterm continuation = NIL; int is_first_trap = (prev_continuation_tptr == NULL); - size_t base_halloc_sz = (is_first_trap ? PROC_BIN_SIZE : 0); + size_t base_halloc_sz = (is_first_trap ? ERTS_MAGIC_REF_THING_SIZE : 0); BUMP_ALL_REDS(p); if (IS_USMALL(0, got)) { @@ -1509,7 +1509,7 @@ static ERTS_INLINE int on_mtraversal_simple_trap(Export* trap_function, } if (is_first_trap) { - mpb = db_make_mp_binary(p, *mpp, &hp); + mpb = erts_db_make_match_prog_ref(p, *mpp, &hp); *mpp = NULL; /* otherwise the caller will destroy it */ } else { @@ -1539,17 +1539,14 @@ static ERTS_INLINE int unpack_simple_mtraversal_continuation(Eterm continuation, if (arityval(*tptr) != 4) return 1; - if (! is_small(tptr[2]) - || !(is_binary(tptr[3]) && thing_subtag(*binary_val(tptr[3])) == REFC_BINARY_SUBTAG) - || !(is_big(tptr[4]) || is_small(tptr[4]))) - { + if (! is_small(tptr[2]) || !(is_big(tptr[4]) || is_small(tptr[4]))) { return 1; } *tptr_ptr = tptr; *tid_ptr = tptr[1]; *slot_ix_p = unsigned_val(tptr[2]); - *mpp = ((ProcBin *) binary_val(tptr[3]))->val; + *mpp = erts_db_get_match_prog_binary_unchecked(tptr[3]); if (is_big(tptr[4])) { *got_p = big_to_uint32(tptr[4]); } @@ -1629,8 +1626,8 @@ static int mtraversal_select_chunk_on_loop_ended(void* context_ptr, Sint slot_ix been in 'user space' */ } if (rest != NIL || slot_ix >= 0) { /* Need more calls */ - sc_context_ptr->hp = HAlloc(sc_context_ptr->p, 3 + 7 + PROC_BIN_SIZE); - mpb = db_make_mp_binary(sc_context_ptr->p, *mpp, &sc_context_ptr->hp); + sc_context_ptr->hp = HAlloc(sc_context_ptr->p, 3 + 7 + ERTS_MAGIC_REF_THING_SIZE); + mpb = erts_db_make_match_prog_ref(sc_context_ptr->p, *mpp, &sc_context_ptr->hp); continuation = TUPLE6( sc_context_ptr->hp, sc_context_ptr->tid, @@ -1671,8 +1668,8 @@ static int mtraversal_select_chunk_on_trap(void* context_ptr, Sint slot_ix, Sint if (sc_context_ptr->prev_continuation_tptr == NULL) { /* First time we're trapping */ - hp = HAlloc(sc_context_ptr->p, 7 + PROC_BIN_SIZE); - mpb = db_make_mp_binary(sc_context_ptr->p, *mpp, &hp); + hp = HAlloc(sc_context_ptr->p, 7 + ERTS_MAGIC_REF_THING_SIZE); + mpb = erts_db_make_match_prog_ref(sc_context_ptr->p, *mpp, &hp); continuation = TUPLE6( hp, sc_context_ptr->tid, @@ -1807,16 +1804,14 @@ static int db_select_continue_hash(Process* p, DbTable* tbl, Eterm continuation, if (arityval(*tptr) != 6) return DB_ERROR_BADPARAM; - if (!is_small(tptr[2]) || !is_small(tptr[3]) || !is_binary(tptr[4]) || + if (!is_small(tptr[2]) || !is_small(tptr[3]) || !(is_list(tptr[5]) || tptr[5] == NIL) || !is_small(tptr[6])) return DB_ERROR_BADPARAM; if ((chunk_size = signed_val(tptr[3])) < 0) return DB_ERROR_BADPARAM; - if (!(thing_subtag(*binary_val(tptr[4])) == REFC_BINARY_SUBTAG)) - return DB_ERROR_BADPARAM; - mp = ((ProcBin *) binary_val(tptr[4]))->val; - if (!IsMatchProgBinary(mp)) + mp = erts_db_get_match_prog_binary(tptr[4]); + if (mp == NULL) return DB_ERROR_BADPARAM; if ((got = signed_val(tptr[6])) < 0) diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 83cbf059c5..fea888b12d 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -1779,11 +1779,7 @@ static int db_select_replace_continue_tree(Process *p, lastkey = tptr[2]; end_condition = tptr[3]; - if (!(thing_subtag(*binary_val(tptr[4])) == REFC_BINARY_SUBTAG)) - RET_TO_BIF(NIL,DB_ERROR_BADPARAM); - mp = ((ProcBin *) binary_val(tptr[4]))->val; - if (!IsMatchProgBinary(mp)) - RET_TO_BIF(NIL,DB_ERROR_BADPARAM); + mp = erts_db_get_match_prog_binary_unchecked(tptr[4]); sc.p = p; sc.mp = mp; @@ -1923,18 +1919,18 @@ static int db_select_replace_tree(Process *p, DbTable *tbl, key = GETKEY(tb, sc.lastobj); sz = size_object(key); if (IS_USMALL(0, sc.replaced)) { - hp = HAlloc(p, sz + PROC_BIN_SIZE + 6); + hp = HAlloc(p, sz + ERTS_MAGIC_REF_THING_SIZE + 6); ereplaced = make_small(sc.replaced); } else { - hp = HAlloc(p, BIG_UINT_HEAP_SIZE + sz + PROC_BIN_SIZE + 6); + hp = HAlloc(p, BIG_UINT_HEAP_SIZE + sz + ERTS_MAGIC_REF_THING_SIZE + 6); ereplaced = uint_to_big(sc.replaced, hp); hp += BIG_UINT_HEAP_SIZE; } key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; - mpb = db_make_mp_binary(p,mpi.mp,&hp); + mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp); continuation = TUPLE5 (hp, -- cgit v1.2.3 From ed71ea35bad9a511125c82ce42160cad9fa8311f Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Sat, 25 Feb 2017 22:00:17 +0000 Subject: Reject unsafe matchspecs on ets:select_replace/2 Preemptively fail operation with badarg if the replacement object might have a different key. --- erts/emulator/beam/erl_db_hash.c | 63 +++++++++++++++++++-------- erts/emulator/beam/erl_db_tree.c | 52 +++++++++++++++------- erts/emulator/beam/erl_db_util.c | 94 ++++++++++++++++++++++++++++++++++++++++ erts/emulator/beam/erl_db_util.h | 1 + 4 files changed, 177 insertions(+), 33 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 44d11dd5e7..ac31569181 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -355,6 +355,9 @@ static ERTS_INLINE void SET_SEGTAB(DbTableHash* tb, erts_smp_atomic_set_nob(&tb->segtab, (erts_aint_t) segtab); } +/* Used by select_replace on analyze_pattern */ +typedef int (*extra_match_validator_t)(int keypos, Eterm match, Eterm guard, Eterm body); + /* ** Forward decl's (static functions) */ @@ -369,8 +372,9 @@ static void shrink(DbTableHash* tb, int nitems); static void grow(DbTableHash* tb, int nitems); static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, Uint sz, DbTableHash*); -static int analyze_pattern(DbTableHash *tb, Eterm pattern, - struct mp_info *mpi); +static int analyze_pattern(DbTableHash *tb, Eterm pattern, + extra_match_validator_t extra_validator, /* Optional callback */ + struct mp_info *mpi); /* * Method interface functions @@ -1202,7 +1206,9 @@ typedef int (*mtraversal_on_trap_t)(void* context_ptr, Sint slot_ix, Sint got, B /* * Begin hash table match traversal */ -static int match_traverse(Process* p, DbTableHash* tb, Eterm pattern, +static int match_traverse(Process* p, DbTableHash* tb, + Eterm pattern, + extra_match_validator_t extra_match_validator, /* Optional */ Sint chunk_size, /* If 0, no chunking */ Sint iterations_left, /* Nr. of iterations left */ Eterm** hpp, /* Heap */ @@ -1240,7 +1246,9 @@ static int match_traverse(Process* p, DbTableHash* tb, Eterm pattern, *ret = NIL; - if ((ret_value = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) { + if ((ret_value = analyze_pattern(tb, pattern, extra_match_validator, &mpi)) + != DB_ERROR_NONE) + { goto done; } @@ -1713,7 +1721,9 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl, Eterm tid, Eterm patte sc_context.prev_continuation_tptr = NULL; return match_traverse( - sc_context.p, sc_context.tb, pattern, sc_context.chunk_size, + sc_context.p, sc_context.tb, + pattern, NULL, + sc_context.chunk_size, MAX_SELECT_CHUNK_ITERATIONS, &sc_context.hp, 0, mtraversal_select_chunk_on_nothing_can_match, @@ -1906,8 +1916,8 @@ static int db_select_count_hash(Process *p, DbTable *tbl, Eterm tid, Eterm patte return match_traverse( scnt_context.p, scnt_context.tb, - pattern, chunk_size, - iterations_left, NULL, 0, + pattern, NULL, + chunk_size, iterations_left, NULL, 0, mtraversal_select_count_on_nothing_can_match, mtraversal_select_count_on_match_res, mtraversal_select_count_on_loop_ended, @@ -2046,7 +2056,9 @@ static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid, Eterm patt sd_context.last_pseudo_delete = (Uint) -1; return match_traverse( - sd_context.p, sd_context.tb, pattern, chunk_size, + sd_context.p, sd_context.tb, + pattern, NULL, + chunk_size, MAX_SELECT_DELETE_ITERATIONS, NULL, 1, mtraversal_select_delete_on_nothing_can_match, mtraversal_select_delete_on_match_res, @@ -2120,15 +2132,18 @@ static int mtraversal_select_replace_on_match_res(void* context_ptr, Sint slot_i { mtraversal_select_replace_context_t* sr_context_ptr = (mtraversal_select_replace_context_t*) context_ptr; DbTableHash* tb = sr_context_ptr->tb; +#ifdef DEBUG Eterm key = NIL; +#endif HashDbTerm* new = NULL; HashDbTerm* next = NULL; HashValue hval = INVALID_HASH; - if (is_value(match_res) && - is_value(key = db_getkey(tb->common.keypos, match_res)) && - eq(key, GETKEY(tb, (**current_ptr_ptr)->dbterm.tpl))) - { + if (is_value(match_res)) { +#ifdef DEBUG + ASSERT(is_value(key = db_getkey(tb->common.keypos, match_res))); + ASSERT(eq(key, GETKEY(tb, (**current_ptr_ptr)->dbterm.tpl))); +#endif next = (**current_ptr_ptr)->next; hval = (**current_ptr_ptr)->hvalue; new = replace_dbterm(tb, **current_ptr_ptr, match_res); @@ -2184,7 +2199,9 @@ static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm pattern, Eterm sr_context.prev_continuation_tptr = NULL; return match_traverse( - sr_context.p, sr_context.tb, pattern, chunk_size, + sr_context.p, sr_context.tb, + pattern, db_match_keeps_key, + chunk_size, MAX_SELECT_REPLACE_ITERATIONS, NULL, 1, mtraversal_select_replace_on_nothing_can_match, mtraversal_select_replace_on_match_res, @@ -2428,7 +2445,8 @@ static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds) ** slots should be searched. Also compiles the match program */ static int analyze_pattern(DbTableHash *tb, Eterm pattern, - struct mp_info *mpi) + extra_match_validator_t extra_validator, /* Optional callback */ + struct mp_info *mpi) { Eterm *ptpl; Eterm lst, tpl, ttpl; @@ -2466,7 +2484,10 @@ static int analyze_pattern(DbTableHash *tb, Eterm pattern, i = 0; for(lst = pattern; is_list(lst); lst = CDR(list_val(lst))) { - Eterm body; + Eterm match = NIL; + Eterm guard = NIL; + Eterm body = NIL; + ttpl = CAR(list_val(lst)); if (!is_tuple(ttpl)) { if (buff != sbuff) { @@ -2481,9 +2502,17 @@ static int analyze_pattern(DbTableHash *tb, Eterm pattern, } return DB_ERROR_BADPARAM; } - matches[i] = tpl = ptpl[1]; - guards[i] = ptpl[2]; + matches[i] = match = tpl = ptpl[1]; + guards[i] = guard = ptpl[2]; bodies[i] = body = ptpl[3]; + + if(extra_validator != NULL && !extra_validator(tb->common.keypos, match, guard, body)) { + if (buff != sbuff) { + erts_free(ERTS_ALC_T_DB_TMP, buff); + } + return DB_ERROR_BADPARAM; + } + if (!is_list(body) || CDR(list_val(body)) != NIL || CAR(list_val(body)) != am_DollarUnderscore) { mpi->all_objects = 0; diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index fea888b12d..949f8c46b6 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -300,6 +300,9 @@ struct select_replace_context { Sint replaced; }; +/* Used by select_replace on analyze_pattern */ +typedef int (*extra_match_validator_t)(int keypos, Eterm match, Eterm guard, Eterm body); + /* ** Forward declarations */ @@ -351,7 +354,8 @@ static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key); static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done); static int analyze_pattern(DbTableTree *tb, Eterm pattern, - struct mp_info *mpi); + extra_match_validator_t extra_validator, /* Optional callback */ + struct mp_info *mpi); static int doit_select(DbTableTree *tb, TreeDbTerm *this, void *ptr, @@ -1132,7 +1136,7 @@ static int db_select_tree(Process *p, DbTable *tbl, Eterm tid, sc.got = 0; sc.chunk_size = 0; - if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) { + if ((errcode = analyze_pattern(tb, pattern, NULL, &mpi)) != DB_ERROR_NONE) { RET_TO_BIF(NIL,errcode); } @@ -1335,7 +1339,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, Eterm tid, sc.keypos = tb->common.keypos; sc.got = 0; - if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) { + if ((errcode = analyze_pattern(tb, pattern, NULL, &mpi)) != DB_ERROR_NONE) { RET_TO_BIF(NIL,errcode); } @@ -1438,7 +1442,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, Eterm tid, sc.got = 0; sc.chunk_size = chunk_size; - if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) { + if ((errcode = analyze_pattern(tb, pattern, NULL, &mpi)) != DB_ERROR_NONE) { RET_TO_BIF(NIL,errcode); } @@ -1680,7 +1684,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, Eterm tid, sc.keypos = tb->common.keypos; sc.tb = tb; - if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) { + if ((errcode = analyze_pattern(tb, pattern, NULL, &mpi)) != DB_ERROR_NONE) { RET_TO_BIF(0,errcode); } @@ -1872,7 +1876,7 @@ static int db_select_replace_tree(Process *p, DbTable *tbl, sc.keypos = tb->common.keypos; sc.replaced = 0; - if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) { + if ((errcode = analyze_pattern(tb, pattern, db_match_keeps_key, &mpi)) != DB_ERROR_NONE) { RET_TO_BIF(NIL,errcode); } @@ -2193,8 +2197,9 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb, ** For the select functions, analyzes the pattern and determines which ** part of the tree should be searched. Also compiles the match program */ -static int analyze_pattern(DbTableTree *tb, Eterm pattern, - struct mp_info *mpi) +static int analyze_pattern(DbTableTree *tb, Eterm pattern, + extra_match_validator_t extra_validator, /* Optional callback */ + struct mp_info *mpi) { Eterm lst, tpl, ttpl; Eterm *matches,*guards, *bodies; @@ -2232,7 +2237,10 @@ static int analyze_pattern(DbTableTree *tb, Eterm pattern, i = 0; for(lst = pattern; is_list(lst); lst = CDR(list_val(lst))) { - Eterm body; + Eterm match = NIL; + Eterm guard = NIL; + Eterm body = NIL; + ttpl = CAR(list_val(lst)); if (!is_tuple(ttpl)) { if (buff != sbuff) { @@ -2247,9 +2255,17 @@ static int analyze_pattern(DbTableTree *tb, Eterm pattern, } return DB_ERROR_BADPARAM; } - matches[i] = tpl = ptpl[1]; - guards[i] = ptpl[2]; + matches[i] = match = tpl = ptpl[1]; + guards[i] = guard = ptpl[2]; bodies[i] = body = ptpl[3]; + + if(extra_validator != NULL && !extra_validator(tb->common.keypos, match, guard, body)) { + if (buff != sbuff) { + erts_free(ERTS_ALC_T_DB_TMP, buff); + } + return DB_ERROR_BADPARAM; + } + if (!is_list(body) || CDR(list_val(body)) != NIL || CAR(list_val(body)) != am_DollarUnderscore) { mpi->all_objects = 0; @@ -3443,7 +3459,10 @@ static int doit_select_replace(DbTableTree *tb, TreeDbTerm **this, void *ptr, int forward) { struct select_replace_context *sc = (struct select_replace_context *) ptr; - Eterm ret, key; + Eterm ret = NIL; +#ifdef DEBUG + Eterm key = NIL; +#endif sc->lastobj = (*this)->dbterm.tpl; @@ -3456,10 +3475,11 @@ static int doit_select_replace(DbTableTree *tb, TreeDbTerm **this, void *ptr, ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0, &(*this)->dbterm, NULL, 0); - if (is_value(ret) && - is_value(key = db_getkey(tb->common.keypos, ret)) && - (cmp_key(tb, key, *this) == 0)) - { + if (is_value(ret)) { +#ifdef DEBUG + ASSERT(is_value(key = db_getkey(tb->common.keypos, ret))); + ASSERT(cmp_key(tb, key, *this) == 0); +#endif *this = replace_dbterm(tb, *this, ret); sc->lastobj = (*this)->dbterm.tpl; ++(sc->replaced); diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 6f30b1d3dd..3b2e7f4407 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1119,6 +1119,100 @@ error: return NULL; } +/* This is used by select_replace */ +int db_match_keeps_key(int keypos, Eterm match, Eterm guard, Eterm body) { + Eterm match_key = NIL; + int match_key_variable = -1; + Eterm* body_list = NULL; + Eterm single_body_term = NIL; + Eterm* single_body_term_tpl = NULL; + Eterm single_body_subterm = NIL; + Eterm single_body_subterm_key = NIL; + int single_body_subterm_key_variable = -1; + Eterm* single_body_subterm_key_tpl = NULL; + + if (!is_list(body)) { + return 0; + } + + body_list = list_val(body); + if (CDR(body_list) != NIL) { + return 0; + } + + single_body_term = CAR(body_list); + if (single_body_term == am_DollarUnderscore) { + /* same tuple is returned */ + return 1; + } + + if (!is_tuple(single_body_term)) { + return 0; + } + + single_body_term_tpl = tuple_val(single_body_term); + if (arityval(*single_body_term_tpl) != 1) { + // not the 1-element tuple we're expecting + return 0; + } + + match_key = db_getkey(keypos, match); + if (!is_value(match_key)) { + // can't get key out of match + return 0; + } + + single_body_subterm = single_body_term_tpl[1]; + single_body_subterm_key = db_getkey(keypos, single_body_subterm); + if (!is_value(single_body_subterm_key)) { + // can't get key out of single body subterm + return 0; + } + + match_key_variable = db_is_variable(match_key); + single_body_subterm_key_variable = db_is_variable(single_body_subterm_key); + if (match_key_variable != -1 && match_key_variable == single_body_subterm_key_variable) { + /* tuple with same key is returned */ + return 1; + } + + if (!is_tuple(single_body_subterm_key)) { + /* can't possibly be an element instruction */ + return 0; + } + + single_body_subterm_key_tpl = tuple_val(single_body_subterm_key); + if (arityval(*single_body_subterm_key_tpl) != 3) { + /* can't possibly be an element instruction */ + return 0; + } + + if (single_body_subterm_key_tpl[1] != am_element) { + /* tag is not of an element instruction */ + return 0; + } + if (single_body_subterm_key_tpl[3] != am_DollarUnderscore) { + /* even if it's an element instruction, it's not fetching from the original tuple */ + return 0; + } + + if (is_big(single_body_subterm_key_tpl[2]) + && (big_to_uint32(single_body_subterm_key_tpl[2]) != keypos)) + { + /* the key comes from a different position */ + return 0; + } + + if (is_small(single_body_subterm_key_tpl[2]) + && (unsigned_val(single_body_subterm_key_tpl[2]) != keypos)) + { + /* the key comes from a different position */ + return 0; + } + + return 1; +} + /* This is used when tracing */ Eterm erts_match_set_lint(Process *p, Eterm matchexpr) { return db_match_set_lint(p, matchexpr, DCOMP_TRACE); diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 89ef79d2dc..1a84f20e5d 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -382,6 +382,7 @@ Eterm db_add_counter(Eterm** hpp, Wterm counter, Eterm incr); Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags); Binary *db_match_set_compile(Process *p, Eterm matchexpr, Uint flags); +int db_match_keeps_key(int keypos, Eterm match, Eterm guard, Eterm body); int erts_db_match_prog_destructor(Binary *); typedef struct match_prog { -- cgit v1.2.3 From d3edd7a070d50ae99cb42f0564013f122bda80b2 Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Sat, 25 Feb 2017 22:03:23 +0000 Subject: Fix typo that broke debug builds --- erts/emulator/beam/erl_db_hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index ac31569181..21af322632 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -2190,7 +2190,7 @@ static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm pattern, Eterm /* Bag implementation presented both semantic consistency and performance issues, * unsupported for now */ - ASSERT(!(tb->common.status & DB_BAG)); + ASSERT(!(tbl->hash.common.status & DB_BAG)); sr_context.p = p; sr_context.tb = &tbl->hash; -- cgit v1.2.3 From 4855e8bd2d9933020479a4fe684a0cb8bbaf6f21 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 2 Mar 2017 13:37:58 +0100 Subject: Add more complete key-safety check --- erts/emulator/beam/erl_db_util.c | 145 ++++++++++++++++++++++++++++++--------- 1 file changed, 111 insertions(+), 34 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 3b2e7f4407..ab45c3fe0f 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1119,17 +1119,111 @@ error: return NULL; } +/* + * Compare a matching term 'a' with a constructing term 'b' for equality. + * + * Returns true if 'b' is guaranteed to always construct + * the same term as 'a' has matched. + */ +static int db_match_eq_body(Eterm a, Eterm b) +{ + DECLARE_ESTACK(s); + Uint arity; + Eterm *ap, *bp; + int const_mode = 0; + const Eterm CONST_MODE_OFF = THE_NON_VALUE; + + while (1) { + switch(b & _TAG_PRIMARY_MASK) { + case TAG_PRIMARY_LIST: + if (!is_list(a)) + return 0; + ESTACK_PUSH2(s, CDR(list_val(a)), CDR(list_val(b))); + a = CAR(list_val(a)); + b = CAR(list_val(b)); + continue; /* loop without pop */ + + case TAG_PRIMARY_BOXED: + if (is_tuple(b)) { + bp = tuple_val(b); + if (!const_mode) { + if (bp[0] == make_arityval(1) && is_tuple(bp[1])) { + b = bp[1]; /* double-tuple syntax */ + } + else if (bp[0] == make_arityval(2) && bp[1] == am_const) { + ESTACK_PUSH(s, CONST_MODE_OFF); + const_mode = 1; /* {const, term()} syntax */ + b = bp[2]; + continue; /* loop without pop */ + } + else + return 0; /* function call or invalid tuple syntax */ + } + if (!is_tuple(a)) + return 0; + + ap = tuple_val(a); + bp = tuple_val(b); + if (ap[0] != bp[0]) + return 0; + arity = arityval(ap[0]); + if (arity > 0) { + a = *(++ap); + b = *(++bp); + while(--arity) { + ESTACK_PUSH2(s, *(++ap), *(++bp)); + } + continue; /* loop without pop */ + } + } + else if (is_map(b)) { + /* We don't know what other pairs the matched map may contain */ + return 0; + } + else if (!eq(a,b)) /* other boxed */ + return 0; + break; + + case TAG_PRIMARY_IMMED1: + if (a != b || a == am_Underscore || a == am_DollarDollar + || a == am_DollarUnderscore + || (const_mode && db_is_variable(a) >= 0)) { + + return 0; + } + break; + default: + erts_exit(ERTS_ABORT_EXIT, "db_compare: " + "Bad object on ESTACK: 0x%bex\n", b); + } + +pop_next: + if (ESTACK_ISEMPTY(s)) + break; /* done */ + + b = ESTACK_POP(s); + if (b == CONST_MODE_OFF) { + ASSERT(const_mode); + const_mode = 0; + goto pop_next; + } + a = ESTACK_POP(s); + } + + DESTROY_ESTACK(s); + return 1; +} + /* This is used by select_replace */ -int db_match_keeps_key(int keypos, Eterm match, Eterm guard, Eterm body) { - Eterm match_key = NIL; - int match_key_variable = -1; - Eterm* body_list = NULL; - Eterm single_body_term = NIL; - Eterm* single_body_term_tpl = NULL; - Eterm single_body_subterm = NIL; - Eterm single_body_subterm_key = NIL; - int single_body_subterm_key_variable = -1; - Eterm* single_body_subterm_key_tpl = NULL; +int db_match_keeps_key(int keypos, Eterm match, Eterm guard, Eterm body) +{ + Eterm match_key; + Eterm* body_list; + Eterm single_body_term; + Eterm* single_body_term_tpl; + Eterm single_body_subterm; + Eterm single_body_subterm_key; + Eterm* single_body_subterm_key_tpl; if (!is_list(body)) { return 0; @@ -1169,9 +1263,7 @@ int db_match_keeps_key(int keypos, Eterm match, Eterm guard, Eterm body) { return 0; } - match_key_variable = db_is_variable(match_key); - single_body_subterm_key_variable = db_is_variable(single_body_subterm_key); - if (match_key_variable != -1 && match_key_variable == single_body_subterm_key_variable) { + if (db_match_eq_body(match_key, single_body_subterm_key)) { /* tuple with same key is returned */ return 1; } @@ -1187,30 +1279,15 @@ int db_match_keeps_key(int keypos, Eterm match, Eterm guard, Eterm body) { return 0; } - if (single_body_subterm_key_tpl[1] != am_element) { - /* tag is not of an element instruction */ - return 0; - } - if (single_body_subterm_key_tpl[3] != am_DollarUnderscore) { - /* even if it's an element instruction, it's not fetching from the original tuple */ - return 0; - } - - if (is_big(single_body_subterm_key_tpl[2]) - && (big_to_uint32(single_body_subterm_key_tpl[2]) != keypos)) - { - /* the key comes from a different position */ - return 0; - } - - if (is_small(single_body_subterm_key_tpl[2]) - && (unsigned_val(single_body_subterm_key_tpl[2]) != keypos)) + if (single_body_subterm_key_tpl[1] == am_element && + single_body_subterm_key_tpl[3] == am_DollarUnderscore && + single_body_subterm_key_tpl[2] == make_small(keypos)) { - /* the key comes from a different position */ - return 0; + /* {element, KeyPos, '$_'} */ + return 1; } - return 1; + return 0; } /* This is used when tracing */ -- cgit v1.2.3 From 42ece2ed3a7022adf0568d5cd64d8df8c8e64b4b Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 9 Mar 2017 16:09:10 +0100 Subject: Cleanup some unnecessary variable initialization --- erts/emulator/beam/erl_db_hash.c | 12 +++++------- erts/emulator/beam/erl_db_tree.c | 10 ++++------ 2 files changed, 9 insertions(+), 13 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 21af322632..569419265b 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -2132,16 +2132,14 @@ static int mtraversal_select_replace_on_match_res(void* context_ptr, Sint slot_i { mtraversal_select_replace_context_t* sr_context_ptr = (mtraversal_select_replace_context_t*) context_ptr; DbTableHash* tb = sr_context_ptr->tb; -#ifdef DEBUG - Eterm key = NIL; -#endif - HashDbTerm* new = NULL; - HashDbTerm* next = NULL; - HashValue hval = INVALID_HASH; + HashDbTerm* new; + HashDbTerm* next; + HashValue hval; if (is_value(match_res)) { #ifdef DEBUG - ASSERT(is_value(key = db_getkey(tb->common.keypos, match_res))); + Eterm key = db_getkey(tb->common.keypos, match_res); + ASSERT(is_value(key)); ASSERT(eq(key, GETKEY(tb, (**current_ptr_ptr)->dbterm.tpl))); #endif next = (**current_ptr_ptr)->next; diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 949f8c46b6..fadd63be34 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -3459,10 +3459,7 @@ static int doit_select_replace(DbTableTree *tb, TreeDbTerm **this, void *ptr, int forward) { struct select_replace_context *sc = (struct select_replace_context *) ptr; - Eterm ret = NIL; -#ifdef DEBUG - Eterm key = NIL; -#endif + Eterm ret; sc->lastobj = (*this)->dbterm.tpl; @@ -3477,8 +3474,9 @@ static int doit_select_replace(DbTableTree *tb, TreeDbTerm **this, void *ptr, if (is_value(ret)) { #ifdef DEBUG - ASSERT(is_value(key = db_getkey(tb->common.keypos, ret))); - ASSERT(cmp_key(tb, key, *this) == 0); + Eterm key = db_getkey(tb->common.keypos, ret); + ASSERT(is_value(key)); + ASSERT(cmp_key(tb, key, old) == 0); #endif *this = replace_dbterm(tb, *this, ret); sc->lastobj = (*this)->dbterm.tpl; -- cgit v1.2.3 From 865f06f93db7a68533bbdc4c961dd45699d6a07f Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 9 Mar 2017 16:15:10 +0100 Subject: erts: Fix benign bug in match spec machine Looks like this line has truly been dead code as ets has (so far) always been using ERTS_PAM_COPY_RESULT and matchPushExpr is not generated for tracing. --- erts/emulator/beam/erl_db_util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index ab45c3fe0f..d4dbe3d434 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -2343,7 +2343,7 @@ restart: } } else { - *esp = term; + *esp++ = term; } break; case matchPushArrayAsList: -- cgit v1.2.3 From 5e18e917f29ed46caffa1211eb52ade01d24366a Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 9 Mar 2017 16:23:04 +0100 Subject: erts: Optimize ets:select_replace to not use heap for temporary matchspec results. ToDo: Would be even nicer if PAM could allocate and build the ETS objects without extra copy_struct needed. --- erts/emulator/beam/erl_db_hash.c | 3 ++- erts/emulator/beam/erl_db_tree.c | 11 +++++++++-- erts/emulator/beam/erl_db_util.c | 7 ++++++- 3 files changed, 17 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 569419265b..92b833468d 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -2144,9 +2144,10 @@ static int mtraversal_select_replace_on_match_res(void* context_ptr, Sint slot_i #endif next = (**current_ptr_ptr)->next; hval = (**current_ptr_ptr)->hvalue; - new = replace_dbterm(tb, **current_ptr_ptr, match_res); + new = new_dbterm(tb, match_res); new->next = next; new->hvalue = hval; + free_term(tb, **current_ptr_ptr); **current_ptr_ptr = new; /* replace 'next' pointer in previous object */ *current_ptr_ptr = &((**current_ptr_ptr)->next); /* advance to next object */ return 1; diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index fadd63be34..60bf7fc979 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -3473,13 +3473,20 @@ static int doit_select_replace(DbTableTree *tb, TreeDbTerm **this, void *ptr, &(*this)->dbterm, NULL, 0); if (is_value(ret)) { + TreeDbTerm* new; + TreeDbTerm* old = *this; #ifdef DEBUG Eterm key = db_getkey(tb->common.keypos, ret); ASSERT(is_value(key)); ASSERT(cmp_key(tb, key, old) == 0); #endif - *this = replace_dbterm(tb, *this, ret); - sc->lastobj = (*this)->dbterm.tpl; + new = new_dbterm(tb, ret); + new->left = old->left; + new->right = old->right; + new->balance = old->balance; + sc->lastobj = new->dbterm.tpl; + *this = new; + free_term(tb, old); ++(sc->replaced); } if (--(sc->max) <= 0) { diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index d4dbe3d434..03cc11bdc4 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -5332,6 +5332,7 @@ void db_free_tmp_uncompressed(DbTerm* obj) Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog, int all, DbTerm* obj, Eterm** hpp, Uint extra) { + enum erts_pam_run_flags flags; Uint32 dummy; Eterm res; @@ -5339,9 +5340,13 @@ Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog, obj = db_alloc_tmp_uncompressed(tb, obj); } + flags = (hpp ? + ERTS_PAM_COPY_RESULT | ERTS_PAM_CONTIGUOUS_TUPLE : + ERTS_PAM_TMP_RESULT | ERTS_PAM_CONTIGUOUS_TUPLE); + res = db_prog_match(c_p, c_p, bprog, make_tuple(obj->tpl), NULL, 0, - ERTS_PAM_COPY_RESULT|ERTS_PAM_CONTIGUOUS_TUPLE, &dummy); + flags, &dummy); if (is_value(res) && hpp!=NULL) { *hpp = HAlloc(c_p, extra); -- cgit v1.2.3 From 7cb2ca89f96d9724267b46d2f1eb52a2cbe7c06f Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Wed, 22 Mar 2017 23:56:56 +0000 Subject: Use ETS table id references on select_replace --- erts/emulator/beam/erl_db.c | 2 +- erts/emulator/beam/erl_db_hash.c | 6 +++--- erts/emulator/beam/erl_db_tree.c | 5 ++--- erts/emulator/beam/erl_db_util.h | 1 + 4 files changed, 7 insertions(+), 7 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index d77c585628..378328856d 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -2994,7 +2994,7 @@ BIF_RETTYPE ets_select_replace_2(BIF_ALIST_2) if (safety == ITER_UNSAFE) { local_fix_table(tb); } - cret = tb->common.meth->db_select_replace(BIF_P, tb, BIF_ARG_2, &ret); + cret = tb->common.meth->db_select_replace(BIF_P, tb, BIF_ARG_1, BIF_ARG_2, &ret); if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) { fix_table_locked(BIF_P,tb); diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 92b833468d..297bef3023 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -416,7 +416,7 @@ static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid, static int db_select_delete_continue_hash(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); -static int db_select_replace_hash(Process *p, DbTable *tbl, +static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret); static int db_select_replace_continue_hash(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); @@ -2181,7 +2181,7 @@ static int mtraversal_select_replace_on_trap(void* context_ptr, Sint slot_ix, Si slot_ix, got, mpp, ret); } -static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm pattern, Eterm *ret) +static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret) { mtraversal_select_replace_context_t sr_context = {0}; Sint chunk_size = 0; @@ -2193,7 +2193,7 @@ static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm pattern, Eterm sr_context.p = p; sr_context.tb = &tbl->hash; - sr_context.tid = NIL; // TODO + sr_context.tid = tid; sr_context.hp = NULL; sr_context.prev_continuation_tptr = NULL; diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 60bf7fc979..6334723d39 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -424,7 +424,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret); static int db_select_delete_continue_tree(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); -static int db_select_replace_tree(Process *p, DbTable *tbl, +static int db_select_replace_tree(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret); static int db_select_replace_continue_tree(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); @@ -1839,11 +1839,10 @@ static int db_select_replace_continue_tree(Process *p, #undef RET_TO_BIF } -static int db_select_replace_tree(Process *p, DbTable *tbl, +static int db_select_replace_tree(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret) { DbTableTree *tb = &tbl->tree; - Eterm tid = NIL; // TODO DbTreeStack* stack; struct select_replace_context sc; struct mp_info mpi; diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 1a84f20e5d..9be77fcefa 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -171,6 +171,7 @@ typedef struct db_table_method Eterm* ret); int (*db_select_replace)(Process* p, DbTable* tb, /* [in out] */ + Eterm tid, Eterm pattern, Eterm* ret); int (*db_select_replace_continue)(Process* p, -- cgit v1.2.3 From 4dd2ccbdf374b4ce631d4d796bb690a55149e160 Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Thu, 23 Mar 2017 00:32:44 +0000 Subject: Remove redundant variable initializations --- erts/emulator/beam/erl_db_hash.c | 111 ++++++++++++++++++--------------------- erts/emulator/beam/erl_db_tree.c | 6 +-- 2 files changed, 55 insertions(+), 62 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 297bef3023..0d908fc2c8 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -1221,17 +1221,17 @@ static int match_traverse(Process* p, DbTableHash* tb, void* context_ptr, /* State for callbacks above */ Eterm* ret) { - Sint slot_ix = -1; /* Slot index */ - HashDbTerm** current_ptr = NULL; /* Refers to either the bucket pointer or - * the 'next' pointer in the previous term - */ - HashDbTerm* saved_current = NULL; /* Helper to avoid double skip on match */ + Sint slot_ix; /* Slot index */ + HashDbTerm** current_ptr; /* Refers to either the bucket pointer or + * the 'next' pointer in the previous term + */ + HashDbTerm* saved_current; /* Helper to avoid double skip on match */ struct mp_info mpi; - unsigned current_list_pos = 0; /* Prefound buckets list index */ + unsigned current_list_pos = 0; /* Prefound buckets list index */ Eterm match_res; - Sint got = 0; /* Matched terms counter */ - erts_smp_rwmtx_t* lck = NULL; /* Slot lock */ - int ret_value = DB_ERROR_NONE; + Sint got = 0; /* Matched terms counter */ + erts_smp_rwmtx_t* lck; /* Slot lock */ + int ret_value; #ifdef ERTS_SMP erts_smp_rwmtx_t* (*lock_hash_function)(DbTableHash*, HashValue) = (lock_for_write ? WLOCK_HASH : RLOCK_HASH); @@ -1244,8 +1244,6 @@ static int match_traverse(Process* p, DbTableHash* tb, Sint (*next_slot_function)(DbTableHash*, Uint, erts_smp_rwmtx_t**) = (lock_for_write ? next_slot_w : next_slot); - *ret = NIL; - if ((ret_value = analyze_pattern(tb, pattern, extra_match_validator, &mpi)) != DB_ERROR_NONE) { @@ -1385,13 +1383,13 @@ static int match_traverse_continue(Process* p, DbTableHash* tb, Eterm* ret) { int all_objects = (*mpp)->flags & BIN_FLAG_ALL_OBJECTS; - HashDbTerm** current_ptr = NULL; /* Refers to either the bucket pointer or + HashDbTerm** current_ptr; /* Refers to either the bucket pointer or * the 'next' pointer in the previous term */ - HashDbTerm* saved_current = NULL; /* Helper to avoid double skip on match */ - Eterm match_res = NIL; - erts_smp_rwmtx_t* lck = NULL; - int ret_value = DB_ERROR_NONE; + HashDbTerm* saved_current; /* Helper to avoid double skip on match */ + Eterm match_res; + erts_smp_rwmtx_t* lck; + int ret_value; #ifdef ERTS_SMP erts_smp_rwmtx_t* (*lock_hash_function)(DbTableHash*, HashValue) = (lock_for_write ? WLOCK_HASH : RLOCK_HASH); @@ -1404,8 +1402,6 @@ static int match_traverse_continue(Process* p, DbTableHash* tb, Sint (*next_slot_function)(DbTableHash* tb, Uint ix, erts_smp_rwmtx_t** lck_ptr) = (lock_for_write ? next_slot_w : next_slot); - *ret = NIL; - if (got < 0) return DB_ERROR_BADPARAM; @@ -1498,10 +1494,10 @@ static ERTS_INLINE int on_mtraversal_simple_trap(Export* trap_function, Binary** mpp, Eterm* ret) { - Eterm* hp = NULL; - Eterm egot = NIL; - Eterm mpb = NIL; - Eterm continuation = NIL; + Eterm* hp; + Eterm egot; + Eterm mpb; + Eterm continuation; int is_first_trap = (prev_continuation_tptr == NULL); size_t base_halloc_sz = (is_first_trap ? ERTS_MAGIC_REF_THING_SIZE : 0); @@ -1541,7 +1537,7 @@ static ERTS_INLINE int unpack_simple_mtraversal_continuation(Eterm continuation, Binary** mpp, Sint* got_p) { - Eterm* tptr = NULL; + Eterm* tptr; ASSERT(is_tuple(continuation)); tptr = tuple_val(continuation); if (arityval(*tptr) != 4) @@ -1605,7 +1601,7 @@ static int mtraversal_select_chunk_on_loop_ended(void* context_ptr, Sint slot_ix Sint iterations_left, Binary** mpp, Eterm* ret) { mtraversal_select_chunk_context_t* sc_context_ptr = (mtraversal_select_chunk_context_t*) context_ptr; - Eterm mpb = NIL; + Eterm mpb; if (iterations_left == MAX_SELECT_CHUNK_ITERATIONS) { /* We didn't get to iterate a single time, which means EOT */ @@ -1668,9 +1664,9 @@ static int mtraversal_select_chunk_on_trap(void* context_ptr, Sint slot_ix, Sint Binary** mpp, Eterm* ret) { mtraversal_select_chunk_context_t* sc_context_ptr = (mtraversal_select_chunk_context_t*) context_ptr; - Eterm mpb = NIL; - Eterm continuation = NIL; - Eterm* hp = NULL; + Eterm mpb; + Eterm continuation; + Eterm* hp; BUMP_ALL_REDS(sc_context_ptr->p); @@ -1711,7 +1707,7 @@ static int db_select_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, in static int db_select_chunk_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Sint chunk_size, int reverse, Eterm *ret) { - mtraversal_select_chunk_context_t sc_context = {0}; + mtraversal_select_chunk_context_t sc_context; sc_context.p = p; sc_context.tb = &tbl->hash; sc_context.tid = tid; @@ -1743,9 +1739,9 @@ static int mtraversal_select_chunk_continue_on_loop_ended(void* context_ptr, Sin Sint iterations_left, Binary** mpp, Eterm* ret) { mtraversal_select_chunk_context_t* sc_context_ptr = (mtraversal_select_chunk_context_t*) context_ptr; - Eterm continuation = NIL; + Eterm continuation; Eterm rest = NIL; - Eterm* hp = NULL; + Eterm* hp; ASSERT(iterations_left <= MAX_SELECT_CHUNK_ITERATIONS); BUMP_REDS(sc_context_ptr->p, MAX_SELECT_CHUNK_ITERATIONS - iterations_left); @@ -1755,7 +1751,6 @@ static int mtraversal_select_chunk_continue_on_loop_ended(void* context_ptr, Sin /* Cannot write destructively here, the list may have been in user space */ - rest = NIL; hp = HAlloc(sc_context_ptr->p, (got - sc_context_ptr->chunk_size) * 2); while (got-- > sc_context_ptr->chunk_size) { rest = CONS(hp, CAR(list_val(sc_context_ptr->match_list)), rest); @@ -1797,15 +1792,14 @@ static int mtraversal_select_chunk_continue_on_loop_ended(void* context_ptr, Sin */ static int db_select_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret) { mtraversal_select_chunk_context_t sc_context = {0}; - Eterm* tptr = NULL; - Eterm tid = NIL; - Binary* mp = NULL; - Sint got = 0; - Sint slot_ix = 0; - Sint chunk_size = 0; - Eterm match_list = NIL; + Eterm* tptr; + Eterm tid; + Binary* mp; + Sint got; + Sint slot_ix; + Sint chunk_size; + Eterm match_list; Sint iterations_left = MAX_SELECT_CHUNK_ITERATIONS; - *ret = NIL; /* Decode continuation. We know it's a tuple but not the arity or anything else */ ASSERT(is_tuple(continuation)); @@ -1930,11 +1924,11 @@ static int db_select_count_hash(Process *p, DbTable *tbl, Eterm tid, Eterm patte */ static int db_select_count_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret) { mtraversal_select_count_context_t scnt_context = {0}; - Eterm* tptr = NULL; - Eterm tid = NIL; - Binary* mp = NULL; - Sint got = 0; - Sint slot_ix = 0; + Eterm* tptr; + Eterm tid; + Binary* mp; + Sint got; + Sint slot_ix; Sint chunk_size = 0; *ret = NIL; @@ -1990,7 +1984,7 @@ static int mtraversal_select_delete_on_match_res(void* context_ptr, Sint slot_ix { HashDbTerm** current_ptr = *current_ptr_ptr; mtraversal_select_delete_context_t* sd_context_ptr = (mtraversal_select_delete_context_t*) context_ptr; - HashDbTerm* del = NULL; + HashDbTerm* del; if (match_res != am_true) return 0; @@ -2072,13 +2066,12 @@ static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid, Eterm patt */ static int db_select_delete_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret) { mtraversal_select_delete_context_t sd_context = {0}; - Eterm* tptr = NULL; - Eterm tid = NIL; - Binary* mp = NULL; - Sint got = 0; - Sint slot_ix = 0; + Eterm* tptr; + Eterm tid; + Binary* mp; + Sint got; + Sint slot_ix; Sint chunk_size = 0; - *ret = NIL; if (unpack_simple_mtraversal_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) { return DB_ERROR_BADPARAM; @@ -2215,11 +2208,11 @@ static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pat static int db_select_replace_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret) { mtraversal_select_replace_context_t sr_context = {0}; - Eterm* tptr = NULL; - Eterm tid = NIL; - Binary* mp = NULL; - Sint got = 0; - Sint slot_ix = 0; + Eterm* tptr; + Eterm tid ; + Binary* mp; + Sint got; + Sint slot_ix; Sint chunk_size = 0; *ret = NIL; @@ -2483,9 +2476,9 @@ static int analyze_pattern(DbTableHash *tb, Eterm pattern, i = 0; for(lst = pattern; is_list(lst); lst = CDR(list_val(lst))) { - Eterm match = NIL; - Eterm guard = NIL; - Eterm body = NIL; + Eterm match; + Eterm guard; + Eterm body; ttpl = CAR(list_val(lst)); if (!is_tuple(ttpl)) { diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 6334723d39..ab8da6ccf6 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -2236,9 +2236,9 @@ static int analyze_pattern(DbTableTree *tb, Eterm pattern, i = 0; for(lst = pattern; is_list(lst); lst = CDR(list_val(lst))) { - Eterm match = NIL; - Eterm guard = NIL; - Eterm body = NIL; + Eterm match; + Eterm guard; + Eterm body; ttpl = CAR(list_val(lst)); if (!is_tuple(ttpl)) { -- cgit v1.2.3 From 14d709b0e07e899161a40bb43fc43fd6916f59ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 29 Aug 2014 18:27:45 +0200 Subject: compiler: Add is_tagged_tuple instruction Rewrite the instruction stream on tagged tuple tests. Tagged tuples means a tuple of any arity with an atom as its first element. Typically records, ok-tuples and error-tuples. from: ... {test,is_tuple,Fail,[Src]}. {test,test_arity,Fail,[Src,Sz]}. ... {get_tuple_element,Src,0,Dst}. ... {test,is_eq_exact,Fail,[Dst,Atom]}. ... to: ... {test,is_tagged_tuple,Fail,[Src,Sz,Atom]}. ... --- erts/emulator/beam/beam_emu.c | 9 +++++++++ erts/emulator/beam/ops.tab | 14 ++++++++++++++ 2 files changed, 23 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 8be0f58227..9a91fdce08 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -850,6 +850,15 @@ do { \ } while (0) #endif +#define IsTaggedTuple(Src,Arityval,Tag,Fail) \ + do { \ + if (!(is_tuple(Src) && \ + (tuple_val(Src))[0] == Arityval && \ + (tuple_val(Src))[1] == Tag)) { \ + Fail; \ + } \ + } while (0) + #define IsBoolean(X, Fail) if ((X) != am_true && (X) != am_false) { Fail; } #define IsBinary(Src, Fail) \ diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index ec36b23059..9b5bd7a749 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -620,6 +620,20 @@ test_heap Need u==1 | put_list Y=y x==0 x==0 => test_heap_1_put_list Need Y %macro: test_heap_1_put_list TestHeapPutList -pack test_heap_1_put_list I y +# +# is_tagged_tuple Fail=f Src=rxy Arity Atom=a +# + +is_tagged_tuple Fail Literal=q Arity Atom => \ + move Literal x | is_tagged_tuple Fail x Arity Atom +is_tagged_tuple Fail=f c Arity Atom => jump Fail + +%macro:is_tagged_tuple IsTaggedTuple -fail_action + +is_tagged_tuple f r A a +is_tagged_tuple f x A a +is_tagged_tuple f y A a + # Test tuple & arity (head) is_tuple Fail Literal=q => move Literal x | is_tuple Fail x -- cgit v1.2.3 From 1bd867a8dbd88ffcee38b2179d42b563e5078820 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 24 Mar 2017 14:37:45 +0100 Subject: Fix dirty scheduler type tests Old test for dirty schedulers didn't work with Visual C++ --- erts/emulator/beam/beam_debug.c | 18 ++++++++++++----- erts/emulator/beam/erl_nif.c | 17 ++++++++++------ erts/emulator/beam/erl_process.c | 42 ++++++++++++++++++++++++++-------------- erts/emulator/beam/erl_process.h | 27 +++++--------------------- 4 files changed, 57 insertions(+), 47 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 8326d348af..b2663d3931 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -853,15 +853,23 @@ dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I) goto badarg; esdp = erts_proc_sched_data(c_p); if (!esdp) - ERTS_BIF_PREP_RET(ret, am_error); - else if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) + goto scheduler_type_error; + + switch (esdp->type) { + case ERTS_SCHED_NORMAL: ERTS_BIF_PREP_RET(ret, am_normal); - else if (ERTS_SCHEDULER_IS_DIRTY_CPU(esdp)) + break; + case ERTS_SCHED_DIRTY_CPU: ERTS_BIF_PREP_RET(ret, am_dirty_cpu); - else if (ERTS_SCHEDULER_IS_DIRTY_IO(esdp)) + break; + case ERTS_SCHED_DIRTY_IO: ERTS_BIF_PREP_RET(ret, am_dirty_io); - else + break; + default: + scheduler_type_error: ERTS_BIF_PREP_RET(ret, am_error); + break; + } } else if (am_error == arg1) { switch (arg2) { diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index e6da4c1a76..7c7d265257 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2820,14 +2820,19 @@ enif_thread_type(void) if (!esdp) return ERL_NIF_THR_UNDEFINED; - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) + switch (esdp->type) { + case ERTS_SCHED_NORMAL: return ERL_NIF_THR_NORMAL_SCHEDULER; - - if (ERTS_SCHEDULER_IS_DIRTY_CPU(esdp)) +#ifdef ERTS_DIRTY_SCHEDULERS + case ERTS_SCHED_DIRTY_CPU: return ERL_NIF_THR_DIRTY_CPU_SCHEDULER; - - ASSERT(ERTS_SCHEDULER_IS_DIRTY_IO(esdp)); - return ERL_NIF_THR_DIRTY_IO_SCHEDULER; + case ERTS_SCHED_DIRTY_IO: + return ERL_NIF_THR_DIRTY_IO_SCHEDULER; +#endif + default: + ERTS_INTERNAL_ERROR("Invalid scheduler type"); + return -1; + } } /* Maps */ diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 52d9f9ddf7..1a30f70588 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -5888,14 +5888,30 @@ erts_sched_set_wake_cleanup_threshold(char *str) static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp) { - if (!esdp) - awdp->sched_id = 0; + int id = 0; + if (esdp) { + switch (esdp->type) { + case ERTS_SCHED_NORMAL: + id = (int) esdp->no; + break; #ifdef ERTS_DIRTY_SCHEDULERS - else if (ERTS_SCHEDULER_IS_DIRTY(esdp)) - awdp->sched_id = (int) ERTS_DIRTY_SCHEDULER_NO(esdp); + case ERTS_SCHED_DIRTY_CPU: + id = (int) erts_no_schedulers; + id += (int) esdp->dirty_no; + break; + case ERTS_SCHED_DIRTY_IO: + id = (int) erts_no_schedulers; + id += (int) erts_no_dirty_cpu_schedulers; + id += (int) esdp->dirty_no; + break; #endif - else - awdp->sched_id = (int) esdp->no; + default: + ERTS_INTERNAL_ERROR("Invalid scheduler type"); + break; + } + } + + awdp->sched_id = id; awdp->esdp = esdp; awdp->ssi = esdp ? esdp->ssi : NULL; #ifdef ERTS_SMP @@ -5968,7 +5984,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, ASSERT(runq == ERTS_DIRTY_IO_RUNQ); esdp->type = ERTS_SCHED_DIRTY_IO; } - ERTS_DIRTY_SCHEDULER_NO(esdp) = (Uint) num; + esdp->dirty_no = (Uint) num; if (num == 1) { /* * Multi-scheduling block functionality depends @@ -5980,7 +5996,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, else { esdp->type = ERTS_SCHED_NORMAL; esdp->no = (Uint) num; - ERTS_DIRTY_SCHEDULER_NO(esdp) = 0; + esdp->dirty_no = 0; runq->scheduler = esdp; } esdp->dirty_shadow_process = shadow_proc; @@ -7717,11 +7733,11 @@ suspend_scheduler(ErtsSchedulerData *esdp) break; case ERTS_SCHED_DIRTY_CPU: online_flag = ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN; - no = ERTS_DIRTY_SCHEDULER_NO(esdp); + no = esdp->dirty_no; break; case ERTS_SCHED_DIRTY_IO: online_flag = 0; - no = ERTS_DIRTY_SCHEDULER_NO(esdp); + no = esdp->dirty_no; break; default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); @@ -8743,8 +8759,7 @@ sched_dirty_cpu_thread_func(void *vesdp) { ErtsThrPrgrCallbacks callbacks; ErtsSchedulerData *esdp = vesdp; - Uint no = ERTS_DIRTY_SCHEDULER_NO(esdp); - ERTS_DIRTY_SCHEDULER_TYPE(esdp) = ERTS_DIRTY_CPU_SCHEDULER; + Uint no = esdp->dirty_no; ASSERT(no != 0); ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(no-1)->event = erts_tse_fetch(); callbacks.arg = (void *) esdp->ssi; @@ -8792,8 +8807,7 @@ sched_dirty_io_thread_func(void *vesdp) { ErtsThrPrgrCallbacks callbacks; ErtsSchedulerData *esdp = vesdp; - Uint no = ERTS_DIRTY_SCHEDULER_NO(esdp); - ERTS_DIRTY_SCHEDULER_TYPE(esdp) = ERTS_DIRTY_IO_SCHEDULER; + Uint no = esdp->dirty_no; ASSERT(no != 0); ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(no-1)->event = erts_tse_fetch(); callbacks.arg = (void *) esdp->ssi; diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 8bf372dad5..5369148f87 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -627,13 +627,6 @@ typedef enum { ERTS_DIRTY_IO_SCHEDULER } ErtsDirtySchedulerType; -typedef union { - struct { - ErtsDirtySchedulerType type: 1; - Uint num: sizeof(Uint)*8 - 1; - } s; - Uint no; -} ErtsDirtySchedId; #endif struct ErtsSchedulerData_ { @@ -660,7 +653,7 @@ struct ErtsSchedulerData_ { ErtsSchedType type; Uint no; /* Scheduler number for normal schedulers */ #ifdef ERTS_DIRTY_SCHEDULERS - ErtsDirtySchedId dirty_no; /* Scheduler number for dirty schedulers */ + Uint dirty_no; /* Scheduler number for dirty schedulers */ Process *dirty_shadow_process; #endif Port *current_port; @@ -1556,23 +1549,13 @@ extern int erts_system_profile_ts_type; #define ERTS_DIRTY_IO_SCHEDULER_IX(IX) \ (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_io_schedulers), \ &erts_aligned_dirty_io_scheduler_data[(IX)].esd) -#define ERTS_DIRTY_SCHEDULER_NO(ESDP) \ - ((ESDP)->dirty_no.s.num) -#define ERTS_DIRTY_SCHEDULER_TYPE(ESDP) \ - ((ESDP)->dirty_no.s.type) -#ifdef ERTS_SMP #define ERTS_SCHEDULER_IS_DIRTY(ESDP) \ - ((ESDP)->dirty_no.s.num != 0) + ((ESDP)->type != ERTS_SCHED_NORMAL) #define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) \ - (ERTS_SCHEDULER_IS_DIRTY((ESDP)) & ((ESDP)->dirty_no.s.type == 0)) + ((ESDP)->type == ERTS_SCHED_DIRTY_CPU) #define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) \ - (ERTS_SCHEDULER_IS_DIRTY((ESDP)) & ((ESDP)->dirty_no.s.type == 1)) -#else -#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0 -#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0 -#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0 -#endif -#else + ((ESDP)->type == ERTS_SCHED_DIRTY_IO) +#else /* !ERTS_DIRTY_SCHEDULERS */ #define ERTS_RUNQ_IX_IS_DIRTY(IX) 0 #define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0 #define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0 -- cgit v1.2.3 From 44b5223489576a6ec719cbdb3bbdb31624326a32 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 24 Mar 2017 18:21:26 +0100 Subject: Fix double hit bug of select/3 with bound key --- erts/emulator/beam/erl_db_hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 0d908fc2c8..b58fe7a185 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -1311,7 +1311,7 @@ static int match_traverse(Process* p, DbTableHash* tb, else if (mpi.key_given) { /* Key is bound */ unlock_hash_function(lck); if (current_list_pos == mpi.num_lists) { - ret_value = on_loop_ended(context_ptr, slot_ix, got, iterations_left, &mpi.mp, ret); + ret_value = on_loop_ended(context_ptr, -1, got, iterations_left, &mpi.mp, ret); goto done; } else { slot_ix = mpi.lists[current_list_pos].ix; -- cgit v1.2.3 From 1db00aa35f99b6876e1049b5162b31acc4f541c1 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Wed, 22 Mar 2017 16:17:43 -0700 Subject: erts: Get rid of some unused function warnings on os x --- erts/emulator/beam/erl_process.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 88fae30845..4fca47d399 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -280,6 +280,7 @@ schdlr_sspnd_get_nscheds(ErtsSchedTypeCounters *valp, } } +#ifdef DEBUG static ERTS_INLINE Uint32 schdlr_sspnd_get_nscheds_tot(ErtsSchedTypeCounters *valp) { @@ -290,6 +291,7 @@ schdlr_sspnd_get_nscheds_tot(ErtsSchedTypeCounters *valp) #endif return res; } +#endif static ERTS_INLINE void schdlr_sspnd_dec_nscheds(ErtsSchedTypeCounters *valp, -- cgit v1.2.3 From 2079ac0b81d7796d3296d82ed4d11b91f699e139 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Mon, 27 Mar 2017 07:13:37 -0700 Subject: erts: Fix erts_debug:df function info output The func_info instruction should also be dumped, so that we know which function is which in the dump. This was accidentally removed when introducing the new codeinfo/codemfa api. --- erts/emulator/beam/beam_debug.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 8326d348af..3f4b0c0c16 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -308,7 +308,7 @@ erts_debug_disassemble_1(BIF_ALIST_1) BIF_RET(am_undef); } } - code_ptr = erts_codemfa_to_code(cmfa); + code_ptr = (BeamInstr*)erts_code_to_codeinfo(erts_codemfa_to_code(cmfa)); } else { goto error; } -- cgit v1.2.3 From 158de690ba539f4048ee0634e1d2ee1ffb76a68f Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Mon, 27 Mar 2017 07:20:19 -0700 Subject: erts: Fix two compiler warnings on OS X --- erts/emulator/beam/erl_process.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 4fca47d399..2cd65b786f 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -11361,7 +11361,7 @@ erts_execute_dirty_system_task(Process *c_p) switch (st->type) { case ERTS_PSTT_CLA: - ASSERT(is_value(st_res)); + ASSERT(is_value(cla_res)); st_res = cla_res; break; case ERTS_PSTT_GC_MAJOR: -- cgit v1.2.3 From eb9fc2a4361037f06991d30f697f0e3ff6394117 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 28 Mar 2017 17:49:10 +0200 Subject: Fix use of uninitialized variable 'ret' DID_TRAP will read 'ret' even when error is returned. Found by valgrind. --- erts/emulator/beam/erl_db_hash.c | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index b58fe7a185..7ab27df00c 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -1247,6 +1247,7 @@ static int match_traverse(Process* p, DbTableHash* tb, if ((ret_value = analyze_pattern(tb, pattern, extra_match_validator, &mpi)) != DB_ERROR_NONE) { + *ret = NIL; goto done; } @@ -1402,8 +1403,10 @@ static int match_traverse_continue(Process* p, DbTableHash* tb, Sint (*next_slot_function)(DbTableHash* tb, Uint ix, erts_smp_rwmtx_t** lck_ptr) = (lock_for_write ? next_slot_w : next_slot); - if (got < 0) + if (got < 0) { + *ret = NIL; return DB_ERROR_BADPARAM; + } if (slot_ix < 0 /* EOT */ || (chunk_size && got >= chunk_size)) @@ -1416,6 +1419,7 @@ static int match_traverse_continue(Process* p, DbTableHash* tb, lck = lock_hash_function(tb, slot_ix); if (slot_ix >= NACTIVE(tb)) { /* Is this possible? */ unlock_hash_function(lck); + *ret = NIL; ret_value = DB_ERROR_BADPARAM; goto done; } @@ -1806,20 +1810,20 @@ static int db_select_continue_hash(Process* p, DbTable* tbl, Eterm continuation, tptr = tuple_val(continuation); if (arityval(*tptr) != 6) - return DB_ERROR_BADPARAM; + goto badparam; if (!is_small(tptr[2]) || !is_small(tptr[3]) || !(is_list(tptr[5]) || tptr[5] == NIL) || !is_small(tptr[6])) - return DB_ERROR_BADPARAM; + goto badparam; if ((chunk_size = signed_val(tptr[3])) < 0) - return DB_ERROR_BADPARAM; + goto badparam; mp = erts_db_get_match_prog_binary(tptr[4]); if (mp == NULL) - return DB_ERROR_BADPARAM; + goto badparam; if ((got = signed_val(tptr[6])) < 0) - return DB_ERROR_BADPARAM; + goto badparam; tid = tptr[1]; slot_ix = signed_val(tptr[2]); @@ -1841,6 +1845,10 @@ static int db_select_continue_hash(Process* p, DbTable* tbl, Eterm continuation, mtraversal_select_chunk_continue_on_loop_ended, mtraversal_select_chunk_on_trap, /* Reuse callback */ &sc_context, ret); + +badparam: + *ret = NIL; + return DB_ERROR_BADPARAM; } #undef MAX_SELECT_CHUNK_ITERATIONS @@ -1933,6 +1941,7 @@ static int db_select_count_continue_hash(Process* p, DbTable* tbl, Eterm continu *ret = NIL; if (unpack_simple_mtraversal_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) { + *ret = NIL; return DB_ERROR_BADPARAM; } @@ -2074,6 +2083,7 @@ static int db_select_delete_continue_hash(Process* p, DbTable* tbl, Eterm contin Sint chunk_size = 0; if (unpack_simple_mtraversal_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) { + *ret = NIL; return DB_ERROR_BADPARAM; } @@ -2217,6 +2227,7 @@ static int db_select_replace_continue_hash(Process* p, DbTable* tbl, Eterm conti *ret = NIL; if (unpack_simple_mtraversal_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) { + *ret = NIL; return DB_ERROR_BADPARAM; } -- cgit v1.2.3 From d71498ac1bfa204138ec7d7ae9fb97d0ffe071b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Tue, 28 Mar 2017 16:29:01 +0200 Subject: erts: Fix faulty assert for refs in copy sharing --- erts/emulator/beam/copy.c | 5 ----- erts/emulator/beam/erl_term.h | 15 +++++++++++++-- 2 files changed, 13 insertions(+), 7 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index e567eabc82..264ba89e8b 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -286,11 +286,6 @@ do { \ (dst) = result; \ } while(0) -#define BOXED_VISITED_MASK ((Eterm) 3) -#define BOXED_VISITED ((Eterm) 1) -#define BOXED_SHARED_UNPROCESSED ((Eterm) 2) -#define BOXED_SHARED_PROCESSED ((Eterm) 3) - #define COUNT_OFF_HEAP (0) /* diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index a602a8f7c6..097d580d99 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -873,6 +873,12 @@ typedef union { ErtsORefThing o; } ErtsRefThing; +/* for copy sharing */ +#define BOXED_VISITED_MASK ((Eterm) 3) +#define BOXED_VISITED ((Eterm) 1) +#define BOXED_SHARED_UNPROCESSED ((Eterm) 2) +#define BOXED_SHARED_PROCESSED ((Eterm) 3) + #define ERTS_REF_THING_SIZE (sizeof(ErtsORefThing)/sizeof(Uint)) #define ERTS_MAGIC_REF_THING_SIZE (sizeof(ErtsMRefThing)/sizeof(Uint)) #define ERTS_MAX_INTERNAL_REF_SIZE (sizeof(ErtsRefThing)/sizeof(Uint)) @@ -888,9 +894,14 @@ typedef union { # define is_ref_thing_header(x) ((x) == ERTS_REF_THING_HEADER) -#define is_ordinary_ref_thing(x) \ - (ASSERT(is_ref_thing_header(*((Eterm *)(x)))), \ +#ifdef SHCOPY +#define is_ordinary_ref_thing(x) \ + (((ErtsRefThing *) (x))->o.marker == ERTS_ORDINARY_REF_MARKER) +#else +#define is_ordinary_ref_thing(x) \ + (ASSERT(is_ref_thing_header((*((Eterm *)(x))) & ~BOXED_VISITED_MASK)), \ ((ErtsRefThing *) (x))->o.marker == ERTS_ORDINARY_REF_MARKER) +#endif #define is_magic_ref_thing(x) \ (!is_ordinary_ref_thing((x))) -- cgit v1.2.3 From 74dc036b67427987f54b45beb3adcccd2cf2e27a Mon Sep 17 00:00:00 2001 From: Manuel Rubio Date: Fri, 31 Mar 2017 01:14:07 +0100 Subject: Add re:version/0 --- erts/emulator/beam/bif.tab | 1 + erts/emulator/beam/erl_bif_re.c | 11 +++++++++++ 2 files changed, 12 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 6f50297fc5..b0819c6427 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -392,6 +392,7 @@ bif erl_ddll:demonitor/1 # # Bifs in the re module # +bif re:version/0 bif re:compile/1 bif re:compile/2 bif re:run/2 diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index a66b05c6ff..5dbf2a7de0 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -477,6 +477,17 @@ build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, con * Compile BIFs */ +BIF_RETTYPE +re_version_0(BIF_ALIST_0) +{ + Eterm ret; + size_t version_size = 0; + byte *version = (byte *) erts_pcre_version(); + version_size = strlen((const char *) version); + ret = new_binary(BIF_P, version, version_size); + BIF_RET(ret); +} + static BIF_RETTYPE re_compile(Process* p, Eterm arg1, Eterm arg2) { -- cgit v1.2.3 From 566ff495cb0f5f87306c81fc8d8ab2323b34d840 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 4 Apr 2017 18:13:22 +0200 Subject: erts: Refactor ErtsCodeInfo.native into union with actual usage types. --- erts/emulator/beam/beam_bp.c | 26 +++++++++++++------------- erts/emulator/beam/beam_emu.c | 6 +++--- erts/emulator/beam/beam_load.c | 12 +++++++----- erts/emulator/beam/code_ix.h | 8 +++++++- erts/emulator/beam/erl_bif_trace.c | 4 ++-- erts/emulator/beam/erl_nif.c | 4 ++-- erts/emulator/beam/export.c | 2 +- 7 files changed, 35 insertions(+), 27 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index b32c74ce7a..1efe7536d6 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -327,7 +327,7 @@ erts_consolidate_bif_bp_data(void) static void consolidate_bp_data(Module* modp, ErtsCodeInfo *ci, int local) { - GenericBp* g = (GenericBp *) ci->native; + GenericBp* g = ci->u.gen_bp; GenericBpData* src; GenericBpData* dst; Uint flags; @@ -376,7 +376,7 @@ consolidate_bp_data(Module* modp, ErtsCodeInfo *ci, int local) ASSERT(*erts_codeinfo_to_code(ci) != (BeamInstr) BeamOp(op_i_generic_breakpoint)); } - ci->native = 0; + ci->u.gen_bp = NULL; Free(g); return; } @@ -427,7 +427,7 @@ erts_install_breakpoints(BpFunctions* f) for (i = 0; i < n; i++) { ErtsCodeInfo* ci = f->matching[i].ci; BeamInstr *pc = erts_codeinfo_to_code(ci); - GenericBp* g = (GenericBp *) ci->native; + GenericBp* g = ci->u.gen_bp; if (*pc != br && g) { Module* modp = f->matching[i].mod; @@ -468,7 +468,7 @@ uninstall_breakpoint(ErtsCodeInfo *ci) { BeamInstr *pc = erts_codeinfo_to_code(ci); if (*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { - GenericBp* g = (GenericBp *) ci->native; + GenericBp* g = ci->u.gen_bp; if (g->data[erts_active_bp_ix()].flags == 0) { /* * The following write is not protected by any lock. We @@ -549,7 +549,7 @@ erts_clear_trace_break(BpFunctions* f) void erts_clear_call_trace_bif(ErtsCodeInfo *ci, int local) { - GenericBp* g = (GenericBp *) ci->native; + GenericBp* g = ci->u.gen_bp; if (g) { Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE; @@ -624,7 +624,7 @@ erts_clear_module_break(Module *modp) { continue; uninstall_breakpoint(ci); consolidate_bp_data(modp, ci, 1); - ASSERT(ci->native == 0); + ASSERT(ci->u.gen_bp == NULL); } return n; } @@ -638,7 +638,7 @@ erts_clear_export_break(Module* modp, ErtsCodeInfo *ci) erts_commit_staged_bp(); *erts_codeinfo_to_code(ci) = (BeamInstr) 0; consolidate_bp_data(modp, ci, 0); - ASSERT(ci->native == 0); + ASSERT(ci->u.gen_bp == NULL); } BeamInstr @@ -651,7 +651,7 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg) ASSERT(info->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - g = (GenericBp *) info->native; + g = info->u.gen_bp; bp = &g->data[ix]; bp_flags = bp->flags; ASSERT((bp_flags & ~ERTS_BPF_ALL) == 0); @@ -754,7 +754,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); - g = (GenericBp *) ep->info.native; + g = ep->info.u.gen_bp; if (g) { bp = &g->data[erts_active_bp_ix()]; bp_flags = bp->flags; @@ -1511,7 +1511,7 @@ set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags, ErtsBpIndex ix = erts_staging_bp_ix(); ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); - g = (GenericBp *) ci->native; + g = ci->u.gen_bp; if (g == 0) { int i; if (count_op == ERTS_BREAK_RESTART || count_op == ERTS_BREAK_PAUSE) { @@ -1523,7 +1523,7 @@ set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags, for (i = 0; i < ERTS_NUM_BP_IX; i++) { g->data[i].flags = 0; } - ci->native = (BeamInstr) g; + ci->u.gen_bp = g; } bp = &g->data[ix]; @@ -1633,7 +1633,7 @@ clear_function_break(ErtsCodeInfo *ci, Uint break_flags) ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); - if ((g = (GenericBp *) ci->native) == 0) { + if ((g = ci->u.gen_bp) == NULL) { return 1; } @@ -1728,7 +1728,7 @@ get_time_break(ErtsCodeInfo *ci) static GenericBpData* check_break(ErtsCodeInfo *ci, Uint break_flags) { - GenericBp* g = (GenericBp *) ci->native; + GenericBp* g = ci->u.gen_bp; ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); if (erts_is_native_break(ci)) { diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 9a91fdce08..3291f92b71 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -4948,14 +4948,14 @@ do { \ */ ErtsCodeInfo *ci = erts_code_to_codeinfo(I); ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.u.ncallee = (void(*)(void)) ci->native; + c_p->hipe.u.ncallee = ci->u.ncallee; ++hipe_trap_count; HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (ci->mfa.arity << 8)); } OpCase(hipe_trap_call_closure): { ErtsCodeInfo *ci = erts_code_to_codeinfo(I); ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.u.ncallee = (void(*)(void)) ci->native; + c_p->hipe.u.ncallee = ci->u.ncallee; ++hipe_trap_count; HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (ci->mfa.arity << 8)); } @@ -5027,7 +5027,7 @@ do { \ * ... remainder of original BEAM code */ ErtsCodeInfo *ci = erts_code_to_codeinfo(I); - struct hipe_call_count *hcc = (struct hipe_call_count*)ci->native; + struct hipe_call_count *hcc = ci->u.hcc; ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); ASSERT(hcc != NULL); ASSERT(VALID_INSTR(hcc->opcode)); diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 6eea963016..ae98b37956 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -5708,12 +5708,13 @@ erts_is_module_native(BeamCodeHeader* code_hdr) static Eterm native_addresses(Process* p, BeamCodeHeader* code_hdr) { + Eterm result = NIL; +#ifdef HIPE int i; Eterm* hp; Uint num_functions; Uint need; Eterm* hp_end; - Eterm result = NIL; num_functions = code_hdr->num_functions; need = (6+BIG_UINT_HEAP_SIZE)*num_functions; @@ -5725,16 +5726,17 @@ native_addresses(Process* p, BeamCodeHeader* code_hdr) ASSERT(is_atom(ci->mfa.function) || is_nil(ci->mfa.function)); /* [] if BIF stub */ - if (ci->native != 0) { + if (ci->u.ncallee != NULL) { Eterm addr; ASSERT(is_atom(ci->mfa.function)); - addr = erts_bld_uint(&hp, NULL, ci->native); + addr = erts_bld_uint(&hp, NULL, (Uint)ci->u.ncallee); tuple = erts_bld_tuple(&hp, NULL, 3, ci->mfa.function, make_small(ci->mfa.arity), addr); result = erts_bld_cons(&hp, NULL, tuple, result); } } HRelease(p, hp_end, hp); +#endif return result; } @@ -6033,7 +6035,7 @@ make_stub(ErtsCodeInfo* info, Eterm mod, Eterm func, Uint arity, Uint native, Be DBG_TRACE_MFA(mod,func,arity,"make beam stub at %p", erts_codeinfo_to_code(info)); ASSERT(WORDS_PER_FUNCTION == 6); info->op = (BeamInstr) BeamOp(op_i_func_info_IaaI); - info->native = native; + info->u.ncallee = (void (*)(void)) native; info->mfa.module = mod; info->mfa.function = func; info->mfa.arity = arity; @@ -6104,7 +6106,7 @@ stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci) Lambda* lp; if (is_bif(ci->mfa.module, ci->mfa.function, ci->mfa.arity)) { - ci->native = 0; + ci->u.ncallee = NULL; ci->mfa.module = 0; ci->mfa.function = 0; ci->mfa.arity = 0; diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h index 1b451bf921..e802ad5dd7 100644 --- a/erts/emulator/beam/code_ix.h +++ b/erts/emulator/beam/code_ix.h @@ -80,7 +80,13 @@ typedef struct ErtsCodeMFA_ { in ops.tab to reflect the new func_info size */ typedef struct ErtsCodeInfo_ { BeamInstr op; /* OpCode(i_func_info) */ - BeamInstr native; /* Used by hipe and trace to store extra data */ + union { + struct generic_bp* gen_bp; /* Trace breakpoint */ +#ifdef HIPE + void (*ncallee)(void); + struct hipe_call_count* hcc; +#endif + }u; ErtsCodeMFA mfa; } ErtsCodeInfo; diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index f471390501..023bfca797 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -1737,7 +1737,7 @@ setup_bif_trace(void) for (i = 0; i < BIF_SIZE; ++i) { Export *ep = bif_export[i]; - GenericBp* g = (GenericBp *) ep->info.native; + GenericBp* g = ep->info.u.gen_bp; if (g) { if (ExportIsBuiltIn(ep)) { ASSERT(ep->beam[1]); @@ -1755,7 +1755,7 @@ reset_bif_trace(void) for (i = 0; i < BIF_SIZE; ++i) { Export *ep = bif_export[i]; - GenericBp* g = (GenericBp *) ep->info.native; + GenericBp* g = ep->info.u.gen_bp; if (g && g->data[active].flags == 0) { if (ExportIsBuiltIn(ep)) { ASSERT(ep->beam[1]); diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 63a4a997da..deed74ee49 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -3653,11 +3653,11 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ci = *get_func_pp(this_mi->code_hdr, f_atom, f->arity); code_ptr = erts_codeinfo_to_code(ci); - if (ci->native == 0) { + if (ci->u.gen_bp == NULL) { code_ptr[0] = (BeamInstr) BeamOp(op_call_nif); } else { /* Function traced, patch the original instruction word */ - GenericBp* g = (GenericBp *) ci->native; + GenericBp* g = ci->u.gen_bp; ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)); g->orig_instr = (BeamInstr) BeamOp(op_call_nif); diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 33ed6d7ec1..ecfef28c57 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -132,7 +132,7 @@ export_alloc(struct export_entry* tmpl_e) erts_smp_atomic_add_nob(&total_entries_bytes, sizeof(*blob)); obj = &blob->exp; obj->info.op = 0; - obj->info.native = 0; + obj->info.u.gen_bp = NULL; obj->info.mfa.module = tmpl->info.mfa.module; obj->info.mfa.function = tmpl->info.mfa.function; obj->info.mfa.arity = tmpl->info.mfa.arity; -- cgit v1.2.3 From 647984f14188ba2cf73128c09e888d4bcf733a9f Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 5 Apr 2017 14:35:56 +0200 Subject: erts: Remove deliberate leak of hipe fun entries should be fixed since 32729cab75325de58bf127e6e8836348071b8682 --- erts/emulator/beam/beam_load.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index ae98b37956..baa8a183d3 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -6269,16 +6269,7 @@ patch_funentries(Eterm Patchlist) fe = erts_get_fun_entry(Mod, uniq, index); fe->native_address = (Uint *)native_address; - /* Deliberate MEMORY LEAK of native fun entries!!! - * - * Uncomment line below when hipe code upgrade and purging works correctly. - * Today we may get cases when old (leaked) native code of a purged module - * gets called and tries to create instances of a deleted fun entry. - * - * Reproduced on a debug emulator with stdlib_test/qlc_SUITE:join_merge - * - * erts_smp_refc_dec(&fe->refc, 1); - */ + erts_smp_refc_dec(&fe->refc, 1); if (!patch(Addresses, (Uint) fe)) return 0; -- cgit v1.2.3 From bb89d065dbe47554c0cb929d244e0d38cad52e65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Fri, 7 Apr 2017 12:04:11 +0200 Subject: beam_emu: Slightly optimize the bxor/2 operator bxor is used in the rand module, so even small optimizations could be worthwile. Suggested by Raimo Niskanen. --- erts/emulator/beam/beam_emu.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 9a91fdce08..6010c17c17 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -3044,10 +3044,12 @@ do { \ GetArg2(2, Op1, Op2); if (is_both_small(Op1, Op2)) { /* - * We could extract the tag from one argument, but a tag extraction - * could mean a shift. Therefore, play it safe here. + * TAG ^ TAG == 0. + * + * Therefore, we perform the XOR operation on the tagged values, + * and OR in the tag bits. */ - Eterm result = make_small(signed_val(Op1) ^ signed_val(Op2)); + Eterm result = (Op1 ^ Op2) | make_small(0); StoreBifResult(4, result); } DO_OUTLINED_ARITH_2(bxor, Op1, Op2); -- cgit v1.2.3 From 8f452530e61b299d4d48f82f41ab5364723607ae Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 5 Apr 2017 14:32:16 +0200 Subject: Stack guard for PCRE --- erts/emulator/beam/beam_debug.c | 46 ++++++++++++++++++++++++++++++++++ erts/emulator/beam/erl_bif_info.c | 15 +++++++++++ erts/emulator/beam/erl_bif_re.c | 35 ++++++++++++++++++++++++++ erts/emulator/beam/erl_init.c | 52 ++++++++++++++++++++++++++++----------- erts/emulator/beam/global.h | 15 +++++++++++ erts/emulator/beam/sys.h | 2 ++ erts/emulator/beam/utils.c | 47 +++++++++++++++++++++++++++++++++++ 7 files changed, 197 insertions(+), 15 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 21d336049f..d708cf89a7 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -731,3 +731,49 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) return size; } + + +#ifdef ERTS_SMP +# define ERTS_STACK_LIMIT ((char *) ethr_get_stacklimit()) +#else +# define ERTS_STACK_LIMIT ((char *) erts_scheduler_stack_limit) +#endif + +/* + * The below functions is for testing of the stack + * limit functionality. They are intentionally + * written body recursive in order to prevent + * last call optimization... + */ + +UWord +erts_check_stack_recursion_downwards(char *start_c) +{ + char *limit = ERTS_STACK_LIMIT; + char c; + UWord res; + if (erts_check_below_limit(&c, limit + 1024)) + return (char *) erts_ptr_id(start_c) - (char *) erts_ptr_id(&c); + res = erts_check_stack_recursion_downwards(start_c); + erts_ptr_id(&c); + return res; +} + +UWord +erts_check_stack_recursion_upwards(char *start_c) +{ + char *limit = ERTS_STACK_LIMIT; + char c; + UWord res; + if (erts_check_above_limit(&c, limit - 1024)) + return (char *) erts_ptr_id(&c) - (char *) erts_ptr_id(start_c); + res = erts_check_stack_recursion_upwards(start_c); + erts_ptr_id(&c); + return res; +} + +int +erts_is_above_stack_limit(char *ptr) +{ + return (char *) ptr > ERTS_STACK_LIMIT; +} diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 7bd45916f5..dd1299ca57 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -3674,6 +3674,21 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(erts_sint64_to_big(value, &hp)); } } + else if (ERTS_IS_ATOM_STR("stack_check", BIF_ARG_1)) { + UWord size; + char c; + if (erts_is_above_stack_limit(&c)) + size = erts_check_stack_recursion_downwards(&c); + else + size = erts_check_stack_recursion_upwards(&c); + if (IS_SSMALL(size)) + BIF_RET(make_small(size)); + else { + Uint hsz = BIG_UWORD_HEAP_SIZE(size); + Eterm *hp = HAlloc(BIF_P, hsz); + BIF_RET(uword_to_big(size, hp)); + } + } } else if (is_tuple(BIF_ARG_1)) { Eterm* tp = tuple_val(BIF_ARG_1); diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index ff7746ce1d..35b196743f 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -64,12 +64,47 @@ static void erts_erts_pcre_stack_free(void *ptr) { erts_free(ERTS_ALC_T_RE_STACK,ptr); } +#define ERTS_PCRE_STACK_MARGIN (10*1024) + +#ifdef ERTS_SMP +# define ERTS_STACK_LIMIT ((char *) ethr_get_stacklimit()) +#else +# define ERTS_STACK_LIMIT ((char *) erts_scheduler_stack_limit) +#endif + +static int +stack_guard_downwards(void) +{ + char *limit = ERTS_STACK_LIMIT; + char c; + + ASSERT(limit); + + return erts_check_below_limit(&c, limit + ERTS_PCRE_STACK_MARGIN); +} + +static int +stack_guard_upwards(void) +{ + char *limit = ERTS_STACK_LIMIT; + char c; + + ASSERT(limit); + + return erts_check_above_limit(&c, limit - ERTS_PCRE_STACK_MARGIN); +} + void erts_init_bif_re(void) { + char c; erts_pcre_malloc = &erts_erts_pcre_malloc; erts_pcre_free = &erts_erts_pcre_free; erts_pcre_stack_malloc = &erts_erts_pcre_stack_malloc; erts_pcre_stack_free = &erts_erts_pcre_stack_free; + if ((char *) erts_ptr_id(&c) > ERTS_STACK_LIMIT) + erts_pcre_stack_guard = stack_guard_downwards; + else + erts_pcre_stack_guard = stack_guard_upwards; default_table = NULL; /* ISO8859-1 default, forced into pcre */ max_loop_limit = CONTEXT_REDS * LOOP_FACTOR; diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index e8b8739852..4e408e8305 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -61,6 +61,9 @@ #define ERTS_DEFAULT_NO_ASYNC_THREADS 10 +#define ERTS_DEFAULT_SCHED_STACK_SIZE 256 +#define ERTS_MIN_SCHED_STACK_SIZE 20 + /* * The variables below (prefixed with etp_) are for erts/etc/unix/etp-commands * only. Do not remove even though they aren't used elsewhere in the emulator! @@ -123,6 +126,8 @@ const Eterm etp_hole_marker = ERTS_HOLE_MARKER; const Eterm etp_hole_marker = 0; #endif +static int modified_sched_thread_suggested_stack_size = 0; + /* * Note about VxWorks: All variables must be initialized by executable code, * not by an initializer. Otherwise a new instance of the emulator will @@ -1231,24 +1236,38 @@ early_init(int *argc, char **argv) /* } #ifndef ERTS_SMP + +void *erts_scheduler_stack_limit; + + static void set_main_stack_size(void) { - if (erts_sched_thread_suggested_stack_size > 0) { + char c; + UWord stacksize; # if HAVE_DECL_GETRLIMIT && HAVE_DECL_SETRLIMIT && HAVE_DECL_RLIMIT_STACK - struct rlimit rl; - int bytes = erts_sched_thread_suggested_stack_size * sizeof(Uint) * 1024; - if (getrlimit(RLIMIT_STACK, &rl) != 0 || - (rl.rlim_cur = bytes, setrlimit(RLIMIT_STACK, &rl) != 0)) { - erts_fprintf(stderr, "failed to set stack size for scheduler " - "thread to %d bytes\n", bytes); - erts_usage(); - } + struct rlimit rl; + int bytes; + stacksize = erts_sched_thread_suggested_stack_size * sizeof(Uint) * 1024; + /* Add some extra pages... neede by some systems... */ + bytes = (int) stacksize + 3*erts_sys_get_page_size(); + if (getrlimit(RLIMIT_STACK, &rl) != 0 || + (rl.rlim_cur = bytes, setrlimit(RLIMIT_STACK, &rl) != 0)) { + erts_fprintf(stderr, "failed to set stack size for scheduler " + "thread to %d bytes\n", bytes); + erts_usage(); + } # else + if (modified_sched_thread_suggested_stack_size) { erts_fprintf(stderr, "no OS support for dynamic stack size limit\n"); - erts_usage(); -# endif + erts_usage(); } + /* Be conservative and hope it is not more than 64 kWords... */ + stacksize = 64*1024*sizeof(void *); +# endif + + erts_scheduler_stack_limit = erts_calc_stacklimit(&c, stacksize); } + #endif void @@ -1293,12 +1312,11 @@ erl_start(int argc, char **argv) port_tab_sz_ignore_files = 1; } -#if (defined(__APPLE__) && defined(__MACH__)) || defined(__DARWIN__) /* - * The default stack size on MacOS X is too small for pcre. + * A default stack size suitable for pcre which might use quite + * a lot of stack. */ - erts_sched_thread_suggested_stack_size = 256; -#endif + erts_sched_thread_suggested_stack_size = ERTS_DEFAULT_SCHED_STACK_SIZE; #ifdef DEBUG verbose = DEBUG_DEFAULT; @@ -1921,6 +1939,7 @@ erl_start(int argc, char **argv) /* suggested stack size (Kilo Words) for scheduler threads */ arg = get_arg(sub_param+2, argv[i+1], &i); erts_sched_thread_suggested_stack_size = atoi(arg); + modified_sched_thread_suggested_stack_size = 1; if ((erts_sched_thread_suggested_stack_size < ERTS_SCHED_THREAD_MIN_STACK_SIZE) @@ -2230,6 +2249,9 @@ erl_start(int argc, char **argv) boot_argc = argc - i; /* Number of arguments to init */ boot_argv = &argv[i]; + if (erts_sched_thread_suggested_stack_size < ERTS_MIN_SCHED_STACK_SIZE) + erts_sched_thread_suggested_stack_size = ERTS_MIN_SCHED_STACK_SIZE; + erl_init(ncpu, proc_tab_sz, legacy_proc_tab, diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 19286e1310..45c01673d8 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1218,6 +1218,11 @@ void erts_short_init(void); void erl_start(int, char**); void erts_usage(void); Eterm erts_preloaded(Process* p); + +#ifndef ERTS_SMP +extern void *erts_scheduler_stack_limit; +#endif + /* erl_md5.c */ typedef struct { @@ -1278,6 +1283,11 @@ Uint64 erts_timestamp_millis(void); Export* erts_find_function(Eterm, Eterm, unsigned int, ErtsCodeIndex); +void *erts_calc_stacklimit(char *prev_c, UWord stacksize); +int erts_check_below_limit(char *ptr, char *limit); +int erts_check_above_limit(char *ptr, char *limit); +void *erts_ptr_id(void *ptr); + Eterm store_external_or_ref_in_proc_(Process *, Eterm); Eterm store_external_or_ref_(Uint **, ErlOffHeap*, Eterm); @@ -1313,6 +1323,11 @@ void erts_init_external(void); /* erl_map.c */ void erts_init_map(void); +/* beam_debug.c */ +UWord erts_check_stack_recursion_downwards(char *start_c); +UWord erts_check_stack_recursion_upwards(char *start_c); +int erts_is_above_stack_limit(char *ptr); + /* erl_unicode.c */ void erts_init_unicode(void); Sint erts_unicode_set_loop_limit(Sint limit); diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index d49edad6dc..41ba51d037 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -604,6 +604,8 @@ __decl_noreturn void __noreturn erts_exit(int n, char*, ...); Eterm erts_check_io_info(void *p); +UWord erts_sys_get_page_size(void); + /* Size of misc memory allocated from system dependent code */ Uint erts_sys_misc_mem_sz(void); diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index d90c282c7e..d91ae1ffe6 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -5031,6 +5031,53 @@ Uint64 erts_timestamp_millis(void) #endif } +void * +erts_calc_stacklimit(char *prev_c, UWord stacksize) +{ + /* + * We *don't* want this function inlined, i.e., it is + * risky to call this function from another function + * in utils.c + */ + + UWord pagesize = erts_sys_get_page_size(); + char c; + char *start; + if (&c > prev_c) { + start = (char *) ((((UWord) prev_c) / pagesize) * pagesize); + return (void *) (start + stacksize); + } + else { + start = (char *) (((((UWord) prev_c) - 1) / pagesize + 1) * pagesize); + return (void *) (start - stacksize); + } +} + +/* + * erts_check_below_limit() and + * erts_check_above_limit() are put + * in utils.c in order to prevent + * inlining. + */ + +int +erts_check_below_limit(char *ptr, char *limit) +{ + return ptr < limit; +} + +int +erts_check_above_limit(char *ptr, char *limit) +{ + return ptr > limit; +} + +void * +erts_ptr_id(void *ptr) +{ + return ptr; +} + #ifdef DEBUG /* * Handy functions when using a debugger - don't use in the code! -- cgit v1.2.3 From 5e8f498b810aeae9cb4c9c3715fe62a33b84cdde Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 11 Apr 2017 15:30:11 +0200 Subject: erts: Init refc=1 in erts_bin_nrml_alloc Only term_to_binary needed some extra attention as it used to initialize refc as 0 instead of 1. --- erts/emulator/beam/beam_emu.c | 2 -- erts/emulator/beam/binary.c | 2 -- erts/emulator/beam/erl_bif_binary.c | 1 - erts/emulator/beam/erl_binary.h | 1 + erts/emulator/beam/erl_bits.c | 3 --- erts/emulator/beam/erl_trace.c | 1 - erts/emulator/beam/external.c | 15 +++++---------- erts/emulator/beam/io.c | 2 -- 8 files changed, 6 insertions(+), 21 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 3291f92b71..321008616d 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -3885,7 +3885,6 @@ do { \ * Allocate the binary struct itself. */ bptr = erts_bin_nrml_alloc(num_bytes); - erts_refc_init(&bptr->refc, 1); erts_current_bin = (byte *) bptr->orig_bytes; /* @@ -3980,7 +3979,6 @@ do { \ * Allocate the binary struct itself. */ bptr = erts_bin_nrml_alloc(BsOp1); - erts_refc_init(&bptr->refc, 1); erts_current_bin = (byte *) bptr->orig_bytes; /* diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index 4dd8316dad..0df6bbb289 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -84,7 +84,6 @@ new_binary(Process *p, byte *buf, Uint len) * Allocate the binary struct itself. */ bptr = erts_bin_nrml_alloc(len); - erts_refc_init(&bptr->refc, 1); if (buf != NULL) { sys_memcpy(bptr->orig_bytes, buf, len); } @@ -121,7 +120,6 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, Uint len) * Allocate the binary struct itself. */ bptr = erts_bin_nrml_alloc(len); - erts_refc_init(&bptr->refc, 1); if (buf != NULL) { sys_memcpy(bptr->orig_bytes, buf, len); } diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index 62a752d854..f79b5b6843 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -2669,7 +2669,6 @@ static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en) } cbs->result = erts_bin_nrml_alloc(target_size); /* Always offheap if trapping */ - erts_refc_init(&(cbs->result->refc), 1); t = (byte *) cbs->result->orig_bytes; /* No offset or anything */ pos = 0; i = 0; diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index 6ff71ec6d1..cb26ee3407 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -411,6 +411,7 @@ erts_bin_nrml_alloc(Uint size) ERTS_CHK_BIN_ALIGNMENT(res); res->orig_size = size; res->flags = 0; + erts_refc_init(&res->refc, 1); return res; } diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index 885e955332..de4fa3594d 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -1404,7 +1404,6 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term, * Allocate the binary data struct itself. */ bptr = erts_bin_nrml_alloc(bin_size); - erts_refc_init(&bptr->refc, 1); erts_current_bin = (byte *) bptr->orig_bytes; /* @@ -1518,7 +1517,6 @@ erts_bs_private_append(Process* p, Eterm bin, Eterm build_size_term, Uint unit) * binary and copy the contents of the old binary into it. */ Binary* bptr = erts_bin_nrml_alloc(new_size); - erts_refc_init(&bptr->refc, 1); sys_memcpy(bptr->orig_bytes, binp->orig_bytes, binp->orig_size); pb->flags |= PB_IS_WRITABLE | PB_ACTIVE_WRITER; pb->val = bptr; @@ -1565,7 +1563,6 @@ erts_bs_init_writable(Process* p, Eterm sz) * Allocate the binary data struct itself. */ bptr = erts_bin_nrml_alloc(bin_size); - erts_refc_init(&bptr->refc, 1); /* * Now allocate the ProcBin on the heap. diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 04f3160d42..85b63208a3 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1875,7 +1875,6 @@ trace_port_tmp_binary(char *bin, Sint sz, Binary **bptrp, Eterm **hp) } else { ProcBin* pb = (ProcBin *)*hp; Binary *bptr = erts_bin_nrml_alloc(sz); - erts_refc_init(&bptr->refc, 1); sys_memcpy(bptr->orig_bytes, bin, sz); pb->thing_word = HEADER_PROC_BIN; pb->size = sz; diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 205a7711ec..873945c3b5 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1821,7 +1821,7 @@ static int ttb_context_destructor(Binary *context_bin) case TTBEncode: DESTROY_SAVED_WSTACK(&context->s.ec.wstack); if (context->s.ec.result_bin != NULL) { /* Set to NULL if ever made alive! */ - ASSERT(erts_refc_read(&(context->s.ec.result_bin->refc),0) == 0); + ASSERT(erts_refc_read(&(context->s.ec.result_bin->refc),1)); erts_bin_free(context->s.ec.result_bin); context->s.ec.result_bin = NULL; } @@ -1830,13 +1830,13 @@ static int ttb_context_destructor(Binary *context_bin) erl_zlib_deflate_finish(&(context->s.cc.stream)); if (context->s.cc.destination_bin != NULL) { /* Set to NULL if ever made alive! */ - ASSERT(erts_refc_read(&(context->s.cc.destination_bin->refc),0) == 0); + ASSERT(erts_refc_read(&(context->s.cc.destination_bin->refc),1)); erts_bin_free(context->s.cc.destination_bin); context->s.cc.destination_bin = NULL; } if (context->s.cc.result_bin != NULL) { /* Set to NULL if ever made alive! */ - ASSERT(erts_refc_read(&(context->s.cc.result_bin->refc),0) == 0); + ASSERT(erts_refc_read(&(context->s.cc.result_bin->refc),1)); erts_bin_free(context->s.cc.result_bin); context->s.cc.result_bin = NULL; } @@ -1920,7 +1920,6 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla } result_bin = erts_bin_nrml_alloc(size); - erts_refc_init(&result_bin->refc, 0); result_bin->orig_bytes[0] = VERSION_MAGIC; /* Next state immediately, no need to export context */ context->state = TTBEncode; @@ -1960,7 +1959,6 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla pb->bytes = (byte*) result_bin->orig_bytes; pb->flags = 0; OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); - erts_refc_inc(&result_bin->refc, 1); if (context_b && erts_refc_read(&context_b->refc,0) == 0) { erts_bin_free(context_b); } @@ -1980,7 +1978,6 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla context->s.cc.result_bin = result_bin; result_bin = erts_bin_nrml_alloc(real_size); - erts_refc_init(&result_bin->refc, 0); result_bin->orig_bytes[0] = VERSION_MAGIC; context->s.cc.destination_bin = result_bin; @@ -2028,10 +2025,10 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla pb->next = MSO(p).first; MSO(p).first = (struct erl_off_heap_header*)pb; pb->val = result_bin; + ASSERT(erts_refc_read(&result_bin->refc, 1)); pb->bytes = (byte*) result_bin->orig_bytes; pb->flags = 0; OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); - erts_refc_inc(&result_bin->refc, 1); erts_bin_free(context->s.cc.result_bin); context->s.cc.result_bin = NULL; context->alive = 0; @@ -2055,7 +2052,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla pb->bytes = (byte*) result_bin->orig_bytes; pb->flags = 0; OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); - erts_refc_inc(&result_bin->refc, 1); + ASSERT(erts_refc_read(&result_bin->refc, 1)); erl_zlib_deflate_finish(&(context->s.cc.stream)); erts_bin_free(context->s.cc.destination_bin); context->s.cc.destination_bin = NULL; @@ -3536,7 +3533,6 @@ dec_term_atom_common: } else { Binary* dbin = erts_bin_nrml_alloc(n); ProcBin* pb; - erts_refc_init(&dbin->refc, 1); pb = (ProcBin *) hp; hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; @@ -3589,7 +3585,6 @@ dec_term_atom_common: Binary* dbin = erts_bin_nrml_alloc(n); ProcBin* pb; - erts_refc_init(&dbin->refc, 1); pb = (ProcBin *) hp; pb->thing_word = HEADER_PROC_BIN; pb->size = n; diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index ddff862607..c85f900de5 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -3791,7 +3791,6 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, Binary* bptr; bptr = erts_bin_nrml_alloc(len); - erts_refc_init(&bptr->refc, 1); sys_memcpy(bptr->orig_bytes, buf, len); pb = (ProcBin *) hp; @@ -6400,7 +6399,6 @@ driver_deliver_term(Port *prt, Eterm to, ErlDrvTermData* data, int len) ProcBin* pbp; Binary* bp = erts_bin_nrml_alloc(size); ASSERT(bufp); - erts_refc_init(&bp->refc, 1); sys_memcpy((void *) bp->orig_bytes, (void *) bufp, size); pbp = (ProcBin *) erts_produce_heap(&factory, PROC_BIN_SIZE, HEAP_EXTRA); -- cgit v1.2.3 From 37bdfb3b5b56c2311db1780b052b7e2c0f139cef Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 11 Apr 2017 16:11:37 +0200 Subject: erts: Init refc=1 in erts_bin_drv_alloc* --- erts/emulator/beam/dist.c | 1 - erts/emulator/beam/erl_binary.h | 2 ++ erts/emulator/beam/erl_nif.c | 1 - erts/emulator/beam/io.c | 1 - 4 files changed, 2 insertions(+), 3 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index d1c2da9074..49a48ac02e 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -629,7 +629,6 @@ alloc_dist_obuf(Uint size) ErtsDistOutputBuf *obuf; Uint obuf_size = sizeof(ErtsDistOutputBuf)+sizeof(byte)*(size-1); Binary *bin = erts_bin_drv_alloc(obuf_size); - erts_refc_init(&bin->refc, 1); obuf = (ErtsDistOutputBuf *) &bin->orig_bytes[0]; #ifdef DEBUG obuf->dbg_pattern = ERTS_DIST_OUTPUT_BUF_DBG_PATTERN; diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index cb26ee3407..51072f4b0f 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -375,6 +375,7 @@ erts_bin_drv_alloc_fnf(Uint size) if (res) { res->orig_size = size; res->flags = BIN_FLAG_DRV; + erts_refc_init(&res->refc, 1); } return res; } @@ -393,6 +394,7 @@ erts_bin_drv_alloc(Uint size) ERTS_CHK_BIN_ALIGNMENT(res); res->orig_size = size; res->flags = BIN_FLAG_DRV; + erts_refc_init(&res->refc, 1); return res; } diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index deed74ee49..94d25ba85a 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1074,7 +1074,6 @@ int enif_alloc_binary(size_t size, ErlNifBinary* bin) if (refbin == NULL) { return 0; /* The NIF must take action */ } - erts_refc_init(&refbin->refc, 1); bin->size = size; bin->data = (unsigned char*) refbin->orig_bytes; diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index c85f900de5..36b8ee6860 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -6938,7 +6938,6 @@ driver_alloc_binary(ErlDrvSizeT size) bin = erts_bin_drv_alloc_fnf((Uint) size); if (!bin) return NULL; /* The driver write must take action */ - erts_refc_init(&bin->refc, 1); return Binary2ErlDrvBinary(bin); } -- cgit v1.2.3 From 486a758ef245effab01d880493a0274de49c1797 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 11 Apr 2017 16:28:12 +0200 Subject: erts: Introduce erts_bin_release --- erts/emulator/beam/beam_load.c | 8 ++------ erts/emulator/beam/dist.c | 3 +-- erts/emulator/beam/erl_binary.h | 9 +++++++++ erts/emulator/beam/erl_bits.c | 4 +--- erts/emulator/beam/erl_db.c | 17 ++++++----------- erts/emulator/beam/erl_db_util.c | 7 ++----- erts/emulator/beam/erl_gc.c | 7 ++----- erts/emulator/beam/erl_message.c | 7 ++----- erts/emulator/beam/erl_nif.c | 12 +++--------- erts/emulator/beam/erl_trace.c | 8 ++++---- erts/emulator/beam/global.h | 4 ++-- erts/emulator/beam/io.c | 6 ++---- 12 files changed, 36 insertions(+), 56 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index baa8a183d3..3a3d1f57ce 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -996,9 +996,7 @@ static void free_loader_state(Binary* magic) { loader_state_dtor(magic); - if (erts_refc_dectest(&magic->refc, 0) == 0) { - erts_bin_free(magic); - } + erts_bin_release(magic); } static ErlHeapFragment* new_literal_fragment(Uint size) @@ -5672,9 +5670,7 @@ erts_release_literal_area(ErtsLiteralArea* literal_area) Binary* bptr; ASSERT(thing_subtag(oh->thing_word) == REFC_BINARY_SUBTAG); bptr = ((ProcBin*)oh)->val; - if (erts_refc_dectest(&bptr->refc, 0) == 0) { - erts_bin_free(bptr); - } + erts_bin_release(bptr); oh = oh->next; } erts_free(ERTS_ALC_T_LITERAL, literal_area); diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 49a48ac02e..a1581908e5 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -642,8 +642,7 @@ free_dist_obuf(ErtsDistOutputBuf *obuf) { Binary *bin = ErtsDistOutputBuf2Binary(obuf); ASSERT(obuf->dbg_pattern == ERTS_DIST_OUTPUT_BUF_DBG_PATTERN); - if (erts_refc_dectest(&bin->refc, 0) == 0) - erts_bin_free(bin); + erts_bin_release(bin); } static ERTS_INLINE Sint diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index 51072f4b0f..eb3f6dfb3b 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -316,6 +316,7 @@ ERTS_GLB_INLINE Binary *erts_bin_nrml_alloc(Uint size); ERTS_GLB_INLINE Binary *erts_bin_realloc_fnf(Binary *bp, Uint size); ERTS_GLB_INLINE Binary *erts_bin_realloc(Binary *bp, Uint size); ERTS_GLB_INLINE void erts_bin_free(Binary *bp); +ERTS_GLB_INLINE void erts_bin_release(Binary *bp); ERTS_GLB_INLINE Binary *erts_create_magic_binary_x(Uint size, int (*destructor)(Binary *), ErtsAlcType_t alloc_type, @@ -469,6 +470,14 @@ erts_bin_free(Binary *bp) erts_free(ERTS_ALC_T_BINARY, (void *) bp); } +ERTS_GLB_INLINE void +erts_bin_release(Binary *bp) +{ + if (erts_refc_dectest(&bp->refc, 0) == 0) { + erts_bin_free(bp); + } +} + ERTS_GLB_INLINE Binary * erts_create_magic_binary_x(Uint size, int (*destructor)(Binary *), ErtsAlcType_t alloc_type, diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index de4fa3594d..df3f6ad557 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -1521,9 +1521,7 @@ erts_bs_private_append(Process* p, Eterm bin, Eterm build_size_term, Uint unit) pb->flags |= PB_IS_WRITABLE | PB_ACTIVE_WRITER; pb->val = bptr; pb->bytes = (byte *) bptr->orig_bytes; - if (erts_refc_dectest(&binp->refc, 0) == 0) { - erts_bin_free(binp); - } + erts_bin_release(binp); } } erts_current_bin = pb->bytes; diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 378328856d..e5601e7571 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -402,8 +402,8 @@ free_dbtable(void *vtb) #endif ASSERT(is_immed(tb->common.heir_data)); - if (tb->common.btid && erts_refc_dectest(&tb->common.btid->refc, 0) == 0) - erts_bin_free(tb->common.btid); + if (tb->common.btid) + erts_bin_release(tb->common.btid); erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable)); } @@ -3616,9 +3616,7 @@ static SWord proc_cleanup_fixed_table(Process* p, DbFixation* fix) ASSERT(fix->counter == 0); } - if (erts_refc_dectest(&fix->tabs.btid->refc, 0) == 0) { - erts_bin_free(fix->tabs.btid); - } + erts_bin_release(fix->tabs.btid); erts_free(ERTS_ALC_T_DB_FIXATION, fix); ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation)); ++work; @@ -3889,9 +3887,7 @@ static void free_fixations_op(DbFixation* fix, void* vctx) { fixed_tabs_delete(fix->procs.p, fix); - if (erts_refc_dectest(&fix->tabs.btid->refc, 0) == 0) { - erts_bin_free(fix->tabs.btid); - } + erts_bin_release(fix->tabs.btid); erts_db_free(ERTS_ALC_T_DB_FIXATION, ctx->tb, (void *) fix, sizeof(DbFixation)); @@ -3906,9 +3902,8 @@ int erts_db_execute_free_fixation(Process* p, DbFixation* fix) ASSERT(fix->counter == 0); fixed_tabs_delete(p, fix); - if (erts_refc_dectest(&fix->tabs.btid->refc, 0) == 0) { - erts_bin_free(fix->tabs.btid); - } + erts_bin_release(fix->tabs.btid); + erts_free(ERTS_ALC_T_DB_FIXATION, fix); ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation)); return 1; diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 03cc11bdc4..24b22eafb8 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -3265,9 +3265,7 @@ void db_cleanup_offheap_comp(DbTerm* obj) } switch (thing_subtag(u.hdr->thing_word)) { case REFC_BINARY_SUBTAG: - if (erts_refc_dectest(&u.pb->val->refc, 0) == 0) { - erts_bin_free(u.pb->val); - } + erts_bin_release(u.pb->val); break; case FUN_SUBTAG: ASSERT(u.pb != &tmp); @@ -3277,8 +3275,7 @@ void db_cleanup_offheap_comp(DbTerm* obj) break; case REF_SUBTAG: ASSERT(is_magic_ref_thing(u.hdr)); - if (erts_refc_dectest(&u.mref->mb->refc, 0) == 0) - erts_bin_free((Binary *)u.mref->mb); + erts_bin_release((Binary *)u.mref->mb); break; default: ASSERT(is_external_header(u.hdr->thing_word)); diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 50805d9cd9..4398da36d8 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -2879,9 +2879,7 @@ sweep_off_heap(Process *p, int fullsweep) case REFC_BINARY_SUBTAG: { Binary* bptr = ((ProcBin*)ptr)->val; - if (erts_refc_dectest(&bptr->refc, 0) == 0) { - erts_bin_free(bptr); - } + erts_bin_release(bptr); break; } case FUN_SUBTAG: @@ -2897,8 +2895,7 @@ sweep_off_heap(Process *p, int fullsweep) ErtsMagicBinary *bptr; ASSERT(is_magic_ref_thing(ptr)); bptr = ((ErtsMRefThing *) ptr)->mb; - if (erts_refc_dectest(&bptr->refc, 0) == 0) - erts_bin_free((Binary *) bptr); + erts_bin_release((Binary *) bptr); break; } default: diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index f181c1e3cb..17982a2d14 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -167,9 +167,7 @@ erts_cleanup_offheap(ErlOffHeap *offheap) for (u.hdr = offheap->first; u.hdr; u.hdr = u.hdr->next) { switch (thing_subtag(u.hdr->thing_word)) { case REFC_BINARY_SUBTAG: - if (erts_refc_dectest(&u.pb->val->refc, 0) == 0) { - erts_bin_free(u.pb->val); - } + erts_bin_release(u.pb->val); break; case FUN_SUBTAG: if (erts_smp_refc_dectest(&u.fun->fe->refc, 0) == 0) { @@ -178,8 +176,7 @@ erts_cleanup_offheap(ErlOffHeap *offheap) break; case REF_SUBTAG: ASSERT(is_magic_ref_thing(u.hdr)); - if (erts_refc_dectest(&u.mref->mb->refc, 0) == 0) - erts_bin_free((Binary *)u.mref->mb); + erts_bin_release((Binary *)u.mref->mb); break; default: ASSERT(is_external_header(u.hdr->thing_word)); diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 94d25ba85a..e76226c5eb 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1112,9 +1112,7 @@ void enif_release_binary(ErlNifBinary* bin) if (bin->ref_bin != NULL) { Binary* refbin = bin->ref_bin; ASSERT(bin->bin_term == THE_NON_VALUE); - if (erts_refc_dectest(&refbin->refc, 0) == 0) { - erts_bin_free(refbin); - } + erts_bin_release(refbin); } #ifdef DEBUG bin->data = NULL; @@ -2347,9 +2345,7 @@ void erts_fire_nif_monitor(ErtsResource* resource, Eterm pid, Eterm ref) resource->type->down(&msg_env.env, resource->data, &nif_pid, &nif_monitor); post_nif_noproc(&msg_env); - if (erts_refc_dectest(&bin->binary.refc, 0) == 0) { - erts_bin_free(&bin->binary); - } + erts_bin_release(&bin->binary); } erts_destroy_monitor(rmon); } @@ -2407,9 +2403,7 @@ void enif_release_resource(void* obj) #ifdef DEBUG erts_refc_dec(&resource->nif_refc, 0); #endif - if (erts_refc_dectest(&bin->binary.refc, 0) == 0) { - erts_bin_free(&bin->binary); - } + erts_bin_release(&bin->binary); } void enif_keep_resource(void* obj) diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 85b63208a3..870f1f142d 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1999,8 +1999,8 @@ trace_port_receive(Port *t_p, Eterm caller, Eterm what, ...) TRACE_FUN_T_RECEIVE, am_receive, data, THE_NON_VALUE, am_true); - if (bptr && erts_refc_dectest(&bptr->refc, 1) == 0) - erts_bin_free(bptr); + if (bptr) + erts_bin_release(bptr); if (orig_hp) erts_free(ERTS_ALC_T_TMP, orig_hp); @@ -2050,8 +2050,8 @@ void trace_port_send_binary(Port *t_p, Eterm to, Eterm what, char *bin, Sint sz) send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_SEND, am_send, msg, to, am_true); - if (bptr && erts_refc_dectest(&bptr->refc, 1) == 0) - erts_bin_free(bptr); + if (bptr) + erts_bin_release(bptr); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index c4c848f49f..4d864d1402 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1416,8 +1416,8 @@ do { \ #define MatchSetUnref(MPSP) \ do { \ - if (((MPSP) != NULL) && erts_refc_dectest(&(MPSP)->refc, 0) <= 0) { \ - erts_bin_free(MPSP); \ + if (((MPSP) != NULL)) { \ + erts_bin_release(MPSP); \ } \ } while(0) diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 36b8ee6860..a1d3f8e29f 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -4557,8 +4557,7 @@ static void cleanup_scheduled_control(Binary *binp, char *bufp) { if (binp) { - if (erts_refc_dectest(&binp->refc, 0) == 0) - erts_bin_free(binp); + erts_bin_release(binp); } else { if (bufp) @@ -6967,8 +6966,7 @@ void driver_free_binary(ErlDrvBinary* dbin) return; bin = ErlDrvBinary2Binary(dbin); - if (erts_refc_dectest(&bin->refc, 0) == 0) - erts_bin_free(bin); + erts_bin_release(bin); } -- cgit v1.2.3 From e2abc143613c83e00c7c2e22721d30fd2f6309a3 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 11 Apr 2017 17:25:49 +0200 Subject: Fix +SDPcpu --- erts/emulator/beam/erl_init.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 56c0f19230..541bfec532 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -1138,8 +1138,12 @@ early_init(int *argc, char **argv) /* } if (dirty_cpu_scheds > schdlrs) dirty_cpu_scheds = schdlrs; + if (dirty_cpu_scheds < 1) + dirty_cpu_scheds = 1; if (dirty_cpu_scheds_online > schdlrs_onln) dirty_cpu_scheds_online = schdlrs_onln; + if (dirty_cpu_scheds_online < 1) + dirty_cpu_scheds_online = 1; #endif } -- cgit v1.2.3 From 53f11c147d7c7a1557e12a7c262c2289d89047aa Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 11 Apr 2017 17:50:10 +0200 Subject: Fix multi-scheduling with only one normal scheduler --- erts/emulator/beam/bif.c | 66 +++++++++++++++++++--------------------- erts/emulator/beam/erl_process.c | 7 +++-- 2 files changed, 35 insertions(+), 38 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 2d37f977c0..214de3652f 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -4506,41 +4506,37 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) || BIF_ARG_2 == am_block_normal); int normal = (BIF_ARG_2 == am_block_normal || BIF_ARG_2 == am_unblock_normal); - if (erts_no_schedulers == 1) - BIF_RET(am_disabled); - else { - switch (erts_block_multi_scheduling(BIF_P, - ERTS_PROC_LOCK_MAIN, - block, - normal, - 0)) { - case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED: - BIF_RET(am_blocked); - case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED: - BIF_RET(am_blocked_normal); - case ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED: - ERTS_BIF_YIELD_RETURN_X(BIF_P, am_blocked, - am_multi_scheduling); - case ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED: - ERTS_BIF_YIELD_RETURN_X(BIF_P, am_blocked_normal, - am_multi_scheduling); - case ERTS_SCHDLR_SSPND_DONE: - BIF_RET(am_enabled); - case ERTS_SCHDLR_SSPND_YIELD_RESTART: - ERTS_VBUMP_ALL_REDS(BIF_P); - BIF_TRAP2(bif_export[BIF_system_flag_2], - BIF_P, BIF_ARG_1, BIF_ARG_2); - case ERTS_SCHDLR_SSPND_YIELD_DONE: - ERTS_BIF_YIELD_RETURN_X(BIF_P, am_enabled, - am_multi_scheduling); - case ERTS_SCHDLR_SSPND_EINVAL: - goto error; - default: - ASSERT(0); - BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); - break; - } - } + switch (erts_block_multi_scheduling(BIF_P, + ERTS_PROC_LOCK_MAIN, + block, + normal, + 0)) { + case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED: + BIF_RET(am_blocked); + case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED: + BIF_RET(am_blocked_normal); + case ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED: + ERTS_BIF_YIELD_RETURN_X(BIF_P, am_blocked, + am_multi_scheduling); + case ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED: + ERTS_BIF_YIELD_RETURN_X(BIF_P, am_blocked_normal, + am_multi_scheduling); + case ERTS_SCHDLR_SSPND_DONE: + BIF_RET(am_enabled); + case ERTS_SCHDLR_SSPND_YIELD_RESTART: + ERTS_VBUMP_ALL_REDS(BIF_P); + BIF_TRAP2(bif_export[BIF_system_flag_2], + BIF_P, BIF_ARG_1, BIF_ARG_2); + case ERTS_SCHDLR_SSPND_YIELD_DONE: + ERTS_BIF_YIELD_RETURN_X(BIF_P, am_enabled, + am_multi_scheduling); + case ERTS_SCHDLR_SSPND_EINVAL: + goto error; + default: + ASSERT(0); + BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); + break; + } #endif } } else if (BIF_ARG_1 == am_schedulers_online) { diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 88fae30845..f0fd3e3dc9 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -8515,8 +8515,9 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal p->flags |= have_blckd_flg; goto wait_until_msb; } - else if (msbp->blckrs) { - ASSERT(msbp->ongoing); + else if (msbp->blckrs || (normal && erts_no_schedulers == 1)) { + ASSERT(!msbp->blckrs || msbp->ongoing); + msbp->ongoing = 1; plp = proclist_create(p); erts_proclist_store_last(&msbp->blckrs, plp); p->flags |= have_blckd_flg; @@ -8530,7 +8531,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal else res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED; } - else { + else { int online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, ERTS_SCHED_NORMAL); ASSERT(!msbp->ongoing); -- cgit v1.2.3 From b5cd902c38687b9578d2222ace7956fdb26c9401 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 1 Mar 2017 17:05:53 +0100 Subject: Fix dirty GC implementation --- erts/emulator/beam/erl_gc.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 50805d9cd9..d51d4fff45 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -479,9 +479,15 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int p->live_hf_end = live_hf_end; } - if (need == 0) + if (need == 0) { +#ifdef ERTS_DIRTY_SCHEDULERS + if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) { + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p))); + goto force_reschedule; + } +#endif return 1; - + } /* * Satisfy need in a heap fragment... */ @@ -534,6 +540,10 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int p->heap_hfrag = hfrag; #endif +#ifdef ERTS_DIRTY_SCHEDULERS +force_reschedule: +#endif + /* Make sure that we do a proper GC as soon as possible... */ p->flags |= F_FORCE_GC; reds_left = ERTS_REDS_LEFT(p, fcalls); -- cgit v1.2.3 From f0c4d095e0d9be054181785f5c9ca34b52aa2995 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 11 Apr 2017 19:15:05 +0200 Subject: Suggested stack size options for dirty schedulers --- erts/emulator/beam/erl_init.c | 72 ++++++++++++++++++++++++++++++++++++---- erts/emulator/beam/erl_process.c | 7 +++- erts/emulator/beam/erl_process.h | 6 +++- 3 files changed, 77 insertions(+), 8 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 56c0f19230..9fe8a7d361 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -61,8 +61,9 @@ #define ERTS_DEFAULT_NO_ASYNC_THREADS 10 -#define ERTS_DEFAULT_SCHED_STACK_SIZE 256 -#define ERTS_MIN_SCHED_STACK_SIZE 20 +#define ERTS_DEFAULT_SCHED_STACK_SIZE 128 +#define ERTS_DEFAULT_DCPU_SCHED_STACK_SIZE 40 +#define ERTS_DEFAULT_DIO_SCHED_STACK_SIZE 40 /* * The variables below (prefixed with etp_) are for erts/etc/unix/etp-commands @@ -641,9 +642,22 @@ void erts_usage(void) erts_fprintf(stderr, "-swt val set scheduler wakeup threshold, valid values are:\n"); erts_fprintf(stderr, " very_low|low|medium|high|very_high.\n"); erts_fprintf(stderr, "-sss size suggested stack size in kilo words for scheduler threads,\n"); - erts_fprintf(stderr, " valid range is [%d-%d]\n", + erts_fprintf(stderr, " valid range is [%d-%d] (default %d)\n", + ERTS_SCHED_THREAD_MIN_STACK_SIZE, + ERTS_SCHED_THREAD_MAX_STACK_SIZE, + ERTS_DEFAULT_SCHED_STACK_SIZE); +#ifdef ERTS_DIRTY_SCHEDULERS + erts_fprintf(stderr, "-sssdcpu size suggested stack size in kilo words for dirty CPU scheduler\n"); + erts_fprintf(stderr, " threads, valid range is [%d-%d] (default %d)\n", + ERTS_SCHED_THREAD_MIN_STACK_SIZE, + ERTS_SCHED_THREAD_MAX_STACK_SIZE, + ERTS_DEFAULT_DCPU_SCHED_STACK_SIZE); + erts_fprintf(stderr, "-sssdio size suggested stack size in kilo words for dirty IO scheduler\n"); + erts_fprintf(stderr, " threads, valid range is [%d-%d] (default %d)\n", ERTS_SCHED_THREAD_MIN_STACK_SIZE, - ERTS_SCHED_THREAD_MAX_STACK_SIZE); + ERTS_SCHED_THREAD_MAX_STACK_SIZE, + ERTS_DEFAULT_DIO_SCHED_STACK_SIZE); +#endif erts_fprintf(stderr, "-spp Bool set port parallelism scheduling hint\n"); erts_fprintf(stderr, "-S n1:n2 set number of schedulers (n1), and number of\n"); erts_fprintf(stderr, " schedulers online (n2), maximum for both\n"); @@ -1323,6 +1337,10 @@ erl_start(int argc, char **argv) * a lot of stack. */ erts_sched_thread_suggested_stack_size = ERTS_DEFAULT_SCHED_STACK_SIZE; +#ifdef ERTS_DIRTY_SCHEDULERS + erts_dcpu_sched_thread_suggested_stack_size = ERTS_DEFAULT_DCPU_SCHED_STACK_SIZE; + erts_dio_sched_thread_suggested_stack_size = ERTS_DEFAULT_DIO_SCHED_STACK_SIZE; +#endif #ifdef DEBUG verbose = DEBUG_DEFAULT; @@ -1941,6 +1959,42 @@ erl_start(int argc, char **argv) VERBOSE(DEBUG_SYSTEM, ("scheduler wakeup threshold: %s\n", arg)); } +#ifdef ERTS_DIRTY_SCHEDULERS + else if (has_prefix("ssdcpu", sub_param)) { + /* suggested stack size (Kilo Words) for dirty CPU scheduler threads */ + arg = get_arg(sub_param+6, argv[i+1], &i); + erts_dcpu_sched_thread_suggested_stack_size = atoi(arg); + + if ((erts_dcpu_sched_thread_suggested_stack_size + < ERTS_SCHED_THREAD_MIN_STACK_SIZE) + || (erts_dcpu_sched_thread_suggested_stack_size > + ERTS_SCHED_THREAD_MAX_STACK_SIZE)) { + erts_fprintf(stderr, "bad stack size for dirty CPU scheduler threads %s\n", + arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, + ("suggested dirty CPU scheduler thread stack size %d kilo words\n", + erts_dcpu_sched_thread_suggested_stack_size)); + } + else if (has_prefix("ssdio", sub_param)) { + /* suggested stack size (Kilo Words) for dirty IO scheduler threads */ + arg = get_arg(sub_param+5, argv[i+1], &i); + erts_dio_sched_thread_suggested_stack_size = atoi(arg); + + if ((erts_dio_sched_thread_suggested_stack_size + < ERTS_SCHED_THREAD_MIN_STACK_SIZE) + || (erts_dio_sched_thread_suggested_stack_size > + ERTS_SCHED_THREAD_MAX_STACK_SIZE)) { + erts_fprintf(stderr, "bad stack size for dirty IO scheduler threads %s\n", + arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, + ("suggested dirty IO scheduler thread stack size %d kilo words\n", + erts_dio_sched_thread_suggested_stack_size)); + } +#endif else if (has_prefix("ss", sub_param)) { /* suggested stack size (Kilo Words) for scheduler threads */ arg = get_arg(sub_param+2, argv[i+1], &i); @@ -2255,8 +2309,14 @@ erl_start(int argc, char **argv) boot_argc = argc - i; /* Number of arguments to init */ boot_argv = &argv[i]; - if (erts_sched_thread_suggested_stack_size < ERTS_MIN_SCHED_STACK_SIZE) - erts_sched_thread_suggested_stack_size = ERTS_MIN_SCHED_STACK_SIZE; + if (erts_sched_thread_suggested_stack_size < ERTS_SCHED_THREAD_MIN_STACK_SIZE) + erts_sched_thread_suggested_stack_size = ERTS_SCHED_THREAD_MIN_STACK_SIZE; +#ifdef ERTS_DIRTY_SCHEDULERS + if (erts_dcpu_sched_thread_suggested_stack_size < ERTS_SCHED_THREAD_MIN_STACK_SIZE) + erts_dcpu_sched_thread_suggested_stack_size = ERTS_SCHED_THREAD_MIN_STACK_SIZE; + if (erts_dio_sched_thread_suggested_stack_size < ERTS_SCHED_THREAD_MIN_STACK_SIZE) + erts_dio_sched_thread_suggested_stack_size = ERTS_SCHED_THREAD_MIN_STACK_SIZE; +#endif erl_init(ncpu, proc_tab_sz, diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 11a1ce625b..2d80617445 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -194,7 +194,10 @@ static UWord thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_O ErtsPTab erts_proc erts_align_attribute(ERTS_CACHE_LINE_SIZE); int erts_sched_thread_suggested_stack_size = -1; - +#ifdef ERTS_DIRTY_SCHEDULERS +int erts_dcpu_sched_thread_suggested_stack_size = -1; +int erts_dio_sched_thread_suggested_stack_size = -1; +#endif #ifdef ERTS_ENABLE_LOCK_CHECK ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE]; #endif @@ -8976,6 +8979,7 @@ erts_start_schedulers(void) for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) { ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); erts_snprintf(opts.name, 16, "%d_dirty_cpu_scheduler", ix + 1); + opts.suggested_stack_size = erts_dcpu_sched_thread_suggested_stack_size; res = ethr_thr_create(&esdp->tid,sched_dirty_cpu_thread_func,(void*)esdp,&opts); if (res != 0) erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty cpu scheduler thread %d\n", ix); @@ -8983,6 +8987,7 @@ erts_start_schedulers(void) for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix); erts_snprintf(opts.name, 16, "%d_dirty_io_scheduler", ix + 1); + opts.suggested_stack_size = erts_dio_sched_thread_suggested_stack_size; res = ethr_thr_create(&esdp->tid,sched_dirty_io_thread_func,(void*)esdp,&opts); if (res != 0) erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty io scheduler thread %d\n", ix); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 883d9f2a4c..408e9c4c45 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -117,7 +117,11 @@ extern Uint erts_no_dirty_io_schedulers; #endif extern Uint erts_no_run_queues; extern int erts_sched_thread_suggested_stack_size; -#define ERTS_SCHED_THREAD_MIN_STACK_SIZE 4 /* Kilo words */ +#ifdef ERTS_DIRTY_SCHEDULERS +extern int erts_dcpu_sched_thread_suggested_stack_size; +extern int erts_dio_sched_thread_suggested_stack_size; +#endif +#define ERTS_SCHED_THREAD_MIN_STACK_SIZE 20 /* Kilo words */ #define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192 /* Kilo words */ #ifdef ERTS_SMP -- cgit v1.2.3 From 56c4aee677f305f2ed9ca877a39e9c3c4f266f4b Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 12 Apr 2017 19:50:01 +0200 Subject: erts: Introduce struct binary_internals to replace macro ERTS_INTERNAL_BINARY_FIELDS as header in Binary and friends. --- erts/emulator/beam/beam_bif_load.c | 4 +-- erts/emulator/beam/beam_load.c | 2 +- erts/emulator/beam/break.c | 2 +- erts/emulator/beam/copy.c | 16 +++++------ erts/emulator/beam/erl_bif_info.c | 6 ++--- erts/emulator/beam/erl_bif_unique.c | 2 +- erts/emulator/beam/erl_bif_unique.h | 10 +++---- erts/emulator/beam/erl_binary.h | 50 ++++++++++++++++++----------------- erts/emulator/beam/erl_db.c | 6 ++--- erts/emulator/beam/erl_db_hash.c | 4 +-- erts/emulator/beam/erl_db_tree.c | 12 ++++----- erts/emulator/beam/erl_db_util.h | 6 ++--- erts/emulator/beam/erl_gc.c | 6 ++--- erts/emulator/beam/erl_nif.c | 16 +++++------ erts/emulator/beam/erl_process_dump.c | 6 ++--- erts/emulator/beam/external.c | 22 +++++++-------- erts/emulator/beam/global.h | 2 +- erts/emulator/beam/io.c | 8 +++--- erts/emulator/beam/utils.c | 2 +- 19 files changed, 92 insertions(+), 90 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 4ba8c2a669..769fe89219 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -233,7 +233,7 @@ prepare_loading_2(BIF_ALIST_2) } hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE); res = erts_mk_magic_ref(&hp, &MSO(BIF_P), magic); - erts_refc_dec(&magic->refc, 1); + erts_refc_dec(&magic->intern.refc, 1); BIF_RET(res); } @@ -435,7 +435,7 @@ finish_loading_1(BIF_ALIST_1) Eterm mod; Eterm retval; - erts_refc_inc(&p[i].code->refc, 1); + erts_refc_inc(&p[i].code->intern.refc, 1); retval = erts_finish_loading(p[i].code, BIF_P, 0, &mod); ASSERT(retval == NIL || retval == am_on_load); if (retval == am_on_load) { diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 3a3d1f57ce..7b79e6303b 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -913,7 +913,7 @@ erts_alloc_loader_state(void) magic = erts_create_magic_binary(sizeof(LoaderState), loader_state_dtor); - erts_refc_inc(&magic->refc, 1); + erts_refc_inc(&magic->intern.refc, 1); stp = ERTS_MAGIC_BIN_DATA(magic); stp->bin = NULL; stp->function = THE_NON_VALUE; /* Function not known yet */ diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 0b40d70cb7..4703fb0933 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -660,7 +660,7 @@ bin_check(void) erts_printf("%p orig_size: %bpd, norefs = %bpd\n", bp->val, bp->val->orig_size, - erts_refc_read(&bp->val->refc, 1)); + erts_refc_read(&bp->val->intern.refc, 1)); } } if (printed) { diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 264ba89e8b..62c3f9520d 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -761,7 +761,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint } *argp = make_binary(hbot); pb = (ProcBin*) hbot; - erts_refc_inc(&pb->val->refc, 2); + erts_refc_inc(&pb->val->intern.refc, 2); pb->next = off_heap->first; pb->flags = 0; off_heap->first = (struct erl_off_heap_header*) pb; @@ -809,7 +809,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint to->thing_word = HEADER_PROC_BIN; to->size = real_size; to->val = from->val; - erts_refc_inc(&to->val->refc, 2); + erts_refc_inc(&to->val->intern.refc, 2); to->bytes = from->bytes + offset; to->next = off_heap->first; to->flags = 0; @@ -901,7 +901,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint case REF_SUBTAG: if (is_magic_ref_thing(objp)) { ErtsMRefThing *mreft = (ErtsMRefThing *) objp; - erts_refc_inc(&mreft->mb->refc, 2); + erts_refc_inc(&mreft->mb->intern.refc, 2); goto L_off_heap_node_container_common; } /* Fall through... */ @@ -1585,7 +1585,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, while (sz-- > 0) { *hp++ = *ptr++; } - erts_refc_inc(&pb->val->refc, 2); + erts_refc_inc(&pb->val->intern.refc, 2); pb->next = off_heap->first; pb->flags = 0; off_heap->first = (struct erl_off_heap_header*) pb; @@ -1644,7 +1644,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, to->thing_word = HEADER_PROC_BIN; to->size = real_size; to->val = from->val; - erts_refc_inc(&to->val->refc, 2); + erts_refc_inc(&to->val->intern.refc, 2); to->bytes = from->bytes + offset; to->next = off_heap->first; to->flags = 0; @@ -1678,7 +1678,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, case REF_SUBTAG: if (is_magic_ref_thing(ptr)) { ErtsMRefThing *mreft = (ErtsMRefThing *) ptr; - erts_refc_inc(&mreft->mb->refc, 2); + erts_refc_inc(&mreft->mb->intern.refc, 2); goto off_heap_node_container_common; } /* Fall through... */ @@ -1847,7 +1847,7 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) case REFC_BINARY_SUBTAG: { ProcBin* pb = (ProcBin *) (tp-1); - erts_refc_inc(&pb->val->refc, 2); + erts_refc_inc(&pb->val->intern.refc, 2); OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); } goto off_heap_common; @@ -1882,7 +1882,7 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) ErtsRefThing *rtp = (ErtsRefThing *) (tp - 1); if (is_magic_ref_thing(rtp)) { ErtsMRefThing *mreft = (ErtsMRefThing *) rtp; - erts_refc_inc(&mreft->mb->refc, 2); + erts_refc_inc(&mreft->mb->intern.refc, 2); goto off_heap_common; } /* Fall through... */ diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 025f99330a..6d10ced87a 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -177,7 +177,7 @@ bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh) if (szp) *szp += 4+2; if (hpp) { - Uint refc = (Uint) erts_refc_read(&pb->val->refc, 1); + Uint refc = (Uint) erts_refc_read(&pb->val->intern.refc, 1); tuple = TUPLE3(*hpp, val, orig_size, make_small(refc)); res = CONS(*hpp + 4, tuple, res); *hpp += 4+2; @@ -203,7 +203,7 @@ bld_magic_ref_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh) if (szp) *szp += 4+2; if (hpp) { - Uint refc = (Uint) erts_refc_read(&mrtp->mb->refc, 1); + Uint refc = (Uint) erts_refc_read(&mrtp->mb->intern.refc, 1); tuple = TUPLE3(*hpp, val, orig_size, make_small(refc)); res = CONS(*hpp + 4, tuple, res); *hpp += 4+2; @@ -3969,7 +3969,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(am_false); } bin = erts_magic_ref2bin(tp[2]); - refc = erts_refc_read(&bin->refc, 1); + refc = erts_refc_read(&bin->intern.refc, 1); bin_addr = (UWord) bin; sz = 4; erts_bld_uword(NULL, &sz, bin_addr); diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c index 3fd4c87094..7f438daec0 100644 --- a/erts/emulator/beam/erl_bif_unique.c +++ b/erts/emulator/beam/erl_bif_unique.c @@ -197,7 +197,7 @@ erts_magic_ref_lookup_bin__(Uint32 refn[ERTS_REF_NUMBERS]) else { erts_aint_t refc; mb = tep->mb; - refc = erts_refc_inc_unless(&mb->refc, 0, 0); + refc = erts_refc_inc_unless(&mb->intern.refc, 0, 0); if (refc == 0) mb = NULL; } diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h index 27c2a15a5e..0f3f794878 100644 --- a/erts/emulator/beam/erl_bif_unique.h +++ b/erts/emulator/beam/erl_bif_unique.h @@ -178,10 +178,10 @@ ERTS_GLB_INLINE Eterm erts_mk_magic_ref(Eterm **hpp, ErlOffHeap *ohp, Binary *bp) { Eterm *hp = *hpp; - ASSERT(bp->flags & BIN_FLAG_MAGIC); + ASSERT(bp->intern.flags & BIN_FLAG_MAGIC); write_magic_ref_thing(hp, ohp, (ErtsMagicBinary *) bp); *hpp += ERTS_MAGIC_REF_THING_SIZE; - erts_refc_inc(&bp->refc, 1); + erts_refc_inc(&bp->intern.refc, 1); OH_OVERHEAD(ohp, bp->orig_size / sizeof(Eterm)); return make_internal_ref(hp); } @@ -297,14 +297,14 @@ erts_iref_storage_save(ErtsIRefStorage *iref, Eterm ref) ASSERT(is_magic_ref_thing(hp)); iref->is_magic = 1; iref->u.mb = mrtp->mb; - erts_refc_inc(&mrtp->mb->refc, 1); + erts_refc_inc(&mrtp->mb->intern.refc, 1); } } ERTS_GLB_INLINE void erts_iref_storage_clean(ErtsIRefStorage *iref) { - if (iref->is_magic && erts_refc_dectest(&iref->u.mb->refc, 0) == 0) + if (iref->is_magic && erts_refc_dectest(&iref->u.mb->intern.refc, 0) == 0) erts_ref_bin_free(iref->u.mb); #ifdef DEBUG memset((void *) iref, 0xf, sizeof(ErtsIRefStorage)); @@ -337,7 +337,7 @@ erts_iref_storage_make_ref(ErtsIRefStorage *iref, * refc increment of the cleaned storage... */ if (!clean_storage) - erts_refc_inc(&iref->u.mb->refc, 1); + erts_refc_inc(&iref->u.mb->intern.refc, 1); } #ifdef DEBUG diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index eb3f6dfb3b..de7dbf4e20 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -42,14 +42,16 @@ #define ERTS_BINARY_STRUCT_ALIGNMENT #endif -/* Add fields in ERTS_INTERNAL_BINARY_FIELDS, otherwise the drivers crash */ -#define ERTS_INTERNAL_BINARY_FIELDS \ - UWord flags; \ - erts_refc_t refc; \ +/* Add fields in binary_internals, otherwise the drivers crash */ +struct binary_internals { + UWord flags; + erts_refc_t refc; ERTS_BINARY_STRUCT_ALIGNMENT +}; + typedef struct binary { - ERTS_INTERNAL_BINARY_FIELDS + struct binary_internals intern; SWord orig_size; char orig_bytes[1]; /* to be continued */ } Binary; @@ -63,7 +65,7 @@ typedef struct binary { typedef struct magic_binary ErtsMagicBinary; struct magic_binary { - ERTS_INTERNAL_BINARY_FIELDS + struct binary_internals intern; SWord orig_size; int (*destructor)(Binary *); Uint32 refn[ERTS_REF_NUMBERS]; @@ -87,7 +89,7 @@ typedef union { Binary binary; ErtsMagicBinary magic_binary; struct { - ERTS_INTERNAL_BINARY_FIELDS + struct binary_internals intern; ErlDrvBinary binary; } driver; } ErtsBinary; @@ -375,8 +377,8 @@ erts_bin_drv_alloc_fnf(Uint size) ERTS_CHK_BIN_ALIGNMENT(res); if (res) { res->orig_size = size; - res->flags = BIN_FLAG_DRV; - erts_refc_init(&res->refc, 1); + res->intern.flags = BIN_FLAG_DRV; + erts_refc_init(&res->intern.refc, 1); } return res; } @@ -394,8 +396,8 @@ erts_bin_drv_alloc(Uint size) res = erts_alloc(ERTS_ALC_T_DRV_BINARY, bsize); ERTS_CHK_BIN_ALIGNMENT(res); res->orig_size = size; - res->flags = BIN_FLAG_DRV; - erts_refc_init(&res->refc, 1); + res->intern.flags = BIN_FLAG_DRV; + erts_refc_init(&res->intern.refc, 1); return res; } @@ -413,8 +415,8 @@ erts_bin_nrml_alloc(Uint size) res = erts_alloc(ERTS_ALC_T_BINARY, bsize); ERTS_CHK_BIN_ALIGNMENT(res); res->orig_size = size; - res->flags = 0; - erts_refc_init(&res->refc, 1); + res->intern.flags = 0; + erts_refc_init(&res->intern.refc, 1); return res; } @@ -423,9 +425,9 @@ erts_bin_realloc_fnf(Binary *bp, Uint size) { Binary *nbp; Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD; - ErtsAlcType_t type = (bp->flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY + ErtsAlcType_t type = (bp->intern.flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY : ERTS_ALC_T_BINARY; - ASSERT((bp->flags & BIN_FLAG_MAGIC) == 0); + ASSERT((bp->intern.flags & BIN_FLAG_MAGIC) == 0); if (bsize < size) /* overflow */ return NULL; nbp = erts_realloc_fnf(type, (void *) bp, bsize); @@ -440,9 +442,9 @@ erts_bin_realloc(Binary *bp, Uint size) { Binary *nbp; Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD; - ErtsAlcType_t type = (bp->flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY + ErtsAlcType_t type = (bp->intern.flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY : ERTS_ALC_T_BINARY; - ASSERT((bp->flags & BIN_FLAG_MAGIC) == 0); + ASSERT((bp->intern.flags & BIN_FLAG_MAGIC) == 0); if (bsize < size) /* overflow */ erts_realloc_enomem(type, bp, size); nbp = erts_realloc_fnf(type, (void *) bp, bsize); @@ -456,7 +458,7 @@ erts_bin_realloc(Binary *bp, Uint size) ERTS_GLB_INLINE void erts_bin_free(Binary *bp) { - if (bp->flags & BIN_FLAG_MAGIC) { + if (bp->intern.flags & BIN_FLAG_MAGIC) { if (!ERTS_MAGIC_BIN_DESTRUCTOR(bp)(bp)) { /* Destructor took control of the deallocation */ return; @@ -464,7 +466,7 @@ erts_bin_free(Binary *bp) erts_magic_ref_remove_bin(ERTS_MAGIC_BIN_REFN(bp)); erts_free(ERTS_MAGIC_BIN_ATYPE(bp), (void *) bp); } - else if (bp->flags & BIN_FLAG_DRV) + else if (bp->intern.flags & BIN_FLAG_DRV) erts_free(ERTS_ALC_T_DRV_BINARY, (void *) bp); else erts_free(ERTS_ALC_T_BINARY, (void *) bp); @@ -473,7 +475,7 @@ erts_bin_free(Binary *bp) ERTS_GLB_INLINE void erts_bin_release(Binary *bp) { - if (erts_refc_dectest(&bp->refc, 0) == 0) { + if (erts_refc_dectest(&bp->intern.refc, 0) == 0) { erts_bin_free(bp); } } @@ -490,10 +492,10 @@ erts_create_magic_binary_x(Uint size, int (*destructor)(Binary *), if (!bptr) erts_alloc_n_enomem(ERTS_ALC_T2N(alloc_type), bsize); ERTS_CHK_BIN_ALIGNMENT(bptr); - bptr->flags = BIN_FLAG_MAGIC; + bptr->intern.flags = BIN_FLAG_MAGIC; bptr->orig_size = unaligned ? ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(size) : ERTS_MAGIC_BIN_ORIG_SIZE(size); - erts_refc_init(&bptr->refc, 0); + erts_refc_init(&bptr->intern.refc, 0); ERTS_MAGIC_BIN_DESTRUCTOR(bptr) = destructor; ERTS_MAGIC_BIN_ATYPE(bptr) = alloc_type; erts_make_magic_ref_in_array(ERTS_MAGIC_BIN_REFN(bptr)); @@ -521,7 +523,7 @@ ERTS_GLB_INLINE erts_smp_atomic_t * erts_smp_binary_to_magic_indirection(Binary *bp) { ErtsMagicIndirectionWord *mip; - ASSERT(bp->flags & BIN_FLAG_MAGIC); + ASSERT(bp->intern.flags & BIN_FLAG_MAGIC); ASSERT(ERTS_MAGIC_BIN_ATYPE(bp) == ERTS_ALC_T_MINDIRECTION); mip = ERTS_MAGIC_BIN_UNALIGNED_DATA(bp); return &mip->smp_atomic_word; @@ -531,7 +533,7 @@ ERTS_GLB_INLINE erts_atomic_t * erts_binary_to_magic_indirection(Binary *bp) { ErtsMagicIndirectionWord *mip; - ASSERT(bp->flags & BIN_FLAG_MAGIC); + ASSERT(bp->intern.flags & BIN_FLAG_MAGIC); ASSERT(ERTS_MAGIC_BIN_ATYPE(bp) == ERTS_ALC_T_MINDIRECTION); mip = ERTS_MAGIC_BIN_UNALIGNED_DATA(bp); return &mip->atomic_word; diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index e5601e7571..98c689f13f 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -217,7 +217,7 @@ make_btid(DbTable *tb) * and table is refered once by being alive... */ erts_smp_refc_init(&tb->common.refc, 2); - erts_refc_inc(&btid->refc, 1); + erts_refc_inc(&btid->intern.refc, 1); } static ERTS_INLINE DbTable* btid2tab(Binary* btid) @@ -3781,7 +3781,7 @@ static void fix_table_locked(Process* p, DbTable* tb) tb, sizeof(DbFixation)); ERTS_ETS_MISC_MEM_ADD(sizeof(DbFixation)); fix->tabs.btid = tb->common.btid; - erts_refc_inc(&fix->tabs.btid->refc, 2); + erts_refc_inc(&fix->tabs.btid->intern.refc, 2); fix->procs.p = p; fix->counter = 1; fixing_procs_rbt_insert(&tb->common.fixing_procs, fix); @@ -3817,7 +3817,7 @@ static void unfix_table_locked(Process* p, DbTable* tb, #endif fixed_tabs_delete(p, fix); - erts_refc_dec(&fix->tabs.btid->refc, 1); + erts_refc_dec(&fix->tabs.btid->intern.refc, 1); erts_db_free(ERTS_ALC_T_DB_FIXATION, tb, (void *) fix, sizeof(DbFixation)); diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 7ab27df00c..80c4824eeb 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -1258,7 +1258,7 @@ static int match_traverse(Process* p, DbTableHash* tb, } if (mpi.all_objects) { - mpi.mp->flags |= BIN_FLAG_ALL_OBJECTS; + mpi.mp->intern.flags |= BIN_FLAG_ALL_OBJECTS; } /* @@ -1383,7 +1383,7 @@ static int match_traverse_continue(Process* p, DbTableHash* tb, void* context_ptr, /* For callbacks */ Eterm* ret) { - int all_objects = (*mpp)->flags & BIN_FLAG_ALL_OBJECTS; + int all_objects = (*mpp)->intern.flags & BIN_FLAG_ALL_OBJECTS; HashDbTerm** current_ptr; /* Refers to either the bucket pointer or * the 'next' pointer in the previous term */ diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index ab8da6ccf6..f6918b8ec4 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -995,7 +995,7 @@ static int db_select_continue_tree(Process *p, sc.lastobj = NULL; sc.max = 1000; sc.keypos = tb->common.keypos; - sc.all_objects = mp->flags & BIN_FLAG_ALL_OBJECTS; + sc.all_objects = mp->intern.flags & BIN_FLAG_ALL_OBJECTS; sc.chunk_size = chunk_size; reverse = unsigned_val(tptr[7]); sc.got = signed_val(tptr[8]); @@ -1187,7 +1187,7 @@ static int db_select_tree(Process *p, DbTable *tbl, Eterm tid, hp = HAlloc(p, 9 + sz + ERTS_MAGIC_REF_THING_SIZE); key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) - (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; + (mpi.mp)->intern.flags |= BIN_FLAG_ALL_OBJECTS; mpb= erts_db_make_match_prog_ref(p,mpi.mp,&hp); continuation = TUPLE8 @@ -1385,7 +1385,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, Eterm tid, } key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) - (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; + (mpi.mp)->intern.flags |= BIN_FLAG_ALL_OBJECTS; mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp); continuation = TUPLE5 @@ -1510,7 +1510,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, Eterm tid, hp = HAlloc(p, 9 + sz + ERTS_MAGIC_REF_THING_SIZE); key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) - (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; + (mpi.mp)->intern.flags |= BIN_FLAG_ALL_OBJECTS; mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp); continuation = TUPLE8 @@ -1536,7 +1536,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, Eterm tid, key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) - (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; + (mpi.mp)->intern.flags |= BIN_FLAG_ALL_OBJECTS; mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp); continuation = TUPLE8 (hp, @@ -1932,7 +1932,7 @@ static int db_select_replace_tree(Process *p, DbTable *tbl, Eterm tid, } key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) - (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; + (mpi.mp)->intern.flags |= BIN_FLAG_ALL_OBJECTS; mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp); continuation = TUPLE5 diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 9be77fcefa..ed7b9c8618 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -504,7 +504,7 @@ ERTS_GLB_INLINE Binary * erts_db_get_match_prog_binary_unchecked(Eterm term) { Binary *bp = erts_magic_ref2bin(term); - ASSERT(bp->flags & BIN_FLAG_MAGIC); + ASSERT(bp->intern.flags & BIN_FLAG_MAGIC); ASSERT((ERTS_MAGIC_BIN_DESTRUCTOR(bp) == erts_db_match_prog_destructor)); return bp; } @@ -516,7 +516,7 @@ erts_db_get_match_prog_binary(Eterm term) if (!is_internal_magic_ref(term)) return NULL; bp = erts_magic_ref2bin(term); - ASSERT(bp->flags & BIN_FLAG_MAGIC); + ASSERT(bp->intern.flags & BIN_FLAG_MAGIC); if (ERTS_MAGIC_BIN_DESTRUCTOR(bp) != erts_db_match_prog_destructor) return NULL; return bp; @@ -528,7 +528,7 @@ erts_db_get_match_prog_binary(Eterm term) ** Convenience when compiling into Binary structures */ #define IsMatchProgBinary(BP) \ - (((BP)->flags & BIN_FLAG_MAGIC) \ + (((BP)->intern.flags & BIN_FLAG_MAGIC) \ && ERTS_MAGIC_BIN_DESTRUCTOR((BP)) == erts_db_match_prog_destructor) #define Binary2MatchProg(BP) \ diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 4398da36d8..8605b8a22a 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1242,7 +1242,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, * link it into the MSO list for the process. */ - erts_refc_inc(&bptr->refc, 1); + erts_refc_inc(&bptr->intern.refc, 1); *prev = ptr; prev = &ptr->next; } @@ -3596,7 +3596,7 @@ erts_check_off_heap2(Process *p, Eterm *htop) erts_aint_t refc; switch (thing_subtag(u.hdr->thing_word)) { case REFC_BINARY_SUBTAG: - refc = erts_refc_read(&u.pb->val->refc, 1); + refc = erts_refc_read(&u.pb->val->intern.refc, 1); break; case FUN_SUBTAG: refc = erts_smp_refc_read(&u.fun->fe->refc, 1); @@ -3608,7 +3608,7 @@ erts_check_off_heap2(Process *p, Eterm *htop) break; case REF_SUBTAG: ASSERT(is_magic_ref_thing(u.hdr)); - refc = erts_refc_read(&u.mref->mb->refc, 1); + refc = erts_refc_read(&u.mref->mb->intern.refc, 1); break; default: ASSERT(!"erts_check_off_heap2: Invalid thing_word"); diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index e76226c5eb..635b2ab90d 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1276,7 +1276,7 @@ Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin) OH_OVERHEAD(&(MSO(env->proc)), pb->size / sizeof(Eterm)); bin_term = make_binary(pb); - if (erts_refc_read(&bptr->refc, 1) == 1) { + if (erts_refc_read(&bptr->intern.refc, 1) == 1) { /* Total ownership transfer */ bin->ref_bin = NULL; bin->bin_term = bin_term; @@ -2246,7 +2246,7 @@ static int nif_resource_dtor(Binary* bin) ASSERT(type->down); erts_smp_mtx_lock(&rm->lock); - ASSERT(erts_refc_read(&bin->refc, 0) == 0); + ASSERT(erts_refc_read(&bin->intern.refc, 0) == 0); if (rm->root) { ASSERT(!rm->is_dying); destroy_all_monitors(rm->root, resource); @@ -2320,13 +2320,13 @@ void erts_fire_nif_monitor(ErtsResource* resource, Eterm pid, Eterm ref) erts_smp_mtx_unlock(&rmp->lock); if (free_me) { - ASSERT(erts_refc_read(&bin->binary.refc, 0) == 0); + ASSERT(erts_refc_read(&bin->binary.intern.refc, 0) == 0); erts_bin_free(&bin->binary); } return; } ASSERT(!rmp->is_dying); - if (erts_refc_inc_unless(&bin->binary.refc, 0, 0) == 0) { + if (erts_refc_inc_unless(&bin->binary.intern.refc, 0, 0) == 0) { /* * Racing resource destruction. * To avoid a more complex refc-dance with destructing thread @@ -2374,7 +2374,7 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t data_sz) ASSERT(type->owner && type->next && type->prev); /* not allowed in load/upgrade */ resource->type = type; - erts_refc_inc(&bin->refc, 1); + erts_refc_inc(&bin->intern.refc, 1); #ifdef DEBUG erts_refc_init(&resource->nif_refc, 1); #endif @@ -2416,7 +2416,7 @@ void enif_keep_resource(void* obj) #ifdef DEBUG erts_refc_inc(&resource->nif_refc, 1); #endif - erts_refc_inc(&bin->binary.refc, 2); + erts_refc_inc(&bin->binary.intern.refc, 2); } Eterm erts_bld_resource_ref(Eterm** hpp, ErlOffHeap* oh, ErtsResource* resource) @@ -2453,7 +2453,7 @@ ERL_NIF_TERM enif_make_resource_binary(ErlNifEnv* env, void* obj, pb->flags = 0; OH_OVERHEAD(ohp, size / sizeof(Eterm)); - erts_refc_inc(&bin->binary.refc, 1); + erts_refc_inc(&bin->binary.intern.refc, 1); return make_binary(hp); } @@ -2478,7 +2478,7 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ } */ mbin = ((ProcBin *) hp)->val; - if (!(mbin->flags & BIN_FLAG_MAGIC)) + if (!(mbin->intern.flags & BIN_FLAG_MAGIC)) return 0; } resource = (ErtsResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(mbin); diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 77c7c5e73c..fbf14df92b 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -447,8 +447,8 @@ heap_dump(fmtfn_t to, void *to_arg, Eterm x) ProcBin* pb = (ProcBin *) binary_val(x); Binary* val = pb->val; - if (erts_atomic_xchg_nob(&val->refc, 0) != 0) { - val->flags = (UWord) all_binaries; + if (erts_atomic_xchg_nob(&val->intern.refc, 0) != 0) { + val->intern.flags = (UWord) all_binaries; all_binaries = val; } erts_print(to, to_arg, @@ -529,7 +529,7 @@ dump_binaries(fmtfn_t to, void *to_arg, Binary* current) erts_print(to, to_arg, "%02X", bytes[i]); } erts_putc(to, to_arg, '\n'); - current = (Binary *) current->flags; + current = (Binary *) current->intern.flags; } } diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 873945c3b5..e5358c2c3f 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1821,7 +1821,7 @@ static int ttb_context_destructor(Binary *context_bin) case TTBEncode: DESTROY_SAVED_WSTACK(&context->s.ec.wstack); if (context->s.ec.result_bin != NULL) { /* Set to NULL if ever made alive! */ - ASSERT(erts_refc_read(&(context->s.ec.result_bin->refc),1)); + ASSERT(erts_refc_read(&(context->s.ec.result_bin->intern.refc),1)); erts_bin_free(context->s.ec.result_bin); context->s.ec.result_bin = NULL; } @@ -1830,13 +1830,13 @@ static int ttb_context_destructor(Binary *context_bin) erl_zlib_deflate_finish(&(context->s.cc.stream)); if (context->s.cc.destination_bin != NULL) { /* Set to NULL if ever made alive! */ - ASSERT(erts_refc_read(&(context->s.cc.destination_bin->refc),1)); + ASSERT(erts_refc_read(&(context->s.cc.destination_bin->intern.refc),1)); erts_bin_free(context->s.cc.destination_bin); context->s.cc.destination_bin = NULL; } if (context->s.cc.result_bin != NULL) { /* Set to NULL if ever made alive! */ - ASSERT(erts_refc_read(&(context->s.cc.result_bin->refc),1)); + ASSERT(erts_refc_read(&(context->s.cc.result_bin->intern.refc),1)); erts_bin_free(context->s.cc.result_bin); context->s.cc.result_bin = NULL; } @@ -1959,7 +1959,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla pb->bytes = (byte*) result_bin->orig_bytes; pb->flags = 0; OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); - if (context_b && erts_refc_read(&context_b->refc,0) == 0) { + if (context_b && erts_refc_read(&context_b->intern.refc,0) == 0) { erts_bin_free(context_b); } return make_binary(pb); @@ -2025,7 +2025,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla pb->next = MSO(p).first; MSO(p).first = (struct erl_off_heap_header*)pb; pb->val = result_bin; - ASSERT(erts_refc_read(&result_bin->refc, 1)); + ASSERT(erts_refc_read(&result_bin->intern.refc, 1)); pb->bytes = (byte*) result_bin->orig_bytes; pb->flags = 0; OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); @@ -2033,7 +2033,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla context->s.cc.result_bin = NULL; context->alive = 0; BUMP_REDS(p, (this_time * CONTEXT_REDS) / TERM_TO_BINARY_COMPRESS_CHUNK); - if (context_b && erts_refc_read(&context_b->refc,0) == 0) { + if (context_b && erts_refc_read(&context_b->intern.refc,0) == 0) { erts_bin_free(context_b); } return make_binary(pb); @@ -2052,13 +2052,13 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla pb->bytes = (byte*) result_bin->orig_bytes; pb->flags = 0; OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); - ASSERT(erts_refc_read(&result_bin->refc, 1)); + ASSERT(erts_refc_read(&result_bin->intern.refc, 1)); erl_zlib_deflate_finish(&(context->s.cc.stream)); erts_bin_free(context->s.cc.destination_bin); context->s.cc.destination_bin = NULL; context->alive = 0; BUMP_REDS(p, (this_time * CONTEXT_REDS) / TERM_TO_BINARY_COMPRESS_CHUNK); - if (context_b && erts_refc_read(&context_b->refc,0) == 0) { + if (context_b && erts_refc_read(&context_b->intern.refc,0) == 0) { erts_bin_free(context_b); } return make_binary(pb); @@ -2773,7 +2773,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, erts_emasculate_writable_binary(pb); bytes += (pb->val->orig_bytes - before_realloc); } - erts_refc_inc(&pb->val->refc, 2); + erts_refc_inc(&pb->val->intern.refc, 2); sys_memcpy(&tmp, pb, sizeof(ProcBin)); tmp.next = *off_heap; @@ -3887,7 +3887,7 @@ dec_term_atom_common: sys_memcpy(pb, ep, sizeof(ProcBin)); ep += sizeof(ProcBin); - erts_refc_inc(&pb->val->refc, 1); + erts_refc_inc(&pb->val->intern.refc, 1); hp += PROC_BIN_SIZE; pb->next = factory->off_heap->first; factory->off_heap->first = (struct erl_off_heap_header*)pb; @@ -3905,7 +3905,7 @@ dec_term_atom_common: sys_memcpy(pb, ep, sizeof(ProcBin)); ep += sizeof(ProcBin); - erts_refc_inc(&pb->val->refc, 1); + erts_refc_inc(&pb->val->intern.refc, 1); hp += PROC_BIN_SIZE; pb->next = factory->off_heap->first; factory->off_heap->first = (struct erl_off_heap_header*)pb; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 4d864d1402..d765547e89 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1410,7 +1410,7 @@ Eterm erts_msacc_request(Process *c_p, int action, Eterm *threads); #define MatchSetRef(MPSP) \ do { \ if ((MPSP) != NULL) { \ - erts_refc_inc(&(MPSP)->refc, 1); \ + erts_refc_inc(&(MPSP)->intern.refc, 1); \ } \ } while (0) diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index a1d3f8e29f..2f3117223f 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -4901,7 +4901,7 @@ erts_port_control(Process* c_p, ASSERT(bufp <= bufp + size); ASSERT(binp->orig_bytes <= bufp && bufp + size <= binp->orig_bytes + binp->orig_size); - erts_refc_inc(&binp->refc, 1); + erts_refc_inc(&binp->intern.refc, 1); } } @@ -6907,21 +6907,21 @@ ErlDrvSInt driver_binary_get_refc(ErlDrvBinary *dbp) { Binary* bp = ErlDrvBinary2Binary(dbp); - return (ErlDrvSInt) erts_refc_read(&bp->refc, 1); + return (ErlDrvSInt) erts_refc_read(&bp->intern.refc, 1); } ErlDrvSInt driver_binary_inc_refc(ErlDrvBinary *dbp) { Binary* bp = ErlDrvBinary2Binary(dbp); - return (ErlDrvSInt) erts_refc_inctest(&bp->refc, 2); + return (ErlDrvSInt) erts_refc_inctest(&bp->intern.refc, 2); } ErlDrvSInt driver_binary_dec_refc(ErlDrvBinary *dbp) { Binary* bp = ErlDrvBinary2Binary(dbp); - return (ErlDrvSInt) erts_refc_dectest(&bp->refc, 1); + return (ErlDrvSInt) erts_refc_dectest(&bp->intern.refc, 1); } diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 092a5320ba..e2b5434284 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -3594,7 +3594,7 @@ store_external_or_ref_(Uint **hpp, ErlOffHeap* oh, Eterm ns) ErtsMRefThing *mreft = (ErtsMRefThing *) from_hp; ErtsMagicBinary *mb = mreft->mb; ASSERT(is_magic_ref_thing(from_hp)); - erts_refc_inc(&mb->refc, 2); + erts_refc_inc(&mb->intern.refc, 2); OH_OVERHEAD(oh, mb->orig_size / sizeof(Eterm)); } -- cgit v1.2.3 From b50650b4c56de5279395ea195e9e68b077a1d3de Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 8 Feb 2017 17:57:52 +0100 Subject: Fix aux-work timer implementation --- erts/emulator/beam/erl_init.c | 1 + erts/emulator/beam/erl_process.c | 32 ++++++++++++++++++++++++++------ erts/emulator/beam/erl_process.h | 1 + 3 files changed, 28 insertions(+), 6 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 541bfec532..ac0324d846 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -2334,6 +2334,7 @@ erl_start(int argc, char **argv) set_main_stack_size(); erts_sched_init_time_sup(esdp); erts_ets_sched_spec_data_init(esdp); + erts_aux_work_timeout_late_init(esdp); process_main(esdp->x_reg_array, esdp->f_reg_array); } #endif diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 6b5b64993f..e2764cf86f 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -560,7 +560,6 @@ static int stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg); static void aux_work_timeout(void *unused); static void aux_work_timeout_early_init(int no_schedulers); -static void aux_work_timeout_late_init(void); static void setup_aux_work_timer(ErtsSchedulerData *esdp); static int execute_sys_tasks(Process *c_p, @@ -2790,6 +2789,9 @@ typedef struct { int initialized; erts_atomic32_t refc; +#ifdef DEBUG + erts_atomic32_t used; +#endif erts_atomic32_t type[1]; } ErtsAuxWorkTmo; @@ -2799,6 +2801,13 @@ static ERTS_INLINE void start_aux_work_timer(ErtsSchedulerData *esdp) { ErtsMonotonicTime tmo = erts_get_monotonic_time(esdp); +#ifdef DEBUG + Uint no = (Uint) erts_atomic32_xchg_mb(&aux_work_tmo->used, + (erts_aint32_t) esdp->no); + ASSERT(esdp->type == ERTS_SCHED_NORMAL); + ASSERT(!no); +#endif + tmo = ERTS_MONOTONIC_TO_CLKTCKS(tmo-1); tmo += ERTS_MSEC_TO_CLKTCKS(1000) + 1; erts_twheel_init_timer(&aux_work_tmo->timer.data); @@ -2835,16 +2844,19 @@ aux_work_timeout_early_init(int no_schedulers) aux_work_tmo = (ErtsAuxWorkTmo *) p; aux_work_tmo->initialized = 0; erts_atomic32_init_nob(&aux_work_tmo->refc, 0); +#ifdef DEBUG + erts_atomic32_init_nob(&aux_work_tmo->used, 0); +#endif for (i = 0; i <= no_schedulers; i++) erts_atomic32_init_nob(&aux_work_tmo->type[i], 0); } void -aux_work_timeout_late_init(void) +erts_aux_work_timeout_late_init(ErtsSchedulerData *esdp) { aux_work_tmo->initialized = 1; - if (erts_atomic32_read_nob(&aux_work_tmo->refc)) - start_aux_work_timer(erts_get_scheduler_data()); + if (erts_atomic32_read_acqb(&aux_work_tmo->refc)) + start_aux_work_timer(esdp); } static void @@ -2852,6 +2864,13 @@ aux_work_timeout(void *vesdp) { erts_aint32_t refc; int i; +#ifdef DEBUG + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + Uint no = (Uint) erts_atomic32_xchg_mb(&aux_work_tmo->used, 0); + ASSERT(no == esdp->no); + ASSERT(esdp == (ErtsSchedulerData *) vesdp); +#endif + #ifdef ERTS_SMP i = 0; #else @@ -6465,8 +6484,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online /* init port tasks */ erts_port_task_init(); - aux_work_timeout_late_init(); - #ifndef ERTS_SMP #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC erts_scheduler_data->verify_unused_temp_alloc @@ -8752,6 +8769,9 @@ sched_thread_func(void *vesdp) erts_sched_init_time_sup(esdp); + if (no == 1) + erts_aux_work_timeout_late_init(esdp); + (void) ERTS_RUNQ_FLGS_SET_NOB(esdp->run_queue, ERTS_RUNQ_FLG_EXEC); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 883d9f2a4c..f6683b6e7b 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1830,6 +1830,7 @@ void erts_schedule_multi_misc_aux_work(int ignore_self, void (*func)(void *), void *arg); erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int); +void erts_aux_work_timeout_late_init(ErtsSchedulerData *esdp); void erts_sched_notify_check_cpu_bind(void); Uint erts_active_schedulers(void); void erts_init_process(int, int, int); -- cgit v1.2.3 From cb7514ab298c715af435d9a8e5a019e91105cb98 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 30 Jan 2017 15:25:57 +0100 Subject: Remove accessor BIF timer implementation --- erts/emulator/beam/erl_alloc.c | 10 -- erts/emulator/beam/erl_alloc.types | 1 - erts/emulator/beam/erl_hl_timer.c | 206 +------------------------------------ erts/emulator/beam/erl_process.c | 22 ---- erts/emulator/beam/erl_process.h | 3 - 5 files changed, 3 insertions(+), 239 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index a374593c5d..71957b2259 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -678,10 +678,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) = erts_timer_type_size(ERTS_ALC_T_HL_PTIMER); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_BIF_TIMER)] = erts_timer_type_size(ERTS_ALC_T_BIF_TIMER); -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_ABIF_TIMER)] - = erts_timer_type_size(ERTS_ALC_T_ABIF_TIMER); -#endif fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_EXP_TRACE)] = sizeof(NifExportTrace); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MREF_NSCHED_ENT)] @@ -2440,12 +2436,6 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) &size.processes_used, fi, ERTS_ALC_T_BIF_TIMER); -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - add_fix_values(&size.processes, - &size.processes_used, - fi, - ERTS_ALC_T_ABIF_TIMER); -#endif add_fix_values(&size.processes, &size.processes_used, fi, diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 43f43f9034..f296a98125 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -168,7 +168,6 @@ type TIMER_SERVICE LONG_LIVED SYSTEM timer_service type LL_PTIMER FIXED_SIZE PROCESSES ll_ptimer type HL_PTIMER FIXED_SIZE PROCESSES hl_ptimer type BIF_TIMER FIXED_SIZE PROCESSES bif_timer -# type ABIF_TIMER FIXED_SIZE PROCESSES accessor_bif_timer type TIMER_REQUEST SHORT_LIVED PROCESSES timer_request type BTM_YIELD_STATE SHORT_LIVED PROCESSES btm_yield_state type REG_TABLE STANDARD SYSTEM reg_tab diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index 26be8c7edf..cfa1709518 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -106,9 +106,6 @@ typedef enum { #define ERTS_TMR_ROFLG_PROC (((Uint32) 1) << 14) #define ERTS_TMR_ROFLG_PORT (((Uint32) 1) << 15) #define ERTS_TMR_ROFLG_CALLBACK (((Uint32) 1) << 16) -#ifdef ERTS_BTM_ACCESSOR_SUPPORT -#define ERTS_TMR_ROFLG_ABIF_TMR (((Uint32) 1) << 17) -#endif #define ERTS_TMR_ROFLG_SID_MASK \ (ERTS_TMR_ROFLG_HLT - (Uint32) 1) @@ -185,21 +182,10 @@ struct ErtsHLTimer_ { Eterm message; ErlHeapFragment *bp; } btm; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - struct { - Eterm accessor; - ErtsHLTimerTree tree; - } abtm; -#endif }; #define ERTS_HL_PTIMER_SIZE offsetof(ErtsHLTimer, btm) -#ifdef ERTS_BTM_ACCESSOR_SUPPORT -#define ERTS_BIF_TIMER_SIZE offsetof(ErtsHLTimer, abtm) -#define ERTS_ABIF_TIMER_SIZE sizeof(ErtsHLTimer) -#else #define ERTS_BIF_TIMER_SIZE sizeof(ErtsHLTimer) -#endif typedef struct { ErtsTmrHead head; /* NEED to be first! */ @@ -606,60 +592,6 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) #include "erl_rbtree.h" -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - -#define ERTS_RBT_PREFIX abtm -#define ERTS_RBT_T ErtsHLTimer -#define ERTS_RBT_KEY_T Uint32 * -#define ERTS_RBT_FLAGS_T UWord -#define ERTS_RBT_INIT_EMPTY_TNODE(T) \ - do { \ - (T)->abtm.tree.parent = (UWord) NULL; \ - (T)->abtm.tree.right = NULL; \ - (T)->abtm.tree.left = NULL; \ - } while (0) -#define ERTS_RBT_IS_RED(T) \ - ((int) ((T)->abtm.tree.parent & ERTS_HLT_PFLG_RED)) -#define ERTS_RBT_SET_RED(T) \ - ((T)->abtm.tree.parent |= ERTS_HLT_PFLG_RED) -#define ERTS_RBT_IS_BLACK(T) \ - (!ERTS_RBT_IS_RED((T))) -#define ERTS_RBT_SET_BLACK(T) \ - ((T)->abtm.tree.parent &= ~ERTS_HLT_PFLG_RED) -#define ERTS_RBT_GET_FLAGS(T) \ - ((T)->abtm.tree.parent & ERTS_HLT_PFLGS_MASK) -#define ERTS_RBT_SET_FLAGS(T, F) \ - do { \ - ERTS_HLT_ASSERT((((UWord) (F)) & ~ERTS_HLT_PFLGS_MASK) == 0); \ - (T)->abtm.tree.parent &= ~ERTS_HLT_PFLGS_MASK; \ - (T)->abtm.tree.parent |= (F); \ - } while (0) -#define ERTS_RBT_GET_PARENT(T) \ - ((ErtsHLTimer *) ((T)->abtm.tree.parent & ~ERTS_HLT_PFLGS_MASK)) -#define ERTS_RBT_SET_PARENT(T, P) \ - do { \ - ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0); \ - (T)->abtm.tree.parent &= ERTS_HLT_PFLGS_MASK; \ - (T)->abtm.tree.parent |= (UWord) (P); \ - } while (0) -#define ERTS_RBT_GET_RIGHT(T) ((T)->abtm.tree.right) -#define ERTS_RBT_SET_RIGHT(T, R) ((T)->abtm.tree.right = (R)) -#define ERTS_RBT_GET_LEFT(T) ((T)->abtm.tree.left) -#define ERTS_RBT_SET_LEFT(T, L) ((T)->abtm.tree.left = (L)) -#define ERTS_RBT_GET_KEY(T) ((T)->btm.refn) -#define ERTS_RBT_IS_LT(KX, KY) refn_is_lt((KX), (KY)) -#define ERTS_RBT_IS_EQ(KX, KY) \ - (((KX)[0] == (KY)[0]) & ((KX)[1] == (KY)[1]) & ((KX)[2] == (KY)[2])) -#define ERTS_RBT_WANT_DELETE -#define ERTS_RBT_WANT_INSERT -#define ERTS_RBT_WANT_LOOKUP -#define ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING -#define ERTS_RBT_UNDEF - -#include "erl_rbtree.h" - -#endif /* ERTS_BTM_ACCESSOR_SUPPORT */ - #ifdef ERTS_SMP static void init_canceled_queue(ErtsHLTCncldTmrQ *cq); #endif @@ -699,9 +631,6 @@ erts_timer_type_size(ErtsAlcType_t type) case ERTS_ALC_T_LL_PTIMER: return sizeof(ErtsTWTimer); case ERTS_ALC_T_HL_PTIMER: return ERTS_HL_PTIMER_SIZE; case ERTS_ALC_T_BIF_TIMER: return ERTS_BIF_TIMER_SIZE; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - case ERTS_ALC_T_ABIF_TIMER: return ERTS_ABIF_TIMER_SIZE; -#endif default: ERTS_INTERNAL_ERROR("Unknown type"); } return 0; @@ -920,10 +849,6 @@ hl_timer_destroy(ErtsHLTimer *tmr) else { if (roflgs & ERTS_TMR_ROFLG_PRE_ALC) bif_timer_pre_free(tmr); -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - else if (roflgs & ERTS_TMR_ROFLG_ABIF_TMR) - erts_free(ERTS_ALC_T_ABIF_TIMER, tmr); -#endif else erts_free(ERTS_ALC_T_BIF_TIMER, tmr); } @@ -1018,33 +943,6 @@ check_canceled_queue(ErtsSchedulerData *esdp, ErtsHLTimerService *srv) #endif } -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - -static void -hlt_delete_abtm(ErtsHLTimer *tmr) -{ - Process *proc; - - ERTS_HLT_ASSERT(tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR); - - proc = erts_proc_lookup(tmr->abtm.accessor); - - if (proc) { - int deref = 0; - erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM); - if (tmr->abtm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { - abtm_rbt_delete(&proc->accessor_bif_timers, tmr); - deref = 1; - tmr->abtm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - } - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); - if (deref) - hl_timer_pre_dec_refc(tmr); - } -} - -#endif - static ErtsHLTimer * create_hl_timer(ErtsSchedulerData *esdp, ErtsMonotonicTime timeout_pos, @@ -1106,9 +1004,6 @@ create_hl_timer(ErtsSchedulerData *esdp, } else { /* ERTS_TMR_BIF */ Uint hsz; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - int is_abif_tmr = is_value(acsr) && acsr != rcvr; -#endif if (short_time) { tmr = bif_timer_pre_alloc(); @@ -1118,14 +1013,8 @@ create_hl_timer(ErtsSchedulerData *esdp, } else { alloc_bif_timer: -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - if (is_abif_tmr) - tmr = erts_alloc(ERTS_ALC_T_ABIF_TIMER, - ERTS_ABIF_TIMER_SIZE); - else -#endif - tmr = erts_alloc(ERTS_ALC_T_BIF_TIMER, - ERTS_BIF_TIMER_SIZE); + tmr = erts_alloc(ERTS_ALC_T_BIF_TIMER, + ERTS_BIF_TIMER_SIZE); } tmr->timeout = timeout_pos; @@ -1160,23 +1049,6 @@ create_hl_timer(ErtsSchedulerData *esdp, tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - if (is_abif_tmr) { - Process *aproc; - roflgs |= ERTS_TMR_ROFLG_ABIF_TMR; - tmr->abtm.accessor = acsr; - aproc = erts_proc_lookup(acsr); - if (!aproc) - tmr->abtm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - else { - refc++; - erts_smp_proc_lock(aproc, ERTS_PROC_LOCK_BTM); - abtm_rbt_insert(&aproc->accessor_bif_timers, tmr); - erts_smp_proc_unlock(aproc, ERTS_PROC_LOCK_BTM); - } - } -#endif - btm_rbt_insert(&srv->btm_tree, tmr); } @@ -1222,11 +1094,6 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs) Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME); ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR); -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - if (tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR) - hlt_delete_abtm(tmr); -#endif - if (is_reg_name) { Eterm pid; ERTS_HLT_ASSERT(is_atom(tmr->receiver.name)); @@ -1409,10 +1276,6 @@ hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr) tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; } -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - if (tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR) - hlt_delete_abtm(tmr); -#endif } if (tmr->time.tree.parent == ERTS_HLT_PFIELD_NOT_IN_TABLE) { @@ -2032,15 +1895,10 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp, /* * Check if the timer is aimed at current - * process of if this process is an accessor - * of the timer... + * process... */ erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_BTM); tmr = proc_btm_rbt_lookup(c_p->bif_timers, trefn); -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - if (!tmr) - tmr = abtm_rbt_lookup(c_p->accessor_bif_timers, trefn); -#endif erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_BTM); if (!tmr) return 0; @@ -2290,13 +2148,6 @@ parse_bif_timer_options(Eterm option_list, int *async, int *info, if (!abs || !bool_arg(tp[2], abs)) return 0; break; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - case am_accessor: - if (!accessor || is_not_internal_pid(tp[2])) - return 0; - *accessor = tp[2]; - break; -#endif default: return 0; } @@ -2358,9 +2209,6 @@ typedef struct { ErtsBifTimers *bif_timers; union { proc_btm_rbt_yield_state_t proc_btm_yield_state; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - abtm_rbt_yield_state_t abtm_yield_state; -#endif } u; } ErtsBifTimerYieldState; @@ -2401,54 +2249,6 @@ int erts_cancel_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp) return res; } -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - -static void -detach_bif_timer(ErtsHLTimer *tmr, void *vesdp) -{ - tmr->abtm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - hl_timer_dec_refc(tmr, tmr->head.roflgs); -} - -int erts_detach_accessor_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp) -{ - ErtsSchedulerData *esdp = erts_proc_sched_data(p); - ErtsBifTimerYieldState ys = {btm, {ERTS_RBT_YIELD_STAT_INITER}}; - ErtsBifTimerYieldState *ysp; - int res; - - ysp = (ErtsBifTimerYieldState *) *vyspp; - if (!ysp) - ysp = &ys; - - res = abtm_rbt_foreach_destroy_yielding(&ysp->bif_timers, - detach_bif_timer, - (void *) esdp, - &ysp->u.abtm_yield_state, - ERTS_BTM_MAX_DESTROY_LIMIT); - - if (res == 0) { - if (ysp != &ys) - erts_free(ERTS_ALC_T_BTM_YIELD_STATE, ysp); - *vyspp = NULL; - } - else { - - if (ysp == &ys) { - ysp = erts_alloc(ERTS_ALC_T_BTM_YIELD_STATE, - sizeof(ErtsBifTimerYieldState)); - sys_memcpy((void *) ysp, (void *) &ys, - sizeof(ErtsBifTimerYieldState)); - } - - *vyspp = (void *) ysp; - } - - return res; -} - -#endif /* ERTS_BTM_ACCESSOR_SUPPORT */ - static ERTS_INLINE int parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg, ErtsMonotonicTime *conv_arg, int abs, diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index e2764cf86f..cb95eab67e 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -12447,9 +12447,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->msg_inq.len = 0; #endif p->bif_timers = NULL; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - p->accessor_bif_timers = NULL; -#endif p->mbuf = NULL; p->msg_frag = NULL; p->mbuf_sz = 0; @@ -12648,9 +12645,6 @@ void erts_init_empty_process(Process *p) p->msg.save = &p->msg.first; p->msg.len = 0; p->bif_timers = NULL; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - p->accessor_bif_timers = NULL; -#endif p->dictionary = NULL; p->seq_trace_clock = 0; p->seq_trace_lastcnt = 0; @@ -12751,9 +12745,6 @@ erts_debug_verify_clean_empty_process(Process* p) ASSERT(p->msg.first == NULL); ASSERT(p->msg.len == 0); ASSERT(p->bif_timers == NULL); -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - ASSERT(p->accessor_bif_timers == NULL); -#endif ASSERT(p->dictionary == NULL); ASSERT(p->catches == 0); ASSERT(p->cp == NULL); @@ -13827,19 +13818,6 @@ erts_continue_exit_process(Process *p) p->bif_timers = NULL; } -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - if (p->accessor_bif_timers) { - if (erts_detach_accessor_bif_timers(p, - p->accessor_bif_timers, - &p->u.terminate)) { - ASSERT(erts_proc_read_refc(p) > 0); - goto yield; - } - ASSERT(erts_proc_read_refc(p) > 0); - p->accessor_bif_timers = NULL; - } -#endif - #ifdef ERTS_SMP if (p->flags & F_SCHDLR_ONLN_WAITQ) abort_sched_onln_chng_waitq(p); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index f6683b6e7b..b21597d63b 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1026,9 +1026,6 @@ struct process { ErlMessageQueue msg; /* Message queue */ ErtsBifTimers *bif_timers; /* Bif timers aiming at this process */ -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - ErtsBifTimers *accessor_bif_timers; /* Accessor bif timers */ -#endif ProcDict *dictionary; /* Process dictionary, may be NULL */ -- cgit v1.2.3 From e46671ce6ea9429fe3d6723b4f93c08c7fa85cd0 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 31 Jan 2017 13:30:49 +0100 Subject: Use magic refs for BIF timers --- erts/emulator/beam/erl_hl_timer.c | 757 +++++++++++++++++++++++++++----------- erts/emulator/beam/erl_hl_timer.h | 2 +- erts/emulator/beam/erl_process.c | 2 +- 3 files changed, 553 insertions(+), 208 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index cfa1709518..6d570d28ec 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -29,6 +29,8 @@ # include "config.h" #endif +#define ERTS_MAGIC_REF_BIF_TIMERS + #include "sys.h" #include "global.h" #include "bif.h" @@ -36,6 +38,9 @@ #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" #include "erl_hl_timer.h" +#ifdef ERTS_MAGIC_REF_BIF_TIMERS +#include "erl_binary.h" +#endif #define ERTS_TMR_CHECK_CANCEL_ON_CREATE 0 @@ -124,6 +129,11 @@ typedef struct ErtsHLTimer_ ErtsHLTimer; #define ERTS_HLT_PFIELD_NOT_IN_TABLE (~((UWord) 0)) +typedef struct { + ErtsHLTimer *next; + ErtsHLTimer *prev; +} ErtsHLTimerList; + typedef struct { UWord parent; /* parent pointer and flags... */ union { @@ -140,11 +150,6 @@ typedef struct { } ErtsHLTimerTimeTree; typedef struct { - UWord parent; /* parent pointer and flags... */ - ErtsHLTimer *right; - ErtsHLTimer *left; -} ErtsHLTimerTree; - typedef struct { Uint32 roflgs; erts_smp_atomic32_t refc; @@ -176,9 +181,14 @@ struct ErtsHLTimer_ { /* BIF timer only fields follow... */ struct { +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsMagicBinary *mbin; + ErtsHLTimerList proc_list; +#else Uint32 refn[ERTS_REF_NUMBERS]; ErtsHLTimerTree proc_tree; ErtsHLTimerTree tree; +#endif Eterm message; ErlHeapFragment *bp; } btm; @@ -282,12 +292,16 @@ struct ErtsHLTimerService_ { ErtsHLTCncldTmrQ canceled_queue; #endif ErtsHLTimer *time_tree; +#ifndef ERTS_MAGIC_REF_BIF_TIMERS ErtsHLTimer *btm_tree; +#endif ErtsHLTimer *next_timeout; ErtsYieldingTimeoutState yield; ErtsTWheelTimer service_timer; }; +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + static ERTS_INLINE int refn_is_lt(Uint32 *x, Uint32 *y) { @@ -303,6 +317,14 @@ refn_is_lt(Uint32 *x, Uint32 *y) return x[0] < y[0]; } +static ERTS_INLINE int +refn_is_eq(Uint32 *x, Uint32 *y) +{ + return (x[0] == y[0]) & (x[1] == y[1]) & (x[2] == y[2]); +} + +#endif + #define ERTS_RBT_PREFIX time #define ERTS_RBT_T ErtsHLTimer #define ERTS_RBT_KEY_T ErtsMonotonicTime @@ -492,6 +514,14 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) #endif /* ERTS_HLT_HARD_DEBUG */ +#ifdef ERTS_MAGIC_REF_BIF_TIMERS +#define ERTS_BTM_HLT2REFN(T) ((T)->btm.mbin->refn) +#else +#define ERTS_BTM_HLT2REFN(T) ((T)->btm.refn) +#endif + +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + #define ERTS_RBT_PREFIX btm #define ERTS_RBT_T ErtsHLTimer #define ERTS_RBT_KEY_T Uint32 * @@ -530,18 +560,92 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) #define ERTS_RBT_SET_RIGHT(T, R) ((T)->btm.tree.right = (R)) #define ERTS_RBT_GET_LEFT(T) ((T)->btm.tree.left) #define ERTS_RBT_SET_LEFT(T, L) ((T)->btm.tree.left = (L)) -#define ERTS_RBT_GET_KEY(T) ((T)->btm.refn) +#define ERTS_RBT_GET_KEY(T) ERTS_BTM_HLT2REFN((T)) #define ERTS_RBT_IS_LT(KX, KY) refn_is_lt((KX), (KY)) -#define ERTS_RBT_IS_EQ(KX, KY) \ - (((KX)[0] == (KY)[0]) & ((KX)[1] == (KY)[1]) & ((KX)[2] == (KY)[2])) +#define ERTS_RBT_IS_EQ(KX, KY) refn_is_eq((KX), (KY)) #define ERTS_RBT_WANT_DELETE #define ERTS_RBT_WANT_INSERT +#ifndef ERTS_MAGIC_REF_BIF_TIMERS #define ERTS_RBT_WANT_LOOKUP +#endif #define ERTS_RBT_WANT_FOREACH #define ERTS_RBT_UNDEF #include "erl_rbtree.h" +#endif /* !ERTS_MAGIC_REF_BIF_TIMERS */ + +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + +static ERTS_INLINE void +proc_btm_list_insert(ErtsHLTimer **list, ErtsHLTimer *x) +{ + ErtsHLTimer *y = *list; + if (!y) { + x->btm.proc_list.next = x; + x->btm.proc_list.prev = x; + *list = x; + } + else { + ERTS_HLT_ASSERT(y->btm.proc_list.prev->btm.proc_list.next == y); + x->btm.proc_list.next = y; + x->btm.proc_list.prev = y->btm.proc_list.prev; + y->btm.proc_list.prev->btm.proc_list.next = x; + y->btm.proc_list.prev = x; + } +} + +static ERTS_INLINE void +proc_btm_list_delete(ErtsHLTimer **list, ErtsHLTimer *x) +{ + ErtsHLTimer *y = *list; + if (y == x && x->btm.proc_list.next == x) { + ERTS_HLT_ASSERT(x->btm.proc_list.prev == x); + *list = NULL; + } + else { + if (y == x) + *list = x->btm.proc_list.next; + ERTS_HLT_ASSERT(x->btm.proc_list.prev->btm.proc_list.next == x); + ERTS_HLT_ASSERT(x->btm.proc_list.next->btm.proc_list.prev == x); + x->btm.proc_list.prev->btm.proc_list.next = x->btm.proc_list.next; + x->btm.proc_list.next->btm.proc_list.prev = x->btm.proc_list.prev; + } + x->btm.proc_list.next = NULL; +} + +static ERTS_INLINE int +proc_btm_list_foreach_destroy_yielding(ErtsHLTimer **list, + void (*destroy)(ErtsHLTimer *, void *), + void *arg, + int limit) +{ + int i; + ErtsHLTimer *first, *last; + + first = *list; + if (!first) + return 0; + + last = first->btm.proc_list.prev; + for (i = 0; i < limit; i++) { + ErtsHLTimer *x = last; + last = last->btm.proc_list.prev; + (*destroy)(x, arg); + x->btm.proc_list.next = NULL; + if (x == first) { + *list = NULL; + return 0; + } + } + + last->btm.proc_list.next = first; + first->btm.proc_list.prev = last; + return 1; +} + +#else /* !ERTS_MAGIC_REF_BIF_TIMERS */ + #define ERTS_RBT_PREFIX proc_btm #define ERTS_RBT_T ErtsHLTimer #define ERTS_RBT_KEY_T Uint32 * @@ -580,18 +684,21 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) #define ERTS_RBT_SET_RIGHT(T, R) ((T)->btm.proc_tree.right = (R)) #define ERTS_RBT_GET_LEFT(T) ((T)->btm.proc_tree.left) #define ERTS_RBT_SET_LEFT(T, L) ((T)->btm.proc_tree.left = (L)) -#define ERTS_RBT_GET_KEY(T) ((T)->btm.refn) +#define ERTS_RBT_GET_KEY(T) ERTS_BTM_HLT2REFN((T)) #define ERTS_RBT_IS_LT(KX, KY) refn_is_lt((KX), (KY)) -#define ERTS_RBT_IS_EQ(KX, KY) \ - (((KX)[0] == (KY)[0]) & ((KX)[1] == (KY)[1]) & ((KX)[2] == (KY)[2])) +#define ERTS_RBT_IS_EQ(KX, KY) refn_is_eq((KX), (KY)) #define ERTS_RBT_WANT_DELETE #define ERTS_RBT_WANT_INSERT +#ifndef ERTS_MAGIC_REF_BIF_TIMERS #define ERTS_RBT_WANT_LOOKUP +#endif #define ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING #define ERTS_RBT_UNDEF #include "erl_rbtree.h" +#endif /* !ERTS_MAGIC_REF_BIF_TIMERS */ + #ifdef ERTS_SMP static void init_canceled_queue(ErtsHLTCncldTmrQ *cq); #endif @@ -612,7 +719,9 @@ erts_create_timer_service(void) srv = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_TIMER_SERVICE, sizeof(ErtsHLTimerService)); srv->time_tree = NULL; +#ifndef ERTS_MAGIC_REF_BIF_TIMERS srv->btm_tree = NULL; +#endif srv->next_timeout = NULL; srv->yield = init_yield; erts_twheel_init_timer(&srv->service_timer); @@ -847,6 +956,11 @@ hl_timer_destroy(ErtsHLTimer *tmr) if (!(roflgs & ERTS_TMR_ROFLG_BIF_TMR)) erts_free(ERTS_ALC_T_HL_PTIMER, tmr); else { +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + Binary *bp = (Binary *) tmr->btm.mbin; + if (erts_refc_dectest(&bp->refc, 0) == 0) + erts_bin_free(bp); +#endif if (roflgs & ERTS_TMR_ROFLG_PRE_ALC) bif_timer_pre_free(tmr); else @@ -892,6 +1006,7 @@ schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs) * once... */ size = sizeof(ErtsHLTimer); + } erts_schedule_thr_prgr_later_cleanup_op( @@ -943,12 +1058,64 @@ check_canceled_queue(ErtsSchedulerData *esdp, ErtsHLTimerService *srv) #endif } +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + +static erts_smp_atomic_t * +mbin_to_btmref__(ErtsMagicBinary *mbin) +{ + return erts_smp_binary_to_magic_indirection((Binary *) mbin); +} + +static ERTS_INLINE void +magic_binary_init(ErtsMagicBinary *mbin, ErtsHLTimer *tmr) +{ + erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin); + erts_smp_atomic_init_nob(aptr, (erts_aint_t) tmr); +} + +static ERTS_INLINE ErtsHLTimer * +magic_binary_to_btm(ErtsMagicBinary *mbin) +{ + erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin); + ErtsHLTimer *tmr = (ErtsHLTimer *) erts_smp_atomic_read_nob(aptr); + ERTS_HLT_ASSERT(!tmr || tmr->btm.mbin == mbin); + return tmr; +} + +static ERTS_INLINE void +btm_clear_magic_binary(ErtsHLTimer *tmr) +{ + erts_smp_atomic_t *aptr = mbin_to_btmref__(tmr->btm.mbin); +#ifdef ERTS_HLT_DEBUG + erts_aint_t tval = erts_smp_atomic_xchg_nob(aptr, + (erts_aint_t) NULL); + ERTS_HLT_ASSERT(tval == (erts_aint_t) tmr); +#else + erts_smp_atomic_set_nob(aptr, (erts_aint_t) NULL); +#endif + hl_timer_dec_refc(tmr, tmr->head.roflgs); +} + +static int +bif_timer_ref_destructor(Binary *unused) +{ + return 1; +} + +#endif + + static ErtsHLTimer * create_hl_timer(ErtsSchedulerData *esdp, ErtsMonotonicTime timeout_pos, int short_time, ErtsTmrType type, void *rcvrp, Eterm rcvr, Eterm acsr, - Eterm msg, Uint32 *refn, + Eterm msg, +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsMagicBinary *mbin, +#else + Uint32 *refn, +#endif void (*callback)(void *), void *arg) { ErtsHLTimerService *srv = esdp->timer_service; @@ -1043,6 +1210,13 @@ create_hl_timer(ErtsSchedulerData *esdp, tmr->btm.message = copy_struct(msg, hsz, &hp, &bp->off_heap); tmr->btm.bp = bp; } +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + tmr->btm.mbin = mbin; + erts_refc_inc(&mbin->refc, 1); + magic_binary_init(mbin, tmr); + refc += 1; /* from magic binary... */ + tmr->btm.proc_list.next = NULL; +#else tmr->btm.refn[0] = refn[0]; tmr->btm.refn[1] = refn[1]; tmr->btm.refn[2] = refn[2]; @@ -1050,6 +1224,7 @@ create_hl_timer(ErtsSchedulerData *esdp, tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; btm_rbt_insert(&srv->btm_tree, tmr); +#endif } tmr->head.roflgs = roflgs; @@ -1094,6 +1269,10 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs) Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME); ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + btm_clear_magic_binary(tmr); +#endif + if (is_reg_name) { Eterm pid; ERTS_HLT_ASSERT(is_atom(tmr->receiver.name)); @@ -1123,11 +1302,18 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs) queued_message = 1; proc_locks &= ~ERTS_PROC_LOCKS_MSG_SEND; tmr->btm.bp = NULL; +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + if (tmr->btm.proc_list.next) { + proc_btm_list_delete(&proc->bif_timers, tmr); + dec_refc = 1; + } +#else if (tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { proc_btm_rbt_delete(&proc->bif_timers, tmr); tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; dec_refc = 1; } +#endif } if (proc_locks) erts_smp_proc_unlock(proc, proc_locks); @@ -1154,7 +1340,9 @@ hlt_port_timeout(ErtsHLTimer *tmr) static void hlt_timeout(ErtsHLTimer *tmr, void *vsrv) { +#if !defined(ERTS_MAGIC_REF_BIF_TIMERS) || defined(ERTS_HLT_HARD_DEBUG) ErtsHLTimerService *srv = (ErtsHLTimerService *) vsrv; +#endif Uint32 roflgs; erts_aint32_t state; @@ -1186,11 +1374,14 @@ static void hlt_timeout(ErtsHLTimer *tmr, void *vsrv) } tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + +#ifndef ERTS_MAGIC_REF_BIF_TIMERS if ((roflgs & ERTS_TMR_ROFLG_BIF_TMR) && tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { btm_rbt_delete(&srv->btm_tree, tmr); tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; } +#endif ERTS_HLT_HDBG_CHK_SRV(srv); @@ -1269,6 +1460,7 @@ hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr) ERTS_HLT_HDBG_CHK_SRV(srv); +#ifndef ERTS_MAGIC_REF_BIF_TIMERS if (tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR) { if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { @@ -1277,6 +1469,7 @@ hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr) } } +#endif if (tmr->time.tree.parent == ERTS_HLT_PFIELD_NOT_IN_TABLE) { /* Already removed... */ @@ -1613,6 +1806,7 @@ continue_cancel_ptimer(ErtsSchedulerData *esdp, ErtsTimer *tmr) * BIF timer specific */ + Uint erts_bif_timer_memory_size(void) { return (Uint) 0; @@ -1627,41 +1821,59 @@ setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos, Eterm ref, tmo_msg, *hp; ErtsHLTimer *tmr; ErtsSchedulerData *esdp; - DeclareTmpHeap(tmp_hp, 4, c_p); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + Binary *mbin; +#endif + Eterm tmp_hp[4]; if (is_not_internal_pid(rcvr) && is_not_atom(rcvr)) goto badarg; esdp = erts_proc_sched_data(c_p); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + mbin = erts_create_magic_indirection(bif_timer_ref_destructor); + hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE); + ref = erts_mk_magic_ref(&hp, &c_p->off_heap, mbin); + ASSERT(erts_get_ref_numbers_thr_id(((ErtsMagicBinary *)mbin)->refn) + == (Uint32) esdp->no); +#else hp = HAlloc(c_p, ERTS_REF_THING_SIZE); ref = erts_sched_make_ref_in_buffer(esdp, hp); - - ASSERT(erts_get_ref_numbers_thr_id( - internal_ref_numbers(ref)) == (Uint32) esdp->no); - - UseTmpHeap(4, c_p); + ASSERT(erts_get_ref_numbers_thr_id(internal_ordinary_ref_numbers(ref)) + == (Uint32) esdp->no); +#endif tmo_msg = wrap ? TUPLE3(tmp_hp, am_timeout, ref, msg) : msg; tmr = create_hl_timer(esdp, timeout_pos, short_time, ERTS_TMR_BIF, NULL, rcvr, acsr, tmo_msg, - internal_ref_numbers(ref), NULL, NULL); - - UnUseTmpHeap(4, c_p); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + (ErtsMagicBinary *) mbin, +#else + internal_ordinary_ref_numbers(ref), +#endif + NULL, NULL); if (is_internal_pid(rcvr)) { Process *proc = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, rcvr, ERTS_PROC_LOCK_BTM, ERTS_P2P_FLG_INC_REFC); if (!proc) { +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + btm_clear_magic_binary(tmr); +#endif if (tmr->btm.bp) free_message_buffer(tmr->btm.bp); hlt_delete_timer(esdp, tmr); hl_timer_destroy(tmr); } else { +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + proc_btm_list_insert(&proc->bif_timers, tmr); +#else proc_btm_rbt_insert(&proc->bif_timers, tmr); +#endif erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); tmr->receiver.proc = proc; } @@ -1689,6 +1901,10 @@ cancel_bif_timer(ErtsHLTimer *tmr) if (state != ERTS_TMR_STATE_ACTIVE) return 0; +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + btm_clear_magic_binary(tmr); +#endif + if (tmr->btm.bp) free_message_buffer(tmr->btm.bp); @@ -1705,25 +1921,215 @@ cancel_bif_timer(ErtsHLTimer *tmr) * the btm tree by itself (it may be in * the middle of tree destruction). */ +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + if (!ERTS_PROC_IS_EXITING(proc) && tmr->btm.proc_list.next) { + proc_btm_list_delete(&proc->bif_timers, tmr); + res = 1; + } +#else if (!ERTS_PROC_IS_EXITING(proc) && tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { proc_btm_rbt_delete(&proc->bif_timers, tmr); tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; res = 1; } +#endif erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); } return res; } +static ERTS_INLINE Sint64 +access_btm(ErtsHLTimer *tmr, Uint32 sid, ErtsSchedulerData *esdp, int cancel) +{ + int cncl_res; + Sint64 time_left; + + if (!tmr) + return -1; + + if (!cancel) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->state); + if (state == ERTS_TMR_STATE_ACTIVE) + return get_time_left(esdp, tmr->timeout); + return -1; + } + + cncl_res = cancel_bif_timer(tmr); + if (!cncl_res) + return -1; + + time_left = get_time_left(esdp, tmr->timeout); + + if (sid != (Uint32) esdp->no) { + if (cncl_res > 0) + queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr); + } + else { + if (cncl_res > 0) + hl_timer_dec_refc(tmr, tmr->head.roflgs); + + hlt_delete_timer(esdp, tmr); + } + + return time_left; +} + +static ERTS_INLINE Eterm +return_info(Process *c_p, Sint64 time_left) +{ + Uint hsz; + Eterm *hp; + + if (time_left < 0) + return am_false; + + if (time_left <= (Sint64) MAX_SMALL) + return make_small((Sint) time_left); + + hsz = ERTS_SINT64_HEAP_SIZE(time_left); + hp = HAlloc(c_p, hsz); + return erts_sint64_to_big(time_left, &hp); +} + +static ERTS_INLINE Eterm +send_async_info(Process *proc, ErtsProcLocks initial_locks, + Eterm tref, int cancel, Sint64 time_left) +{ + ErtsProcLocks locks = initial_locks; + ErtsMessage *mp; + Eterm tag, res, msg, ref; + Uint hsz; + Eterm *hp; + ErlOffHeap *ohp; + + hsz = 4; + hsz += NC_HEAP_SIZE(tref); + + if (time_left > (Sint64) MAX_SMALL) + hsz += ERTS_SINT64_HEAP_SIZE(time_left); + + mp = erts_alloc_message_heap(proc, &locks, hsz, &hp, &ohp); + + if (cancel) + tag = am_cancel_timer; + else + tag = am_read_timer; + + ref = STORE_NC(&hp, ohp, tref); + + if (time_left < 0) + res = am_false; + else if (time_left <= (Sint64) MAX_SMALL) + res = make_small((Sint) time_left); + else + res = erts_sint64_to_big(time_left, &hp); + + msg = TUPLE3(hp, tag, ref, res); + + erts_queue_message(proc, locks, mp, msg, am_clock_service); + + locks &= ~initial_locks; + if (locks) + erts_smp_proc_unlock(proc, locks); + + return am_ok; +} + +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + +static BIF_RETTYPE +access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info) +{ + BIF_RETTYPE ret; + Eterm res; + Sint64 time_left; + + if (!is_internal_magic_ref(tref)) { + if (is_not_ref(tref)) { + ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); + return ret; + } + time_left = -1; + } + else { + ErtsMagicBinary *mbin; + mbin = (ErtsMagicBinary *) erts_magic_ref2bin(tref); + if (mbin->destructor != bif_timer_ref_destructor) + time_left = -1; + else { + ErtsHLTimer *tmr; + Uint32 sid; + tmr = magic_binary_to_btm(mbin); + sid = erts_get_ref_numbers_thr_id(internal_magic_ref_numbers(tref)); + ASSERT(1 <= sid && sid <= erts_no_schedulers); + time_left = access_btm(tmr, sid, erts_proc_sched_data(c_p), cancel); + } + } + + if (!info) + res = am_ok; + else if (!async) + res = return_info(c_p, time_left); + else + res = send_async_info(c_p, ERTS_PROC_LOCK_MAIN, + tref, cancel, time_left); + + ERTS_BIF_PREP_RET(ret, res); + + return ret; +} + +#else /* !ERTS_MAGIC_REF_BIF_TIMERS */ + +static ERTS_INLINE Eterm +send_sync_info(Process *proc, ErtsProcLocks initial_locks, + Uint32 *refn, int cancel, Sint64 time_left) +{ + ErtsProcLocks locks = initial_locks; + ErtsMessage *mp; + Eterm res, msg, ref; + Uint hsz; + Eterm *hp; + ErlOffHeap *ohp; + + hsz = 3 + ERTS_REF_THING_SIZE; + + if (time_left > (Sint64) MAX_SMALL) + hsz += ERTS_SINT64_HEAP_SIZE(time_left); + + mp = erts_alloc_message_heap(proc, &locks, hsz, &hp, &ohp); + + write_ref_thing(hp, refn[0], refn[1], refn[2]); + ref = make_internal_ref(hp); + hp += ERTS_REF_THING_SIZE; + + if (time_left < 0) + res = am_false; + else if (time_left <= (Sint64) MAX_SMALL) + res = make_small((Sint) time_left); + else + res = erts_sint64_to_big(time_left, &hp); + + msg = TUPLE2(hp, ref, res); + + erts_queue_message(proc, locks, mp, msg, am_clock_service); + + locks &= ~initial_locks; + if (locks) + erts_smp_proc_unlock(proc, locks); + + return am_ok; +} + static ERTS_INLINE Eterm access_sched_local_btm(Process *c_p, Eterm pid, - Eterm tref, Uint32 *trefn, - Uint32 *rrefn, - int async, int cancel, - int return_res, - int info) + Eterm tref, Uint32 *trefn, + Uint32 *rrefn, + int async, int cancel, + int return_res, + int info) { ErtsSchedulerData *esdp; ErtsHLTimerService *srv; @@ -1747,111 +2153,40 @@ access_sched_local_btm(Process *c_p, Eterm pid, srv = esdp->timer_service; tmr = btm_rbt_lookup(srv->btm_tree, trefn); - if (tmr) { - if (!cancel) { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->state); - if (state == ERTS_TMR_STATE_ACTIVE) - time_left = get_time_left(esdp, tmr->timeout); - } - else { - int cncl_res = cancel_bif_timer(tmr); - if (cncl_res) { - - time_left = get_time_left(esdp, tmr->timeout); - if (cncl_res > 0) - hl_timer_dec_refc(tmr, tmr->head.roflgs); - - hlt_delete_timer(esdp, tmr); - } - } - } + time_left = access_btm(tmr, (Uint32) esdp->no, esdp, cancel); if (!info) - return am_ok; - - if (return_res) { - ERTS_HLT_ASSERT(c_p); - if (time_left < 0) - return am_false; - else if (time_left <= (Sint64) MAX_SMALL) - return make_small((Sint) time_left); - else { - Uint hsz = ERTS_SINT64_HEAP_SIZE(time_left); - Eterm *hp = HAlloc(c_p, hsz); - return erts_sint64_to_big(time_left, &hp); - } - } + return am_ok; if (c_p) { - proc = c_p; - proc_locks = ERTS_PROC_LOCK_MAIN; + proc = c_p; + proc_locks = ERTS_PROC_LOCK_MAIN; } else { - proc = erts_proc_lookup(pid); - proc_locks = 0; + proc = erts_proc_lookup(pid); + proc_locks = 0; } - if (proc) { - Uint hsz; - ErtsMessage *mp; - Eterm *hp, msg, ref, result; - ErlOffHeap *ohp; - Uint32 *refn; -#ifdef ERTS_HLT_DEBUG - Eterm *hp_end; -#endif - - hsz = ERTS_REF_THING_SIZE; - if (async) { - refn = trefn; /* timer ref */ - hsz += 4; /* 3-tuple */ - } - else { - refn = rrefn; /* request ref */ - hsz += 3; /* 2-tuple */ - } - - ERTS_HLT_ASSERT(refn); - - if (time_left > (Sint64) MAX_SMALL) - hsz += ERTS_SINT64_HEAP_SIZE(time_left); - - mp = erts_alloc_message_heap(proc, &proc_locks, - hsz, &hp, &ohp); - -#ifdef ERTS_HLT_DEBUG - hp_end = hp + hsz; -#endif - - if (time_left < 0) - result = am_false; - else if (time_left <= (Sint64) MAX_SMALL) - result = make_small((Sint) time_left); - else - result = erts_sint64_to_big(time_left, &hp); - - write_ref_thing(hp, - refn[0], - refn[1], - refn[2]); - ref = make_internal_ref(hp); - hp += ERTS_REF_THING_SIZE; - - msg = (async - ? TUPLE3(hp, (cancel - ? am_cancel_timer - : am_read_timer), ref, result) - : TUPLE2(hp, ref, result)); - - ERTS_HLT_ASSERT(hp + (async ? 4 : 3) == hp_end); - - erts_queue_message(proc, proc_locks, mp, msg, am_clock_service); - - if (c_p) - proc_locks &= ~ERTS_PROC_LOCK_MAIN; - if (proc_locks) - erts_smp_proc_unlock(proc, proc_locks); + if (!async) { + if (c_p) + return return_info(c_p, time_left); + + if (proc) + return send_sync_info(proc, proc_locks, + rrefn, cancel, time_left); + } + else if (proc) { + Eterm ref; + Eterm heap[ERTS_REF_THING_SIZE]; + if (is_value(tref)) + ref = tref; + else { + write_ref_thing(&heap[0], trefn[0], trefn[1], trefn[2]); + ref = make_internal_ref(&heap[0]); + } + return send_async_info(proc, proc_locks, + ref, cancel, time_left); } return am_ok; @@ -1884,7 +2219,7 @@ bif_timer_access_request(void *vreq) static int try_access_sched_remote_btm(ErtsSchedulerData *esdp, Process *c_p, Uint32 sid, - Uint32 *trefn, + Eterm tref, Uint32 *trefn, int async, int cancel, int info, Eterm *resp) { @@ -1903,84 +2238,45 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp, if (!tmr) return 0; - if (!cancel) { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->state); - if (state == ERTS_TMR_STATE_ACTIVE) - time_left = get_time_left(esdp, tmr->timeout); - else - time_left = -1; - } - else { - int cncl_res = cancel_bif_timer(tmr); - if (!cncl_res) - time_left = -1; - else { - time_left = get_time_left(esdp, tmr->timeout); - if (cncl_res > 0) - queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr); - } - } + time_left = access_btm(tmr, sid, esdp, cancel); - if (!info) { + if (!info) *resp = am_ok; - return 1; - } - - if (!async) { - if (time_left < 0) - *resp = am_false; - else if (time_left <= (Sint64) MAX_SMALL) - *resp = make_small((Sint) time_left); - else { - Uint hsz = ERTS_SINT64_HEAP_SIZE(time_left); - Eterm *hp = HAlloc(c_p, hsz); - *resp = erts_sint64_to_big(time_left, &hp); - } - } - else { - ErtsMessage *mp; - Eterm tag, res, msg, tref; - Uint hsz; - Eterm *hp; - ErtsProcLocks proc_locks = ERTS_PROC_LOCK_MAIN; - ErlOffHeap *ohp; - - hsz = 4 + ERTS_REF_THING_SIZE; - if (time_left > (Sint64) MAX_SMALL) - hsz += ERTS_SINT64_HEAP_SIZE(time_left); - - mp = erts_alloc_message_heap(c_p, &proc_locks, - hsz, &hp, &ohp); - if (cancel) - tag = am_cancel_timer; - else - tag = am_read_timer; - - write_ref_thing(hp, - trefn[0], - trefn[1], - trefn[2]); - tref = make_internal_ref(hp); - hp += ERTS_REF_THING_SIZE; - - if (time_left < 0) - res = am_false; - else if (time_left <= (Sint64) MAX_SMALL) - res = make_small((Sint) time_left); - else - res = erts_sint64_to_big(time_left, &hp); - - msg = TUPLE3(hp, tag, tref, res); + else if (!async) + *resp = return_info(c_p, time_left); + else + *resp = send_async_info(c_p, ERTS_PROC_LOCK_MAIN, + tref, cancel, time_left); - erts_queue_message(c_p, proc_locks, mp, msg, am_clock_service); + return 1; +} - proc_locks &= ~ERTS_PROC_LOCK_MAIN; - if (proc_locks) - erts_smp_proc_unlock(c_p, proc_locks); +static Eterm +no_timer_result(Process *c_p, Eterm tref, int cancel, int async, int info) +{ + ErtsMessage *mp; + Uint hsz; + Eterm *hp, msg, ref, tag; + ErlOffHeap *ohp; + ErtsProcLocks locks; - *resp = am_ok; - } - return 1; + if (!async) + return am_false; + if (!info) + return am_ok; + + hsz = 4; + hsz += NC_HEAP_SIZE(tref); + locks = ERTS_PROC_LOCK_MAIN; + mp = erts_alloc_message_heap(c_p, &locks, hsz, &hp, &ohp); + ref = STORE_NC(&hp, ohp, tref); + tag = cancel ? am_cancel_timer : am_read_timer; + msg = TUPLE3(hp, tag, ref, am_false); + erts_queue_message(c_p, locks, mp, msg, am_clock_service); + locks &= ~ERTS_PROC_LOCK_MAIN; + if (locks) + erts_smp_proc_unlock(c_p, locks); + return am_ok; } static BIF_RETTYPE @@ -2014,7 +2310,7 @@ access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info) ERTS_BIF_PREP_RET(ret, res); } else if (try_access_sched_remote_btm(esdp, c_p, - sid, trefn, + sid, tref, trefn, async, cancel, info, &res)) { ERTS_BIF_PREP_RET(ret, res); @@ -2093,11 +2389,11 @@ badarg: return ret; no_timer: - ERTS_BIF_PREP_RET(ret, am_false); - return ret; - + return no_timer_result(c_p, tref, cancel, async, info); } +#endif /* !ERTS_MAGIC_REF_BIF_TIMERS */ + static ERTS_INLINE int bool_arg(Eterm val, int *argp) { @@ -2174,14 +2470,20 @@ exit_cancel_bif_timer(ErtsHLTimer *tmr, void *vesdp) roflgs = tmr->head.roflgs; sid = roflgs & ERTS_TMR_ROFLG_SID_MASK; - ERTS_HLT_ASSERT(sid == erts_get_ref_numbers_thr_id(tmr->btm.refn)); + ERTS_HLT_ASSERT(sid == erts_get_ref_numbers_thr_id(ERTS_BTM_HLT2REFN(tmr))); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ERTS_HLT_ASSERT(tmr->btm.proc_list.next); +#else ERTS_HLT_ASSERT(tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE); - tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; +#endif if (sid == (Uint32) esdp->no) { if (state == ERTS_TMR_STATE_ACTIVE) { +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + btm_clear_magic_binary(tmr); +#endif if (tmr->btm.bp) free_message_buffer(tmr->btm.bp); hlt_delete_timer(esdp, tmr); @@ -2190,6 +2492,9 @@ exit_cancel_bif_timer(ErtsHLTimer *tmr, void *vesdp) } else { if (state == ERTS_TMR_STATE_ACTIVE) { +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + btm_clear_magic_binary(tmr); +#endif if (tmr->btm.bp) free_message_buffer(tmr->btm.bp); queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr); @@ -2205,17 +2510,29 @@ exit_cancel_bif_timer(ErtsHLTimer *tmr, void *vesdp) # define ERTS_BTM_MAX_DESTROY_LIMIT 50 #endif +#ifndef ERTS_MAGIC_REF_BIF_TIMERS typedef struct { ErtsBifTimers *bif_timers; union { proc_btm_rbt_yield_state_t proc_btm_yield_state; } u; } ErtsBifTimerYieldState; +#endif -int erts_cancel_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp) +int erts_cancel_bif_timers(Process *p, ErtsBifTimers **btm, void **vyspp) { ErtsSchedulerData *esdp = erts_proc_sched_data(p); - ErtsBifTimerYieldState ys = {btm, {ERTS_RBT_YIELD_STAT_INITER}}; + +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + + return proc_btm_list_foreach_destroy_yielding(btm, + exit_cancel_bif_timer, + (void *) esdp, + ERTS_BTM_MAX_DESTROY_LIMIT); + +#else /* !ERTS_MAGIC_REF_BIF_TIMERS */ + + ErtsBifTimerYieldState ys = {*btm, {ERTS_RBT_YIELD_STAT_INITER}}; ErtsBifTimerYieldState *ysp; int res; @@ -2247,6 +2564,8 @@ int erts_cancel_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp) } return res; + +#endif /* !ERTS_MAGIC_REF_BIF_TIMERS */ } static ERTS_INLINE int @@ -2663,6 +2982,11 @@ btm_print(ErtsHLTimer *tmr, void *vbtmp) ErtsMonotonicTime left; Eterm receiver; +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + if (!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) + return; +#endif + if (tmr->timeout <= btmp->now) left = 0; else @@ -2698,7 +3022,11 @@ erts_print_bif_timer_info(fmtfn_t to, void *to_arg) for (six = 0; six < erts_no_schedulers; six++) { ErtsHLTimerService *srv = erts_aligned_scheduler_data[six].esd.timer_service; +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + time_rbt_foreach(srv->time_tree, btm_print, (void *) &btmp); +#else btm_rbt_foreach(srv->btm_tree, btm_print, (void *) &btmp); +#endif } } @@ -2713,6 +3041,10 @@ typedef struct { static void debug_btm_foreach(ErtsHLTimer *tmr, void *vbtmfd) { +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + if (!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) + return; +#endif if (erts_smp_atomic32_read_nob(&tmr->state) == ERTS_TMR_STATE_ACTIVE) { ErtsBTMForeachDebug *btmfd = (ErtsBTMForeachDebug *) vbtmfd; (*btmfd->func)(((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME) @@ -2743,9 +3075,16 @@ erts_debug_bif_timer_foreach(void (*func)(Eterm, for (six = 0; six < erts_no_schedulers; six++) { ErtsHLTimerService *srv = erts_aligned_scheduler_data[six].esd.timer_service; +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + time_rbt_foreach(srv->time_tree, + debug_btm_foreach, + (void *) &btmfd); + +#else btm_rbt_foreach(srv->btm_tree, debug_btm_foreach, (void *) &btmfd); +#endif } } @@ -2868,7 +3207,9 @@ st_hdbg_func(ErtsHLTimer *tmr, void *vhdbg) } ERTS_HLT_ASSERT(tmr->time.tree.u.l.next->time.tree.u.l.prev == tmr); ERTS_HLT_ASSERT(tmr->time.tree.u.l.prev->time.tree.u.l.next == tmr); - ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, tmr->btm.refn) == tmr); +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, ERTS_BTM_HLT2REFN(tmr)) == tmr); +#endif } static void @@ -2897,8 +3238,10 @@ tt_hdbg_func(ErtsHLTimer *tmr, void *vhdbg) & ~ERTS_HLT_PFLGS_MASK); ERTS_HLT_ASSERT(tmr == prnt); } +#ifndef ERTS_MAGIC_REF_BIF_TIMERS if (tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR) - ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, tmr->btm.refn) == tmr); + ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, ERTS_BTM_HLT2REFN(tmr)) == tmr); +#endif if (tmr->time.tree.same_time) { ErtsHdbgHLT st_hdbg; st_hdbg.srv = hdbg->srv; @@ -2964,6 +3307,7 @@ hdbg_chk_srv(ErtsHLTimerService *srv) time_rbt_foreach(srv->time_tree, tt_hdbg_func, (void *) &hdbg); ERTS_HLT_ASSERT(hdbg.found_root); } +#ifndef ERTS_MAGIC_REF_BIF_TIMERS if (srv->btm_tree) { ErtsHdbgHLT hdbg; hdbg.srv = srv; @@ -2972,6 +3316,7 @@ hdbg_chk_srv(ErtsHLTimerService *srv) btm_rbt_foreach(srv->btm_tree, bt_hdbg_func, (void *) &hdbg); ERTS_HLT_ASSERT(hdbg.found_root); } +#endif } #endif /* ERTS_HLT_HARD_DEBUG */ diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h index 9cdcd581a0..33cdf217d7 100644 --- a/erts/emulator/beam/erl_hl_timer.h +++ b/erts/emulator/beam/erl_hl_timer.h @@ -56,7 +56,7 @@ void erts_cancel_proc_timer(Process *); void erts_set_port_timer(Port *, Sint64); void erts_cancel_port_timer(Port *); Sint64 erts_read_port_timer(Port *); -int erts_cancel_bif_timers(Process *, ErtsBifTimers *, void **); +int erts_cancel_bif_timers(Process *, ErtsBifTimers **, void **); int erts_detach_accessor_bif_timers(Process *, ErtsBifTimers *, void **); ErtsHLTimerService *erts_create_timer_service(void); void erts_hl_timer_init(void); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index cb95eab67e..5baf533acd 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -13810,7 +13810,7 @@ erts_continue_exit_process(Process *p) ASSERT(erts_proc_read_refc(p) > 0); if (p->bif_timers) { - if (erts_cancel_bif_timers(p, p->bif_timers, &p->u.terminate)) { + if (erts_cancel_bif_timers(p, &p->bif_timers, &p->u.terminate)) { ASSERT(erts_proc_read_refc(p) > 0); goto yield; } -- cgit v1.2.3 From a09f867979d8039434271a2177c90aabced3a255 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 3 Feb 2017 03:00:37 +0100 Subject: Use timer wheel for short BIF timers --- erts/emulator/beam/erl_hl_timer.c | 1055 +++++++++++++++++++++++-------------- erts/emulator/beam/erl_hl_timer.h | 2 +- 2 files changed, 662 insertions(+), 395 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index 6d570d28ec..102a739b13 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -29,7 +29,7 @@ # include "config.h" #endif -#define ERTS_MAGIC_REF_BIF_TIMERS +/* #define ERTS_MAGIC_REF_BIF_TIMERS */ #include "sys.h" #include "global.h" @@ -129,10 +129,12 @@ typedef struct ErtsHLTimer_ ErtsHLTimer; #define ERTS_HLT_PFIELD_NOT_IN_TABLE (~((UWord) 0)) +typedef struct ErtsBifTimer_ ErtsBifTimer; + typedef struct { - ErtsHLTimer *next; - ErtsHLTimer *prev; -} ErtsHLTimerList; + ErtsBifTimer *next; + ErtsBifTimer *prev; +} ErtsBifTimerList; typedef struct { UWord parent; /* parent pointer and flags... */ @@ -150,6 +152,11 @@ typedef struct { } ErtsHLTimerTimeTree; typedef struct { + UWord parent; /* parent pointer and flags... */ + ErtsBifTimer *right; + ErtsBifTimer *left; +} ErtsBifTimerTree; + typedef struct { Uint32 roflgs; erts_smp_atomic32_t refc; @@ -176,42 +183,59 @@ struct ErtsHLTimer_ { #ifdef ERTS_HLT_HARD_DEBUG int pending_timeout; #endif +}; - erts_smp_atomic32_t state; +typedef struct { + ErtsTmrHead head; /* NEED to be first! */ + union { + Eterm name; + Process *proc; + Port *port; + void (*callback)(void *); + } receiver; + ErtsTWheelTimer tw_tmr; +} ErtsTWTimer; - /* BIF timer only fields follow... */ +struct ErtsBifTimer_ { + union { + ErtsTmrHead head; + ErtsHLTimer hlt; + ErtsTWTimer twt; + } type; struct { + erts_smp_atomic32_t state; #ifdef ERTS_MAGIC_REF_BIF_TIMERS ErtsMagicBinary *mbin; ErtsHLTimerList proc_list; #else Uint32 refn[ERTS_REF_NUMBERS]; - ErtsHLTimerTree proc_tree; - ErtsHLTimerTree tree; + ErtsBifTimerTree proc_tree; + ErtsBifTimerTree tree; #endif - Eterm message; - ErlHeapFragment *bp; + Eterm message; + ErlHeapFragment *bp; } btm; }; -#define ERTS_HL_PTIMER_SIZE offsetof(ErtsHLTimer, btm) -#define ERTS_BIF_TIMER_SIZE sizeof(ErtsHLTimer) - -typedef struct { - ErtsTmrHead head; /* NEED to be first! */ - union { - void *p; - void (*callback)(void *); - } u; - ErtsTWheelTimer tw_tmr; -} ErtsTWTimer; - typedef union { ErtsTmrHead head; ErtsHLTimer hlt; ErtsTWTimer twt; + ErtsBifTimer btm; } ErtsTimer; +typedef ErtsTimer *(*ErtsCreateTimerFunc)(ErtsSchedulerData *esdp, + ErtsMonotonicTime timeout_pos, + int short_time, ErtsTmrType type, + void *rcvrp, Eterm rcvr, + Eterm msg, +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsMagicBinary *mbin, +#else + Uint32 *refn, +#endif + void (*callback)(void *), void *arg); + #ifdef SMALL_MEMORY #define BIF_TIMER_PREALC_SZ 10 #define PTIMER_PREALC_SZ 10 @@ -221,7 +245,7 @@ typedef union { #endif ERTS_SCHED_PREF_PALLOC_IMPL(bif_timer_pre, - ErtsHLTimer, + ErtsBifTimer, BIF_TIMER_PREALC_SZ) ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(tw_timer, @@ -293,7 +317,7 @@ struct ErtsHLTimerService_ { #endif ErtsHLTimer *time_tree; #ifndef ERTS_MAGIC_REF_BIF_TIMERS - ErtsHLTimer *btm_tree; + ErtsBifTimer *btm_tree; #endif ErtsHLTimer *next_timeout; ErtsYieldingTimeoutState yield; @@ -523,7 +547,7 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) #ifndef ERTS_MAGIC_REF_BIF_TIMERS #define ERTS_RBT_PREFIX btm -#define ERTS_RBT_T ErtsHLTimer +#define ERTS_RBT_T ErtsBifTimer #define ERTS_RBT_KEY_T Uint32 * #define ERTS_RBT_FLAGS_T UWord #define ERTS_RBT_INIT_EMPTY_TNODE(T) \ @@ -549,7 +573,7 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) (T)->btm.tree.parent |= (F); \ } while (0) #define ERTS_RBT_GET_PARENT(T) \ - ((ErtsHLTimer *) ((T)->btm.tree.parent & ~ERTS_HLT_PFLGS_MASK)) + ((ErtsBifTimer *) ((T)->btm.tree.parent & ~ERTS_HLT_PFLGS_MASK)) #define ERTS_RBT_SET_PARENT(T, P) \ do { \ ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0); \ @@ -578,9 +602,9 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) #ifdef ERTS_MAGIC_REF_BIF_TIMERS static ERTS_INLINE void -proc_btm_list_insert(ErtsHLTimer **list, ErtsHLTimer *x) +proc_btm_list_insert(ErtsBifTimer **list, ErtsBifTimer *x) { - ErtsHLTimer *y = *list; + ErtsBifTimer *y = *list; if (!y) { x->btm.proc_list.next = x; x->btm.proc_list.prev = x; @@ -596,9 +620,9 @@ proc_btm_list_insert(ErtsHLTimer **list, ErtsHLTimer *x) } static ERTS_INLINE void -proc_btm_list_delete(ErtsHLTimer **list, ErtsHLTimer *x) +proc_btm_list_delete(ErtsBifTimer **list, ErtsBifTimer *x) { - ErtsHLTimer *y = *list; + ErtsBifTimer *y = *list; if (y == x && x->btm.proc_list.next == x) { ERTS_HLT_ASSERT(x->btm.proc_list.prev == x); *list = NULL; @@ -615,13 +639,13 @@ proc_btm_list_delete(ErtsHLTimer **list, ErtsHLTimer *x) } static ERTS_INLINE int -proc_btm_list_foreach_destroy_yielding(ErtsHLTimer **list, - void (*destroy)(ErtsHLTimer *, void *), +proc_btm_list_foreach_destroy_yielding(ErtsBifTimer **list, + void (*destroy)(ErtsBifTimer *, void *), void *arg, int limit) { int i; - ErtsHLTimer *first, *last; + ErtsBifTimer *first, *last; first = *list; if (!first) @@ -629,7 +653,7 @@ proc_btm_list_foreach_destroy_yielding(ErtsHLTimer **list, last = first->btm.proc_list.prev; for (i = 0; i < limit; i++) { - ErtsHLTimer *x = last; + ErtsBifTimer *x = last; last = last->btm.proc_list.prev; (*destroy)(x, arg); x->btm.proc_list.next = NULL; @@ -647,7 +671,7 @@ proc_btm_list_foreach_destroy_yielding(ErtsHLTimer **list, #else /* !ERTS_MAGIC_REF_BIF_TIMERS */ #define ERTS_RBT_PREFIX proc_btm -#define ERTS_RBT_T ErtsHLTimer +#define ERTS_RBT_T ErtsBifTimer #define ERTS_RBT_KEY_T Uint32 * #define ERTS_RBT_FLAGS_T UWord #define ERTS_RBT_INIT_EMPTY_TNODE(T) \ @@ -673,7 +697,7 @@ proc_btm_list_foreach_destroy_yielding(ErtsHLTimer **list, (T)->btm.proc_tree.parent |= (F); \ } while (0) #define ERTS_RBT_GET_PARENT(T) \ - ((ErtsHLTimer *) ((T)->btm.proc_tree.parent & ~ERTS_HLT_PFLGS_MASK)) + ((ErtsBifTimer *) ((T)->btm.proc_tree.parent & ~ERTS_HLT_PFLGS_MASK)) #define ERTS_RBT_SET_PARENT(T, P) \ do { \ ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0); \ @@ -738,8 +762,8 @@ erts_timer_type_size(ErtsAlcType_t type) { switch (type) { case ERTS_ALC_T_LL_PTIMER: return sizeof(ErtsTWTimer); - case ERTS_ALC_T_HL_PTIMER: return ERTS_HL_PTIMER_SIZE; - case ERTS_ALC_T_BIF_TIMER: return ERTS_BIF_TIMER_SIZE; + case ERTS_ALC_T_HL_PTIMER: return sizeof(ErtsHLTimer); + case ERTS_ALC_T_BIF_TIMER: return sizeof(ErtsBifTimer); default: ERTS_INTERNAL_ERROR("Unknown type"); } return 0; @@ -798,6 +822,111 @@ port_timeout_common(Port *port, void *tmr) return 0; } +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + +static erts_smp_atomic_t * +mbin_to_btmref__(ErtsMagicBinary *mbin) +{ + return erts_smp_binary_to_magic_indirection((Binary *) mbin); +} + +static ERTS_INLINE void +magic_binary_init(ErtsMagicBinary *mbin, ErtsBifTimer *tmr) +{ + erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin); + erts_smp_atomic_init_nob(aptr, (erts_aint_t) tmr); +} + +static ERTS_INLINE ErtsBifTimer * +magic_binary_to_btm(ErtsMagicBinary *mbin) +{ + erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin); + ErtsBifTimer *tmr = (ErtsBifTimer *) erts_smp_atomic_read_nob(aptr); + ERTS_HLT_ASSERT(!tmr || tmr->btm.mbin == mbin); + return tmr; +} + +#endif /* ERTS_MAGIC_REF_BIF_TIMERS */ + +static ERTS_INLINE erts_aint_t +init_btm_specifics(ErtsSchedulerData *esdp, + ErtsBifTimer *tmr, Eterm msg, +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsMagicBinary *mbin +#else + Uint32 *refn +#endif + ) +{ + Uint hsz = is_immed(msg) ? ((Uint) 0) : size_object(msg); + int refc; + if (!hsz) { + tmr->btm.message = msg; + tmr->btm.bp = NULL; + } + else { + ErlHeapFragment *bp = new_message_buffer(hsz); + Eterm *hp = bp->mem; + tmr->btm.message = copy_struct(msg, hsz, &hp, &bp->off_heap); + tmr->btm.bp = bp; + } +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + refc = 1; + tmr->btm.mbin = mbin; + erts_refc_inc(&mbin->refc, 1); + magic_binary_init(mbin, tmr); + tmr->btm.proc_list.next = NULL; +#else + refc = 0; + tmr->btm.refn[0] = refn[0]; + tmr->btm.refn[1] = refn[1]; + tmr->btm.refn[2] = refn[2]; + + tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + + btm_rbt_insert(&esdp->timer_service->btm_tree, tmr); +#endif + + erts_smp_atomic32_init_nob(&tmr->btm.state, ERTS_TMR_STATE_ACTIVE); + return refc; /* refc from magic binary... */ +} + +static void tw_bif_timer_timeout(void *vbtmp); + +static ERTS_INLINE void +timer_destroy(ErtsTimer *tmr, int twt, int btm) +{ + if (!btm) { + if (twt) + tw_timer_free(&tmr->twt); + else + erts_free(ERTS_ALC_T_HL_PTIMER, tmr); + } + else { +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + Binary *bp = (Binary *) tmr->btm.btm.mbin; + if (erts_refc_dectest(&bp->refc, 0) == 0) + erts_bin_free(bp); +#endif + if (tmr->head.roflgs & ERTS_TMR_ROFLG_PRE_ALC) + bif_timer_pre_free(&tmr->btm); + else + erts_free(ERTS_ALC_T_BIF_TIMER, &tmr->btm); + } +} + +static ERTS_INLINE void +timer_pre_dec_refc(ErtsTimer *tmr) +{ +#ifdef ERTS_HLT_DEBUG + erts_aint_t refc; + refc = erts_smp_atomic32_dec_read_nob(&tmr->head.refc); + ERTS_HLT_ASSERT(refc > 0); +#else + erts_smp_atomic32_dec_nob(&tmr->head.refc); +#endif +} + /* * Basic timer wheel timer stuff */ @@ -805,26 +934,39 @@ port_timeout_common(Port *port, void *tmr) static void scheduled_tw_timer_destroy(void *vtmr) { - tw_timer_free((ErtsTWTimer *) vtmr); + ErtsTimer * tmr = (ErtsTimer *) vtmr; + int btm = !!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR); + timer_destroy((ErtsTimer *) vtmr, 1, btm); } static void schedule_tw_timer_destroy(ErtsTWTimer *tmr) { + Uint size; /* * Reference to process/port can be * dropped at once... */ if (tmr->head.roflgs & ERTS_TMR_ROFLG_PROC) - erts_proc_dec_refc((Process *) tmr->u.p); + erts_proc_dec_refc(tmr->receiver.proc); else if (tmr->head.roflgs & ERTS_TMR_ROFLG_PORT) - erts_port_dec_refc((Port *) tmr->u.p); + erts_port_dec_refc(tmr->receiver.port); + + if (!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) + size = sizeof(ErtsHLTimer); + else { + /* Message buffer already dropped... */ + size = sizeof(ErtsBifTimer); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + size += sizeof(ErtsMagicIndirectionWord); +#endif + } erts_schedule_thr_prgr_later_cleanup_op( scheduled_tw_timer_destroy, (void *) tmr, &tmr->tw_tmr.u.cleanup, - sizeof(ErtsTWTimer)); + size); } static ERTS_INLINE void @@ -840,7 +982,7 @@ static void tw_proc_timeout(void *vtwtp) { ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; - Process *proc = (Process *) twtp->u.p; + Process *proc = twtp->receiver.proc; if (proc_timeout_common(proc, vtwtp)) tw_timer_dec_refc(twtp); tw_timer_dec_refc(twtp); @@ -850,7 +992,7 @@ static void tw_port_timeout(void *vtwtp) { ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; - Port *port = (Port *) twtp->u.p; + Port *port = twtp->receiver.port; if (port_timeout_common(port, vtwtp)) tw_timer_dec_refc(twtp); tw_timer_dec_refc(twtp); @@ -874,53 +1016,78 @@ static void tw_callback_timeout(void *vtwtp) { ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; - void (*callback)(void *) = twtp->u.callback; + void (*callback)(void *) = twtp->receiver.callback; void *arg = twtp->head.u.arg; tw_timer_dec_refc(twtp); (*callback)(arg); } -static ErtsTWTimer * -create_tw_timer(ErtsSchedulerData *esdp, - ErtsTmrType type, void *p, - void (*callback)(void *), void *arg, - ErtsMonotonicTime timeout_pos) +static ErtsTimer * +create_tw_timer(ErtsSchedulerData *esdp, + ErtsMonotonicTime timeout_pos, + int short_time, ErtsTmrType type, + void *rcvrp, Eterm rcvr, + Eterm msg, +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsMagicBinary *mbin, +#else + Uint32 *refn, +#endif + void (*callback)(void *), void *arg) { ErtsTWTimer *tmr; void (*timeout_func)(void *); void (*cancel_func)(void *); erts_aint32_t refc; - tmr = tw_timer_alloc(); - erts_twheel_init_timer(&tmr->tw_tmr); + if (type != ERTS_TMR_BIF) { + tmr = tw_timer_alloc(); + tmr->head.roflgs = 0; + } + else { + if (short_time) { + tmr = (ErtsTWTimer *) bif_timer_pre_alloc(); + if (!tmr) + goto alloc_bif_timer; + tmr->head.roflgs = (ERTS_TMR_ROFLG_BIF_TMR + | ERTS_TMR_ROFLG_PRE_ALC); + } + else { + alloc_bif_timer: + tmr = (ErtsTWTimer *) erts_alloc(ERTS_ALC_T_BIF_TIMER, + sizeof(ErtsBifTimer)); + tmr->head.roflgs = ERTS_TMR_ROFLG_BIF_TMR; + } + } - tmr->head.roflgs = (Uint32) esdp->no; - ERTS_HLT_ASSERT((tmr->head.roflgs - & ~ERTS_TMR_ROFLG_SID_MASK) == 0); + erts_twheel_init_timer(&tmr->tw_tmr); + tmr->head.roflgs |= (Uint32) esdp->no; + ERTS_HLT_ASSERT((((Uint32) esdp->no) + & ~ERTS_TMR_ROFLG_SID_MASK) == 0); switch (type) { case ERTS_TMR_PROC: - tmr->u.p = p; + tmr->receiver.proc = (Process *) rcvrp; tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC; timeout_func = tw_proc_timeout; cancel_func = tw_ptimer_cancel; - erts_proc_inc_refc((Process *) p); + erts_proc_inc_refc((Process *) rcvrp); refc = 2; break; case ERTS_TMR_PORT: - tmr->u.p = p; + tmr->receiver.port = (Port *) rcvrp; tmr->head.roflgs |= ERTS_TMR_ROFLG_PORT; timeout_func = tw_port_timeout; cancel_func = tw_ptimer_cancel; - erts_port_inc_refc((Port *) p); + erts_port_inc_refc((Port *) rcvrp); refc = 2; break; case ERTS_TMR_CALLBACK: tmr->head.u.arg = arg; - tmr->u.callback = callback; + tmr->receiver.callback = callback; tmr->head.roflgs |= ERTS_TMR_ROFLG_CALLBACK; timeout_func = tw_callback_timeout; @@ -928,6 +1095,33 @@ create_tw_timer(ErtsSchedulerData *esdp, refc = 1; break; + case ERTS_TMR_BIF: + + timeout_func = tw_bif_timer_timeout; + cancel_func = NULL; + if (is_internal_pid(rcvr)) { + tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC; + tmr->receiver.proc = (Process *) rcvrp; + refc = 2; + } + else { + ERTS_HLT_ASSERT(is_atom(rcvr)); + tmr->head.roflgs |= ERTS_TMR_ROFLG_REG_NAME; + tmr->receiver.name = (Eterm) rcvr; + refc = 1; + } + + refc += init_btm_specifics(esdp, + (ErtsBifTimer *) tmr, + msg, +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + mbin +#else + refn +#endif + ); + break; + default: ERTS_INTERNAL_ERROR("Unsupported timer type"); return NULL; @@ -942,36 +1136,19 @@ create_tw_timer(ErtsSchedulerData *esdp, tmr, timeout_pos); - return tmr; + return (ErtsTimer *) tmr; } /* * Basic high level timer stuff */ -static ERTS_INLINE void -hl_timer_destroy(ErtsHLTimer *tmr) -{ - Uint32 roflgs = tmr->head.roflgs; - if (!(roflgs & ERTS_TMR_ROFLG_BIF_TMR)) - erts_free(ERTS_ALC_T_HL_PTIMER, tmr); - else { -#ifdef ERTS_MAGIC_REF_BIF_TIMERS - Binary *bp = (Binary *) tmr->btm.mbin; - if (erts_refc_dectest(&bp->refc, 0) == 0) - erts_bin_free(bp); -#endif - if (roflgs & ERTS_TMR_ROFLG_PRE_ALC) - bif_timer_pre_free(tmr); - else - erts_free(ERTS_ALC_T_BIF_TIMER, tmr); - } -} - static void scheduled_hl_timer_destroy(void *vtmr) { - hl_timer_destroy((ErtsHLTimer *) vtmr); + ErtsTimer * tmr = (ErtsTimer *) vtmr; + int btm = !!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR); + timer_destroy((ErtsTimer *) vtmr, 0, btm); } static void @@ -999,14 +1176,13 @@ schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs) } if (!(roflgs & ERTS_TMR_ROFLG_BIF_TMR)) - size = ERTS_HL_PTIMER_SIZE; - else { - /* - * Message buffer can be dropped at - * once... - */ size = sizeof(ErtsHLTimer); - + else { + /* Message buffer already dropped... */ + size = sizeof(ErtsBifTimer); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + size += sizeof(ErtsMagicIndirectionWord); +#endif } erts_schedule_thr_prgr_later_cleanup_op( @@ -1014,18 +1190,6 @@ schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs) &tmr->time.cleanup, size); } -static ERTS_INLINE void -hl_timer_pre_dec_refc(ErtsHLTimer *tmr) -{ -#ifdef ERTS_HLT_DEBUG - erts_aint_t refc; - refc = erts_smp_atomic32_dec_read_nob(&tmr->head.refc); - ERTS_HLT_ASSERT(refc > 0); -#else - erts_smp_atomic32_dec_nob(&tmr->head.refc); -#endif -} - static ERTS_INLINE void hl_timer_dec_refc(ErtsHLTimer *tmr, Uint32 roflgs) { @@ -1060,32 +1224,17 @@ check_canceled_queue(ErtsSchedulerData *esdp, ErtsHLTimerService *srv) #ifdef ERTS_MAGIC_REF_BIF_TIMERS -static erts_smp_atomic_t * -mbin_to_btmref__(ErtsMagicBinary *mbin) -{ - return erts_smp_binary_to_magic_indirection((Binary *) mbin); -} - -static ERTS_INLINE void -magic_binary_init(ErtsMagicBinary *mbin, ErtsHLTimer *tmr) -{ - erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin); - erts_smp_atomic_init_nob(aptr, (erts_aint_t) tmr); -} - -static ERTS_INLINE ErtsHLTimer * -magic_binary_to_btm(ErtsMagicBinary *mbin) +static int +bif_timer_ref_destructor(Binary *unused) { - erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin); - ErtsHLTimer *tmr = (ErtsHLTimer *) erts_smp_atomic_read_nob(aptr); - ERTS_HLT_ASSERT(!tmr || tmr->btm.mbin == mbin); - return tmr; + return 1; } static ERTS_INLINE void -btm_clear_magic_binary(ErtsHLTimer *tmr) +btm_clear_magic_binary(ErtsBifTimer *tmr) { erts_smp_atomic_t *aptr = mbin_to_btmref__(tmr->btm.mbin); + Uint32 roflgs = tmr->type.head.roflgs; #ifdef ERTS_HLT_DEBUG erts_aint_t tval = erts_smp_atomic_xchg_nob(aptr, (erts_aint_t) NULL); @@ -1093,23 +1242,117 @@ btm_clear_magic_binary(ErtsHLTimer *tmr) #else erts_smp_atomic_set_nob(aptr, (erts_aint_t) NULL); #endif - hl_timer_dec_refc(tmr, tmr->head.roflgs); + if (roflgs & ERTS_TMR_ROFLG_HLT) + hl_timer_dec_refc(&tmr->type.hlt, roflgs); + else + tw_timer_dec_refc(&tmr->type.twt); } -static int -bif_timer_ref_destructor(Binary *unused) +#endif /* ERTS_MAGIC_REF_BIF_TIMERS */ + +static ERTS_INLINE void +bif_timer_timeout(ErtsHLTimerService *srv, ErtsBifTimer *tmr, + Uint32 roflgs, int is_hlt) { - return 1; -} + erts_aint32_t state; + + ERTS_HLT_ASSERT(tmr->type.head.roflgs == roflgs); + ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR); + ERTS_HLT_ASSERT(!!is_hlt == !!(roflgs & ERTS_TMR_ROFLG_HLT)); + + state = erts_smp_atomic32_cmpxchg_acqb(&tmr->btm.state, + ERTS_TMR_STATE_TIMED_OUT, + ERTS_TMR_STATE_ACTIVE); + ERTS_HLT_ASSERT(state == ERTS_TMR_STATE_CANCELED + || state == ERTS_TMR_STATE_ACTIVE); + + if (state == ERTS_TMR_STATE_ACTIVE) { + Process *proc; + int dec_refc = 0; + Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME); + +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + btm_clear_magic_binary(tmr); #endif + if (is_reg_name) { + Eterm term; + if (is_hlt) + term = tmr->type.hlt.receiver.name; + else + term = tmr->type.twt.receiver.name; + ERTS_HLT_ASSERT(is_atom(term)); + term = erts_whereis_name_to_id(NULL, term); + proc = erts_proc_lookup(term); + } + else { + ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_PROC); + if (is_hlt) + proc = tmr->type.hlt.receiver.proc; + else + proc = tmr->type.twt.receiver.proc; + ERTS_HLT_ASSERT(proc); + } + if (proc) { + if (!ERTS_PROC_IS_EXITING(proc)) { + ErtsMessage *mp; + erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM); + mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = tmr->btm.bp; + erts_queue_message(proc, ERTS_PROC_LOCK_BTM, mp, + tmr->btm.message, am_clock_service); + tmr->btm.bp = NULL; +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + if (tmr->btm.proc_list.next) { + proc_btm_list_delete(&proc->bif_timers, tmr); + dec_refc = 1; + } +#else + if (tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + proc_btm_rbt_delete(&proc->bif_timers, tmr); + tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + dec_refc = 1; + } +#endif + erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); + } + if (dec_refc) + timer_pre_dec_refc((ErtsTimer *) tmr); + } + if (tmr->btm.bp) + free_message_buffer(tmr->btm.bp); + } + +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + btm_rbt_delete(&srv->btm_tree, tmr); + tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + } +#endif + + +} + +static void +tw_bif_timer_timeout(void *vbtmp) +{ +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsHLTimerService *srv = NULL; +#else + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsHLTimerService *srv = esdp->timer_service; +#endif + ErtsBifTimer *btmp = (ErtsBifTimer *) vbtmp; + bif_timer_timeout(srv, btmp, btmp->type.head.roflgs, 0); + tw_timer_dec_refc(&btmp->type.twt); +} -static ErtsHLTimer * +static ErtsTimer * create_hl_timer(ErtsSchedulerData *esdp, ErtsMonotonicTime timeout_pos, int short_time, ErtsTmrType type, - void *rcvrp, Eterm rcvr, Eterm acsr, + void *rcvrp, Eterm rcvr, Eterm msg, #ifdef ERTS_MAGIC_REF_BIF_TIMERS ErtsMagicBinary *mbin, @@ -1134,7 +1377,7 @@ create_hl_timer(ErtsSchedulerData *esdp, if (type != ERTS_TMR_BIF) { tmr = erts_alloc(ERTS_ALC_T_HL_PTIMER, - ERTS_HL_PTIMER_SIZE); + sizeof(ErtsHLTimer)); tmr->timeout = timeout_pos; switch (type) { @@ -1170,19 +1413,18 @@ create_hl_timer(ErtsSchedulerData *esdp, } else { /* ERTS_TMR_BIF */ - Uint hsz; if (short_time) { - tmr = bif_timer_pre_alloc(); + tmr = (ErtsHLTimer *) bif_timer_pre_alloc(); if (!tmr) goto alloc_bif_timer; roflgs |= ERTS_TMR_ROFLG_PRE_ALC; } else { alloc_bif_timer: - tmr = erts_alloc(ERTS_ALC_T_BIF_TIMER, - ERTS_BIF_TIMER_SIZE); - } + tmr = (ErtsHLTimer *) erts_alloc(ERTS_ALC_T_BIF_TIMER, + sizeof(ErtsBifTimer)); + } tmr->timeout = timeout_pos; @@ -1199,37 +1441,19 @@ create_hl_timer(ErtsSchedulerData *esdp, refc = 1; } - hsz = is_immed(msg) ? ((Uint) 0) : size_object(msg); - if (!hsz) { - tmr->btm.message = msg; - tmr->btm.bp = NULL; - } - else { - ErlHeapFragment *bp = new_message_buffer(hsz); - Eterm *hp = bp->mem; - tmr->btm.message = copy_struct(msg, hsz, &hp, &bp->off_heap); - tmr->btm.bp = bp; - } + refc += init_btm_specifics(esdp, + (ErtsBifTimer *) tmr, + msg, #ifdef ERTS_MAGIC_REF_BIF_TIMERS - tmr->btm.mbin = mbin; - erts_refc_inc(&mbin->refc, 1); - magic_binary_init(mbin, tmr); - refc += 1; /* from magic binary... */ - tmr->btm.proc_list.next = NULL; + mbin #else - tmr->btm.refn[0] = refn[0]; - tmr->btm.refn[1] = refn[1]; - tmr->btm.refn[2] = refn[2]; - - tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - - btm_rbt_insert(&srv->btm_tree, tmr); + refn #endif + ); } tmr->head.roflgs = roflgs; erts_smp_atomic32_init_nob(&tmr->head.refc, refc); - erts_smp_atomic32_init_nob(&tmr->state, ERTS_TMR_STATE_ACTIVE); if (!srv->next_timeout || tmr->timeout < srv->next_timeout->timeout) { @@ -1256,72 +1480,7 @@ create_hl_timer(ErtsSchedulerData *esdp, ERTS_HLT_HDBG_CHK_SRV(srv); - return tmr; -} - -static ERTS_INLINE void -hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs) -{ - ErtsProcLocks proc_locks = ERTS_PROC_LOCKS_MSG_SEND; - Process *proc; - int queued_message = 0; - int dec_refc = 0; - Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME); - ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR); - -#ifdef ERTS_MAGIC_REF_BIF_TIMERS - btm_clear_magic_binary(tmr); -#endif - - if (is_reg_name) { - Eterm pid; - ERTS_HLT_ASSERT(is_atom(tmr->receiver.name)); - pid = erts_whereis_name_to_id(NULL, tmr->receiver.name); - proc = erts_proc_lookup(pid); - } - else { - ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_PROC); - ERTS_HLT_ASSERT(tmr->receiver.proc); - - proc = tmr->receiver.proc; - proc_locks |= ERTS_PROC_LOCK_BTM; - } - if (proc) { - erts_smp_proc_lock(proc, proc_locks); - /* - * If process is exiting, let it clean up - * the btm tree by itself (it may be in - * the middle of tree destruction). - */ - if (!ERTS_PROC_IS_EXITING(proc)) { - ErtsMessage *mp = erts_alloc_message(0, NULL); - mp->data.heap_frag = tmr->btm.bp; - erts_queue_message(proc, proc_locks, mp, - tmr->btm.message, am_clock_service); - erts_smp_proc_unlock(proc, ERTS_PROC_LOCKS_MSG_SEND); - queued_message = 1; - proc_locks &= ~ERTS_PROC_LOCKS_MSG_SEND; - tmr->btm.bp = NULL; -#ifdef ERTS_MAGIC_REF_BIF_TIMERS - if (tmr->btm.proc_list.next) { - proc_btm_list_delete(&proc->bif_timers, tmr); - dec_refc = 1; - } -#else - if (tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { - proc_btm_rbt_delete(&proc->bif_timers, tmr); - tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - dec_refc = 1; - } -#endif - } - if (proc_locks) - erts_smp_proc_unlock(proc, proc_locks); - if (dec_refc) - hl_timer_pre_dec_refc(tmr); - } - if (!queued_message && tmr->btm.bp) - free_message_buffer(tmr->btm.bp); + return (ErtsTimer *) tmr; } static ERTS_INLINE void @@ -1340,49 +1499,27 @@ hlt_port_timeout(ErtsHLTimer *tmr) static void hlt_timeout(ErtsHLTimer *tmr, void *vsrv) { -#if !defined(ERTS_MAGIC_REF_BIF_TIMERS) || defined(ERTS_HLT_HARD_DEBUG) ErtsHLTimerService *srv = (ErtsHLTimerService *) vsrv; -#endif Uint32 roflgs; - erts_aint32_t state; ERTS_HLT_HDBG_CHK_SRV(srv); roflgs = tmr->head.roflgs; ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_HLT); - state = erts_smp_atomic32_cmpxchg_acqb(&tmr->state, - ERTS_TMR_STATE_TIMED_OUT, - ERTS_TMR_STATE_ACTIVE); - - ERTS_HLT_ASSERT(state == ERTS_TMR_STATE_CANCELED - || state == ERTS_TMR_STATE_ACTIVE); - - if (state == ERTS_TMR_STATE_ACTIVE) { - - if (roflgs & ERTS_TMR_ROFLG_BIF_TMR) - hlt_bif_timer_timeout(tmr, roflgs); - else if (roflgs & ERTS_TMR_ROFLG_PROC) - hlt_proc_timeout(tmr); - else if (roflgs & ERTS_TMR_ROFLG_PORT) - hlt_port_timeout(tmr); - else { - ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_CALLBACK); - (*tmr->receiver.callback)(tmr->head.u.arg); - } - + if (roflgs & ERTS_TMR_ROFLG_BIF_TMR) + bif_timer_timeout(srv, (ErtsBifTimer *) tmr, roflgs, 1); + else if (roflgs & ERTS_TMR_ROFLG_PROC) + hlt_proc_timeout(tmr); + else if (roflgs & ERTS_TMR_ROFLG_PORT) + hlt_port_timeout(tmr); + else { + ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_CALLBACK); + (*tmr->receiver.callback)(tmr->head.u.arg); } tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; -#ifndef ERTS_MAGIC_REF_BIF_TIMERS - if ((roflgs & ERTS_TMR_ROFLG_BIF_TMR) - && tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { - btm_rbt_delete(&srv->btm_tree, tmr); - tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - } -#endif - ERTS_HLT_HDBG_CHK_SRV(srv); hl_timer_dec_refc(tmr, roflgs); @@ -1460,17 +1597,6 @@ hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr) ERTS_HLT_HDBG_CHK_SRV(srv); -#ifndef ERTS_MAGIC_REF_BIF_TIMERS - if (tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR) { - - if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { - btm_rbt_delete(&srv->btm_tree, tmr); - tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - } - - } -#endif - if (tmr->time.tree.parent == ERTS_HLT_PFIELD_NOT_IN_TABLE) { /* Already removed... */ ERTS_HLT_HDBG_CHK_SRV(srv); @@ -1541,6 +1667,17 @@ cleanup_sched_local_canceled_timer(ErtsSchedulerData *esdp, ERTS_HLT_ASSERT(esdp == erts_get_scheduler_data()); ERTS_HLT_ASSERT((tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK) == (Uint32) esdp->no); + +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + if (roflgs & ERTS_TMR_ROFLG_BIF_TMR) { + ErtsBifTimer *btm = (ErtsBifTimer *) tmr; + if (btm->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + btm_rbt_delete(&esdp->timer_service->btm_tree, btm); + btm->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + } + } +#endif + if (roflgs & ERTS_TMR_ROFLG_HLT) { hlt_delete_timer(esdp, &tmr->hlt); hl_timer_dec_refc(&tmr->hlt, roflgs); @@ -1813,18 +1950,18 @@ Uint erts_bif_timer_memory_size(void) } static BIF_RETTYPE -setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos, - int short_time, Eterm rcvr, Eterm acsr, - Eterm msg, int wrap) +setup_bif_timer(Process *c_p, int twheel, ErtsMonotonicTime timeout_pos, + int short_time, Eterm rcvr, Eterm msg, int wrap) { BIF_RETTYPE ret; Eterm ref, tmo_msg, *hp; - ErtsHLTimer *tmr; + ErtsBifTimer *tmr; ErtsSchedulerData *esdp; #ifdef ERTS_MAGIC_REF_BIF_TIMERS Binary *mbin; #endif Eterm tmp_hp[4]; + ErtsCreateTimerFunc create_timer; if (is_not_internal_pid(rcvr) && is_not_atom(rcvr)) goto badarg; @@ -1846,14 +1983,16 @@ setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos, tmo_msg = wrap ? TUPLE3(tmp_hp, am_timeout, ref, msg) : msg; - tmr = create_hl_timer(esdp, timeout_pos, short_time, - ERTS_TMR_BIF, NULL, rcvr, acsr, tmo_msg, + create_timer = twheel ? create_tw_timer : create_hl_timer; + tmr = (ErtsBifTimer *) create_timer(esdp, timeout_pos, + short_time, ERTS_TMR_BIF, + NULL, rcvr, tmo_msg, #ifdef ERTS_MAGIC_REF_BIF_TIMERS - (ErtsMagicBinary *) mbin, + (ErtsMagicBinary *) mbin, #else - internal_ordinary_ref_numbers(ref), + internal_ordinary_ref_numbers(ref), #endif - NULL, NULL); + NULL, NULL); if (is_internal_pid(rcvr)) { Process *proc = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, @@ -1862,11 +2001,19 @@ setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos, if (!proc) { #ifdef ERTS_MAGIC_REF_BIF_TIMERS btm_clear_magic_binary(tmr); +#else + if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + btm_rbt_delete(&esdp->timer_service->btm_tree, tmr); + tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + } #endif if (tmr->btm.bp) free_message_buffer(tmr->btm.bp); - hlt_delete_timer(esdp, tmr); - hl_timer_destroy(tmr); + if (twheel) + cancel_tw_timer(esdp, &tmr->type.twt); + else + hlt_delete_timer(esdp, &tmr->type.hlt); + timer_destroy((ErtsTimer *) tmr, twheel, 1); } else { #ifdef ERTS_MAGIC_REF_BIF_TIMERS @@ -1875,7 +2022,10 @@ setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos, proc_btm_rbt_insert(&proc->bif_timers, tmr); #endif erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); - tmr->receiver.proc = proc; + if (twheel) + tmr->type.twt.receiver.proc = proc; + else + tmr->type.hlt.receiver.proc = proc; } } @@ -1889,13 +2039,13 @@ badarg: } static int -cancel_bif_timer(ErtsHLTimer *tmr) +cancel_bif_timer(ErtsBifTimer *tmr) { erts_aint_t state; Uint32 roflgs; int res; - state = erts_smp_atomic32_cmpxchg_acqb(&tmr->state, + state = erts_smp_atomic32_cmpxchg_acqb(&tmr->btm.state, ERTS_TMR_STATE_CANCELED, ERTS_TMR_STATE_ACTIVE); if (state != ERTS_TMR_STATE_ACTIVE) @@ -1910,10 +2060,14 @@ cancel_bif_timer(ErtsHLTimer *tmr) res = -1; - roflgs = tmr->head.roflgs; + roflgs = tmr->type.head.roflgs; if (roflgs & ERTS_TMR_ROFLG_PROC) { - Process *proc = tmr->receiver.proc; - ERTS_HLT_ASSERT(!(tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME)); + Process *proc; + + proc = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT) + ? tmr->type.hlt.receiver.proc + : tmr->type.twt.receiver.proc); + ERTS_HLT_ASSERT(!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME)); erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM); /* @@ -1941,18 +2095,25 @@ cancel_bif_timer(ErtsHLTimer *tmr) } static ERTS_INLINE Sint64 -access_btm(ErtsHLTimer *tmr, Uint32 sid, ErtsSchedulerData *esdp, int cancel) +access_btm(ErtsBifTimer *tmr, Uint32 sid, ErtsSchedulerData *esdp, int cancel) { int cncl_res; Sint64 time_left; + ErtsMonotonicTime timeout; + int is_hlt; if (!tmr) return -1; + is_hlt = !!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT); + timeout = (is_hlt + ? tmr->type.hlt.timeout + : tmr->type.twt.tw_tmr.timeout_pos); + if (!cancel) { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->state); + erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->btm.state); if (state == ERTS_TMR_STATE_ACTIVE) - return get_time_left(esdp, tmr->timeout); + return get_time_left(esdp, timeout); return -1; } @@ -1960,17 +2121,29 @@ access_btm(ErtsHLTimer *tmr, Uint32 sid, ErtsSchedulerData *esdp, int cancel) if (!cncl_res) return -1; - time_left = get_time_left(esdp, tmr->timeout); + time_left = get_time_left(esdp, timeout); if (sid != (Uint32) esdp->no) { if (cncl_res > 0) queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr); } else { - if (cncl_res > 0) - hl_timer_dec_refc(tmr, tmr->head.roflgs); - - hlt_delete_timer(esdp, tmr); +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + btm_rbt_delete(&esdp->timer_service->btm_tree, tmr); + tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + } +#endif + if (is_hlt) { + if (cncl_res > 0) + hl_timer_dec_refc(&tmr->type.hlt, tmr->type.hlt.head.roflgs); + hlt_delete_timer(esdp, &tmr->type.hlt); + } + else { + if (cncl_res > 0) + tw_timer_dec_refc(&tmr->type.twt); + cancel_tw_timer(esdp, &tmr->type.twt); + } } return time_left; @@ -2059,7 +2232,7 @@ access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info) if (mbin->destructor != bif_timer_ref_destructor) time_left = -1; else { - ErtsHLTimer *tmr; + ErtsBifTimer *tmr; Uint32 sid; tmr = magic_binary_to_btm(mbin); sid = erts_get_ref_numbers_thr_id(internal_magic_ref_numbers(tref)); @@ -2133,7 +2306,7 @@ access_sched_local_btm(Process *c_p, Eterm pid, { ErtsSchedulerData *esdp; ErtsHLTimerService *srv; - ErtsHLTimer *tmr; + ErtsBifTimer *tmr; Sint64 time_left; Process *proc; ErtsProcLocks proc_locks; @@ -2223,7 +2396,7 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp, int async, int cancel, int info, Eterm *resp) { - ErtsHLTimer *tmr; + ErtsBifTimer *tmr; Sint64 time_left; ERTS_HLT_ASSERT(c_p); @@ -2405,8 +2578,8 @@ bool_arg(Eterm val, int *argp) } static ERTS_INLINE int -parse_bif_timer_options(Eterm option_list, int *async, int *info, - int *abs, Eterm *accessor) +parse_bif_timer_options(Eterm option_list, int *async, + int *info, int *abs) { Eterm list = option_list; @@ -2416,8 +2589,6 @@ parse_bif_timer_options(Eterm option_list, int *async, int *info, *info = 1; if (abs) *abs = 0; - if (accessor) - *accessor = THE_NON_VALUE; while (is_list(list)) { Eterm *consp, *tp, opt; @@ -2457,18 +2628,20 @@ parse_bif_timer_options(Eterm option_list, int *async, int *info, } static void -exit_cancel_bif_timer(ErtsHLTimer *tmr, void *vesdp) +exit_cancel_bif_timer(ErtsBifTimer *tmr, void *vesdp) { ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp; Uint32 sid, roflgs; erts_aint_t state; + int is_hlt; - state = erts_smp_atomic32_cmpxchg_acqb(&tmr->state, + state = erts_smp_atomic32_cmpxchg_acqb(&tmr->btm.state, ERTS_TMR_STATE_CANCELED, ERTS_TMR_STATE_ACTIVE); - roflgs = tmr->head.roflgs; + roflgs = tmr->type.head.roflgs; sid = roflgs & ERTS_TMR_ROFLG_SID_MASK; + is_hlt = !!(roflgs & ERTS_TMR_ROFLG_HLT); ERTS_HLT_ASSERT(sid == erts_get_ref_numbers_thr_id(ERTS_BTM_HLT2REFN(tmr))); #ifdef ERTS_MAGIC_REF_BIF_TIMERS @@ -2479,29 +2652,33 @@ exit_cancel_bif_timer(ErtsHLTimer *tmr, void *vesdp) tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; #endif - if (sid == (Uint32) esdp->no) { - if (state == ERTS_TMR_STATE_ACTIVE) { + if (state == ERTS_TMR_STATE_ACTIVE) { #ifdef ERTS_MAGIC_REF_BIF_TIMERS - btm_clear_magic_binary(tmr); + btm_clear_magic_binary(tmr); #endif - if (tmr->btm.bp) - free_message_buffer(tmr->btm.bp); - hlt_delete_timer(esdp, tmr); + if (tmr->btm.bp) + free_message_buffer(tmr->btm.bp); + + if (sid != (Uint32) esdp->no) { + queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr); + return; + } + +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + btm_rbt_delete(&esdp->timer_service->btm_tree, tmr); + tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; } - hl_timer_dec_refc(tmr, roflgs); - } - else { - if (state == ERTS_TMR_STATE_ACTIVE) { -#ifdef ERTS_MAGIC_REF_BIF_TIMERS - btm_clear_magic_binary(tmr); #endif - if (tmr->btm.bp) - free_message_buffer(tmr->btm.bp); - queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr); - } - else - hl_timer_dec_refc(tmr, roflgs); + if (is_hlt) + hlt_delete_timer(esdp, &tmr->type.hlt); + else + cancel_tw_timer(esdp, &tmr->type.twt); } + if (is_hlt) + hl_timer_dec_refc(&tmr->type.hlt, roflgs); + else + tw_timer_dec_refc(&tmr->type.twt); } #ifdef ERTS_HLT_DEBUG @@ -2571,10 +2748,11 @@ int erts_cancel_bif_timers(Process *p, ErtsBifTimers **btm, void **vyspp) static ERTS_INLINE int parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg, ErtsMonotonicTime *conv_arg, int abs, - ErtsMonotonicTime *tposp, int *stimep) + ErtsMonotonicTime *tposp, int *stimep, + ErtsMonotonicTime *msp) { - ErtsMonotonicTime t; - + ErtsMonotonicTime t, now; + if (!term_to_Sint64(arg, &t)) { ERTS_HLT_ASSERT(!is_small(arg)); if (!is_big(arg)) @@ -2589,22 +2767,30 @@ parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg, if (conv_arg) *conv_arg = t; + now = erts_get_monotonic_time(esdp); + if (abs) { t += -1*ERTS_MONOTONIC_OFFSET_MSEC; /* external to internal */ if (t < ERTS_MONOTONIC_TO_MSEC(ERTS_MONOTONIC_BEGIN)) return 1; if (t > ERTS_MONOTONIC_TO_MSEC(ERTS_MONOTONIC_END)) return 1; + if (msp) + *msp = t - ERTS_MONOTONIC_TO_MSEC(now); + *stimep = (t - ERTS_MONOTONIC_TO_MSEC(esdp->last_monotonic_time) < ERTS_BIF_TIMER_SHORT_TIME); *tposp = ERTS_MSEC_TO_CLKTCKS(t); } else { - ErtsMonotonicTime now, ticks; + ErtsMonotonicTime ticks; if (t < 0) return -1; + if (msp) + *msp = t; + ticks = ERTS_MSEC_TO_CLKTCKS(t); if (ERTS_CLKTCK_RESOLUTION > 1000 && ticks < 0) @@ -2612,7 +2798,6 @@ parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg, ERTS_HLT_ASSERT(ticks >= 0); - now = erts_get_monotonic_time(esdp); ticks += ERTS_MONOTONIC_TO_CLKTCKS(now-1); ticks += 1; @@ -2635,66 +2820,68 @@ parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg, BIF_RETTYPE send_after_3(BIF_ALIST_3) { - ErtsMonotonicTime timeout_pos; + ErtsMonotonicTime timeout_pos, tmo; int short_time, tres; - tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL, - 0, &timeout_pos, &short_time); + tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, + NULL, 0, &timeout_pos, &short_time, &tmo); if (tres != 0) BIF_ERROR(BIF_P, BADARG); - return setup_bif_timer(BIF_P, timeout_pos, short_time, - BIF_ARG_2, BIF_ARG_2, BIF_ARG_3, 0); + return setup_bif_timer(BIF_P, tmo < ERTS_TIMER_WHEEL_MSEC, + timeout_pos, short_time, BIF_ARG_2, + BIF_ARG_3, 0); } BIF_RETTYPE send_after_4(BIF_ALIST_4) { - ErtsMonotonicTime timeout_pos; - Eterm accessor; + ErtsMonotonicTime timeout_pos, tmo; int short_time, abs, tres; - if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs, &accessor)) + if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs)) BIF_ERROR(BIF_P, BADARG); tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL, - abs, &timeout_pos, &short_time); + abs, &timeout_pos, &short_time, &tmo); if (tres != 0) BIF_ERROR(BIF_P, BADARG); - return setup_bif_timer(BIF_P, timeout_pos, short_time, - BIF_ARG_2, accessor, BIF_ARG_3, 0); + return setup_bif_timer(BIF_P, tmo < ERTS_TIMER_WHEEL_MSEC, + timeout_pos, short_time, BIF_ARG_2, + BIF_ARG_3, 0); } BIF_RETTYPE start_timer_3(BIF_ALIST_3) { - ErtsMonotonicTime timeout_pos; + ErtsMonotonicTime timeout_pos, tmo; int short_time, tres; tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL, - 0, &timeout_pos, &short_time); + 0, &timeout_pos, &short_time, &tmo); if (tres != 0) BIF_ERROR(BIF_P, BADARG); - return setup_bif_timer(BIF_P, timeout_pos, short_time, - BIF_ARG_2, BIF_ARG_2, BIF_ARG_3, !0); + return setup_bif_timer(BIF_P, tmo < ERTS_TIMER_WHEEL_MSEC, + timeout_pos, short_time, BIF_ARG_2, + BIF_ARG_3, !0); } BIF_RETTYPE start_timer_4(BIF_ALIST_4) { - ErtsMonotonicTime timeout_pos; - Eterm accessor; + ErtsMonotonicTime timeout_pos, tmo; int short_time, abs, tres; - if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs, &accessor)) + if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs)) BIF_ERROR(BIF_P, BADARG); tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL, - abs, &timeout_pos, &short_time); + abs, &timeout_pos, &short_time, &tmo); if (tres != 0) BIF_ERROR(BIF_P, BADARG); - return setup_bif_timer(BIF_P, timeout_pos, short_time, - BIF_ARG_2, accessor, BIF_ARG_3, !0); + return setup_bif_timer(BIF_P, tmo < ERTS_TIMER_WHEEL_MSEC, + timeout_pos, short_time, BIF_ARG_2, + BIF_ARG_3, !0); } BIF_RETTYPE cancel_timer_1(BIF_ALIST_1) @@ -2707,7 +2894,7 @@ BIF_RETTYPE cancel_timer_2(BIF_ALIST_2) BIF_RETTYPE ret; int async, info; - if (parse_bif_timer_options(BIF_ARG_2, &async, &info, NULL, NULL)) + if (parse_bif_timer_options(BIF_ARG_2, &async, &info, NULL)) return access_bif_timer(BIF_P, BIF_ARG_1, 1, async, info); ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); @@ -2724,7 +2911,7 @@ BIF_RETTYPE read_timer_2(BIF_ALIST_2) BIF_RETTYPE ret; int async; - if (parse_bif_timer_options(BIF_ARG_2, &async, NULL, NULL, NULL)) + if (parse_bif_timer_options(BIF_ARG_2, &async, NULL, NULL)) return access_bif_timer(BIF_P, BIF_ARG_1, 0, async, 1); ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); @@ -2739,14 +2926,13 @@ start_callback_timer(ErtsSchedulerData *esdp, void *arg) { - if (twt) - create_tw_timer(esdp, ERTS_TMR_CALLBACK, NULL, - callback, arg, timeout_pos); - else - create_hl_timer(esdp, timeout_pos, 0, - ERTS_TMR_CALLBACK, NULL, - NIL, THE_NON_VALUE, NIL, - NULL, callback, arg); + ErtsCreateTimerFunc create_timer = (twt + ? create_tw_timer + : create_hl_timer); + (void) create_timer(esdp, timeout_pos, 0, + ERTS_TMR_CALLBACK, NULL, + NIL, THE_NON_VALUE, NULL, + callback, arg); } typedef struct { @@ -2823,18 +3009,18 @@ set_proc_timer_common(Process *c_p, ErtsSchedulerData *esdp, Sint64 tmo, if (tmo == 0) c_p->flags |= F_TIMO; else { + ErtsCreateTimerFunc create_timer; c_p->flags |= F_INSLPQUEUE; c_p->flags &= ~F_TIMO; - if (tmo < ERTS_TIMER_WHEEL_MSEC) - tmr = (void *) create_tw_timer(esdp, ERTS_TMR_PROC, (void *) c_p, - NULL, NULL, timeout_pos); - else - tmr = (void *) create_hl_timer(esdp, timeout_pos, short_time, - ERTS_TMR_PROC, (void *) c_p, - c_p->common.id, THE_NON_VALUE, - NIL, NULL, NULL, NULL); + create_timer = (tmo < ERTS_TIMER_WHEEL_MSEC + ? create_tw_timer + : create_hl_timer); + tmr = (void *) create_timer(esdp, timeout_pos, short_time, + ERTS_TMR_PROC, (void *) c_p, + c_p->common.id, THE_NON_VALUE, + NULL, NULL, NULL); erts_smp_atomic_set_relb(&c_p->common.timer, (erts_aint_t) tmr); } } @@ -2850,7 +3036,7 @@ erts_set_proc_timer_term(Process *c_p, Eterm etmo) == ERTS_PTMR_NONE); tres = parse_timeout_pos(esdp, etmo, &tmo, 0, - &timeout_pos, &short_time); + &timeout_pos, &short_time, NULL); if (tres != 0) return tres; @@ -2908,6 +3094,7 @@ erts_set_port_timer(Port *c_prt, Sint64 tmo) void *tmr; ErtsSchedulerData *esdp = erts_get_scheduler_data(); ErtsMonotonicTime timeout_pos; + ErtsCreateTimerFunc create_timer; if (erts_smp_atomic_read_nob(&c_prt->common.timer) != ERTS_PTMR_NONE) erts_cancel_port_timer(c_prt); @@ -2916,13 +3103,12 @@ erts_set_port_timer(Port *c_prt, Sint64 tmo) timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp), tmo); - if (tmo < ERTS_TIMER_WHEEL_MSEC) - tmr = (void *) create_tw_timer(esdp, ERTS_TMR_PORT, (void *) c_prt, - NULL, NULL, timeout_pos); - else - tmr = (void *) create_hl_timer(esdp, timeout_pos, 0, ERTS_TMR_PORT, - (void *) c_prt, c_prt->common.id, - THE_NON_VALUE, NIL, NULL, NULL, NULL); + create_timer = (tmo < ERTS_TIMER_WHEEL_MSEC + ? create_tw_timer + : create_hl_timer); + tmr = (void *) create_timer(esdp, timeout_pos, 0, ERTS_TMR_PORT, + (void *) c_prt, c_prt->common.id, + THE_NON_VALUE, NULL, NULL, NULL); erts_smp_atomic_set_relb(&c_prt->common.timer, (erts_aint_t) tmr); } @@ -2976,25 +3162,39 @@ typedef struct { } ErtsBTMPrint; static void -btm_print(ErtsHLTimer *tmr, void *vbtmp) +btm_print(ErtsBifTimer *tmr, void *vbtmp, ErtsMonotonicTime tpos, int is_hlt) { ErtsBTMPrint *btmp = (ErtsBTMPrint *) vbtmp; ErtsMonotonicTime left; Eterm receiver; #ifdef ERTS_MAGIC_REF_BIF_TIMERS - if (!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) + if (!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) return; #endif - if (tmr->timeout <= btmp->now) - left = 0; - else - left = ERTS_CLKTCKS_TO_MSEC(tmr->timeout - btmp->now); + if (is_hlt) { + ERTS_HLT_ASSERT(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT); + if (tmr->type.hlt.timeout <= btmp->now) + left = 0; + else + left = ERTS_CLKTCKS_TO_MSEC(tmr->type.hlt.timeout - btmp->now); - receiver = ((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME) - ? tmr->receiver.name - : tmr->receiver.proc->common.id); + receiver = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) + ? tmr->type.hlt.receiver.name + : tmr->type.hlt.receiver.proc->common.id); + } + else { + ERTS_HLT_ASSERT(!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT)); + if (tpos <= btmp->now) + left = 0; + else + left = ERTS_CLKTCKS_TO_MSEC(tpos - btmp->now); + + receiver = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) + ? tmr->type.twt.receiver.name + : tmr->type.twt.receiver.proc->common.id); + } erts_print(btmp->to, btmp->to_arg, "=timer:%T\n" @@ -3005,6 +3205,36 @@ btm_print(ErtsHLTimer *tmr, void *vbtmp) (Sint64) left); } +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + +static void +hlt_btm_print(ErtsHLTimer *tmr, void *vbtmp) +{ + btm_print((ErtsBifTimer *) tmr, vbtmp, 0, 1); +} + +static void +twt_btm_print(void *vbtmp, ErtsMonotonicTime tpos, void *vtwtp) +{ + btm_print((ErtsBifTimer *) vtwtp, vbtmp, tpos, 0); +} + +#else + +static void +btm_tree_print(ErtsBifTimer *tmr, void *vbtmp) +{ + int is_hlt = !!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT); + ErtsMonotonicTime tpos; + if (is_hlt) + tpos = 0; + else + tpos = tmr->type.twt.tw_tmr.timeout_pos; + btm_print(tmr, vbtmp, tpos, is_hlt); +} + +#endif + void erts_print_bif_timer_info(fmtfn_t to, void *to_arg) { @@ -3023,9 +3253,13 @@ erts_print_bif_timer_info(fmtfn_t to, void *to_arg) ErtsHLTimerService *srv = erts_aligned_scheduler_data[six].esd.timer_service; #ifdef ERTS_MAGIC_REF_BIF_TIMERS - time_rbt_foreach(srv->time_tree, btm_print, (void *) &btmp); + ErtsTimerWheel *twheel = + erts_aligned_scheduler_data[six].esd.timer_wheel; + erts_twheel_debug_foreach(twheel, tw_bif_timer_timeout, + twt_btm_print, (void *) &btmp); + time_rbt_foreach(srv->time_tree, hlt_btm_print, (void *) &btmp); #else - btm_rbt_foreach(srv->btm_tree, btm_print, (void *) &btmp); + btm_rbt_foreach(srv->btm_tree, btm_tree_print, (void *) &btmp); #endif } } @@ -3039,23 +3273,52 @@ typedef struct { } ErtsBTMForeachDebug; static void -debug_btm_foreach(ErtsHLTimer *tmr, void *vbtmfd) +debug_btm_foreach(ErtsBifTimer *tmr, void *vbtmfd, int is_hlt) { #ifdef ERTS_MAGIC_REF_BIF_TIMERS - if (!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) + if (!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) return; #endif - if (erts_smp_atomic32_read_nob(&tmr->state) == ERTS_TMR_STATE_ACTIVE) { + if (erts_smp_atomic32_read_nob(&tmr->btm.state) == ERTS_TMR_STATE_ACTIVE) { ErtsBTMForeachDebug *btmfd = (ErtsBTMForeachDebug *) vbtmfd; - (*btmfd->func)(((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME) - ? tmr->receiver.name - : tmr->receiver.proc->common.id), - tmr->btm.message, - tmr->btm.bp, - btmfd->arg); + Eterm id; + if (is_hlt) + id = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) + ? tmr->type.hlt.receiver.name + : tmr->type.hlt.receiver.proc->common.id); + else + id = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) + ? tmr->type.twt.receiver.name + : tmr->type.twt.receiver.proc->common.id); + (*btmfd->func)(id, tmr->btm.message, tmr->btm.bp, btmfd->arg); } } +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + +static void +hlt_debug_btm_foreach(ErtsHLTimer *tmr, void *vbtmfd) +{ + debug_btm_foreach((ErtsBifTimer *) tmr, vbtmfd, 1); +} + +static void +twt_debug_btm_foreach(void *vbtmfd, ErtsMonotonicTime tpos, void *vtwtp) +{ + debug_btm_foreach((ErtsBifTimer *) vtwtp, vbtmfd, 0); +} + +#else + +static void +debug_btm_tree_foreach(ErtsBifTimer *tmr, void *vbtmfd) +{ + debug_btm_foreach(tmr, vbtmfd, !!(tmr->type.head.roflgs + & ERTS_TMR_ROFLG_HLT)); +} + +#endif + void erts_debug_bif_timer_foreach(void (*func)(Eterm, Eterm, @@ -3076,13 +3339,17 @@ erts_debug_bif_timer_foreach(void (*func)(Eterm, ErtsHLTimerService *srv = erts_aligned_scheduler_data[six].esd.timer_service; #ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsTimerWheel *twheel = + erts_aligned_scheduler_data[six].esd.timer_wheel; + erts_twheel_debug_foreach(twheel, tw_bif_timer_timeout, + twt_debug_btm_foreach, + (void *) &btmfd); time_rbt_foreach(srv->time_tree, - debug_btm_foreach, + hlt_debug_btm_foreach, (void *) &btmfd); - #else btm_rbt_foreach(srv->btm_tree, - debug_btm_foreach, + debug_btm_tree_foreach, (void *) &btmfd); #endif } @@ -3136,7 +3403,7 @@ debug_tw_callback_timer(void *vdfct, ErtsDebugForeachCallbackTimer *dfct = (ErtsDebugForeachCallbackTimer *) vdfct; - if (twtp->u.callback == dfct->tclbk) + if (twtp->receiver.callback == dfct->tclbk) (*dfct->func)(dfct->arg, timeout_pos, twtp->head.u.arg); diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h index 33cdf217d7..f70fcdd1c0 100644 --- a/erts/emulator/beam/erl_hl_timer.h +++ b/erts/emulator/beam/erl_hl_timer.h @@ -21,7 +21,7 @@ #ifndef ERL_HL_TIMER_H__ #define ERL_HL_TIMER_H__ -typedef struct ErtsHLTimer_ ErtsBifTimers; +typedef struct ErtsBifTimer_ ErtsBifTimers; typedef struct ErtsHLTimerService_ ErtsHLTimerService; #include "sys.h" -- cgit v1.2.3 From 79ac974710ecab6dfe37b23b4fd3b52e760b3504 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 3 Feb 2017 15:50:51 +0100 Subject: Rearrange timer struct fields in order to simplify --- erts/emulator/beam/erl_hl_timer.c | 177 +++++++++++++++----------------------- erts/emulator/beam/erl_time.h | 20 +++-- erts/emulator/beam/time.c | 32 +++---- 3 files changed, 98 insertions(+), 131 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index 102a739b13..8e47fdd674 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -164,21 +164,21 @@ typedef struct { void *arg; erts_atomic_t next; } u; + union { + Process *proc; + Port *port; + Eterm name; + void (*callback)(void *); + } receiver; } ErtsTmrHead; struct ErtsHLTimer_ { ErtsTmrHead head; /* NEED to be first! */ + ErtsMonotonicTime timeout; union { ErtsThrPrgrLaterOp cleanup; ErtsHLTimerTimeTree tree; } time; - ErtsMonotonicTime timeout; - union { - Process *proc; - Port *port; - Eterm name; - void (*callback)(void *); - } receiver; #ifdef ERTS_HLT_HARD_DEBUG int pending_timeout; @@ -188,12 +188,9 @@ struct ErtsHLTimer_ { typedef struct { ErtsTmrHead head; /* NEED to be first! */ union { - Eterm name; - Process *proc; - Port *port; - void (*callback)(void *); - } receiver; - ErtsTWheelTimer tw_tmr; + ErtsTWheelTimer tw_tmr; + ErtsThrPrgrLaterOp cleanup; + } u; } ErtsTWTimer; struct ErtsBifTimer_ { @@ -948,9 +945,9 @@ schedule_tw_timer_destroy(ErtsTWTimer *tmr) * dropped at once... */ if (tmr->head.roflgs & ERTS_TMR_ROFLG_PROC) - erts_proc_dec_refc(tmr->receiver.proc); + erts_proc_dec_refc(tmr->head.receiver.proc); else if (tmr->head.roflgs & ERTS_TMR_ROFLG_PORT) - erts_port_dec_refc(tmr->receiver.port); + erts_port_dec_refc(tmr->head.receiver.port); if (!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) size = sizeof(ErtsHLTimer); @@ -965,7 +962,7 @@ schedule_tw_timer_destroy(ErtsTWTimer *tmr) erts_schedule_thr_prgr_later_cleanup_op( scheduled_tw_timer_destroy, (void *) tmr, - &tmr->tw_tmr.u.cleanup, + &tmr->u.cleanup, size); } @@ -982,7 +979,7 @@ static void tw_proc_timeout(void *vtwtp) { ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; - Process *proc = twtp->receiver.proc; + Process *proc = twtp->head.receiver.proc; if (proc_timeout_common(proc, vtwtp)) tw_timer_dec_refc(twtp); tw_timer_dec_refc(twtp); @@ -992,7 +989,7 @@ static void tw_port_timeout(void *vtwtp) { ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; - Port *port = twtp->receiver.port; + Port *port = twtp->head.receiver.port; if (port_timeout_common(port, vtwtp)) tw_timer_dec_refc(twtp); tw_timer_dec_refc(twtp); @@ -1009,14 +1006,14 @@ cancel_tw_timer(ErtsSchedulerData *esdp, ErtsTWTimer *tmr) { ERTS_HLT_ASSERT((tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK) == (Uint32) esdp->no); - erts_twheel_cancel_timer(esdp->timer_wheel, &tmr->tw_tmr); + erts_twheel_cancel_timer(esdp->timer_wheel, &tmr->u.tw_tmr); } static void tw_callback_timeout(void *vtwtp) { ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; - void (*callback)(void *) = twtp->receiver.callback; + void (*callback)(void *) = twtp->head.receiver.callback; void *arg = twtp->head.u.arg; tw_timer_dec_refc(twtp); (*callback)(arg); @@ -1060,7 +1057,7 @@ create_tw_timer(ErtsSchedulerData *esdp, } } - erts_twheel_init_timer(&tmr->tw_tmr); + erts_twheel_init_timer(&tmr->u.tw_tmr); tmr->head.roflgs |= (Uint32) esdp->no; ERTS_HLT_ASSERT((((Uint32) esdp->no) & ~ERTS_TMR_ROFLG_SID_MASK) == 0); @@ -1068,7 +1065,7 @@ create_tw_timer(ErtsSchedulerData *esdp, switch (type) { case ERTS_TMR_PROC: - tmr->receiver.proc = (Process *) rcvrp; + tmr->head.receiver.proc = (Process *) rcvrp; tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC; timeout_func = tw_proc_timeout; cancel_func = tw_ptimer_cancel; @@ -1077,7 +1074,7 @@ create_tw_timer(ErtsSchedulerData *esdp, break; case ERTS_TMR_PORT: - tmr->receiver.port = (Port *) rcvrp; + tmr->head.receiver.port = (Port *) rcvrp; tmr->head.roflgs |= ERTS_TMR_ROFLG_PORT; timeout_func = tw_port_timeout; cancel_func = tw_ptimer_cancel; @@ -1087,7 +1084,7 @@ create_tw_timer(ErtsSchedulerData *esdp, case ERTS_TMR_CALLBACK: tmr->head.u.arg = arg; - tmr->receiver.callback = callback; + tmr->head.receiver.callback = callback; tmr->head.roflgs |= ERTS_TMR_ROFLG_CALLBACK; timeout_func = tw_callback_timeout; @@ -1101,13 +1098,13 @@ create_tw_timer(ErtsSchedulerData *esdp, cancel_func = NULL; if (is_internal_pid(rcvr)) { tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC; - tmr->receiver.proc = (Process *) rcvrp; + tmr->head.receiver.proc = (Process *) rcvrp; refc = 2; } else { ERTS_HLT_ASSERT(is_atom(rcvr)); tmr->head.roflgs |= ERTS_TMR_ROFLG_REG_NAME; - tmr->receiver.name = (Eterm) rcvr; + tmr->head.receiver.name = (Eterm) rcvr; refc = 1; } @@ -1130,7 +1127,7 @@ create_tw_timer(ErtsSchedulerData *esdp, erts_smp_atomic32_init_nob(&tmr->head.refc, refc); erts_twheel_set_timer(esdp->timer_wheel, - &tmr->tw_tmr, + &tmr->u.tw_tmr, timeout_func, cancel_func, tmr, @@ -1164,15 +1161,15 @@ schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs) ERTS_HLT_ASSERT(erts_smp_atomic32_read_nob(&tmr->head.refc) == 0); if (roflgs & ERTS_TMR_ROFLG_REG_NAME) { - ERTS_HLT_ASSERT(is_atom(tmr->receiver.name)); + ERTS_HLT_ASSERT(is_atom(tmr->head.receiver.name)); } else if (roflgs & ERTS_TMR_ROFLG_PROC) { - ERTS_HLT_ASSERT(tmr->receiver.proc); - erts_proc_dec_refc(tmr->receiver.proc); + ERTS_HLT_ASSERT(tmr->head.receiver.proc); + erts_proc_dec_refc(tmr->head.receiver.proc); } else if (roflgs & ERTS_TMR_ROFLG_PORT) { - ERTS_HLT_ASSERT(tmr->receiver.port); - erts_port_dec_refc(tmr->receiver.port); + ERTS_HLT_ASSERT(tmr->head.receiver.port); + erts_port_dec_refc(tmr->head.receiver.port); } if (!(roflgs & ERTS_TMR_ROFLG_BIF_TMR)) @@ -1251,14 +1248,14 @@ btm_clear_magic_binary(ErtsBifTimer *tmr) #endif /* ERTS_MAGIC_REF_BIF_TIMERS */ static ERTS_INLINE void -bif_timer_timeout(ErtsHLTimerService *srv, ErtsBifTimer *tmr, - Uint32 roflgs, int is_hlt) +bif_timer_timeout(ErtsHLTimerService *srv, + ErtsBifTimer *tmr, + Uint32 roflgs) { erts_aint32_t state; ERTS_HLT_ASSERT(tmr->type.head.roflgs == roflgs); ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR); - ERTS_HLT_ASSERT(!!is_hlt == !!(roflgs & ERTS_TMR_ROFLG_HLT)); state = erts_smp_atomic32_cmpxchg_acqb(&tmr->btm.state, ERTS_TMR_STATE_TIMED_OUT, @@ -1269,40 +1266,32 @@ bif_timer_timeout(ErtsHLTimerService *srv, ErtsBifTimer *tmr, if (state == ERTS_TMR_STATE_ACTIVE) { Process *proc; - int dec_refc = 0; - Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME); #ifdef ERTS_MAGIC_REF_BIF_TIMERS btm_clear_magic_binary(tmr); #endif - if (is_reg_name) { + if (roflgs & ERTS_TMR_ROFLG_REG_NAME) { Eterm term; - if (is_hlt) - term = tmr->type.hlt.receiver.name; - else - term = tmr->type.twt.receiver.name; + term = tmr->type.head.receiver.name; ERTS_HLT_ASSERT(is_atom(term)); term = erts_whereis_name_to_id(NULL, term); proc = erts_proc_lookup(term); } else { ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_PROC); - if (is_hlt) - proc = tmr->type.hlt.receiver.proc; - else - proc = tmr->type.twt.receiver.proc; + proc = tmr->type.head.receiver.proc; ERTS_HLT_ASSERT(proc); } if (proc) { if (!ERTS_PROC_IS_EXITING(proc)) { - ErtsMessage *mp; - erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM); - mp = erts_alloc_message(0, NULL); + int dec_refc = 0; + ErtsMessage *mp = erts_alloc_message(0, NULL); mp->data.heap_frag = tmr->btm.bp; - erts_queue_message(proc, ERTS_PROC_LOCK_BTM, mp, - tmr->btm.message, am_clock_service); tmr->btm.bp = NULL; + erts_queue_message(proc, 0, mp, tmr->btm.message, + am_clock_service); + erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM); #ifdef ERTS_MAGIC_REF_BIF_TIMERS if (tmr->btm.proc_list.next) { proc_btm_list_delete(&proc->bif_timers, tmr); @@ -1316,9 +1305,9 @@ bif_timer_timeout(ErtsHLTimerService *srv, ErtsBifTimer *tmr, } #endif erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); + if (dec_refc) + timer_pre_dec_refc((ErtsTimer *) tmr); } - if (dec_refc) - timer_pre_dec_refc((ErtsTimer *) tmr); } if (tmr->btm.bp) free_message_buffer(tmr->btm.bp); @@ -1344,7 +1333,7 @@ tw_bif_timer_timeout(void *vbtmp) ErtsHLTimerService *srv = esdp->timer_service; #endif ErtsBifTimer *btmp = (ErtsBifTimer *) vbtmp; - bif_timer_timeout(srv, btmp, btmp->type.head.roflgs, 0); + bif_timer_timeout(srv, btmp, btmp->type.head.roflgs); tw_timer_dec_refc(&btmp->type.twt); } @@ -1386,7 +1375,7 @@ create_hl_timer(ErtsSchedulerData *esdp, ERTS_HLT_ASSERT(is_internal_pid(rcvr)); erts_proc_inc_refc((Process *) rcvrp); - tmr->receiver.proc = (Process *) rcvrp; + tmr->head.receiver.proc = (Process *) rcvrp; roflgs |= ERTS_TMR_ROFLG_PROC; refc = 2; break; @@ -1394,14 +1383,14 @@ create_hl_timer(ErtsSchedulerData *esdp, case ERTS_TMR_PORT: ERTS_HLT_ASSERT(is_internal_port(rcvr)); erts_port_inc_refc((Port *) rcvrp); - tmr->receiver.port = (Port *) rcvrp; + tmr->head.receiver.port = (Port *) rcvrp; roflgs |= ERTS_TMR_ROFLG_PORT; refc = 2; break; case ERTS_TMR_CALLBACK: roflgs |= ERTS_TMR_ROFLG_CALLBACK; - tmr->receiver.callback = callback; + tmr->head.receiver.callback = callback; tmr->head.u.arg = arg; refc = 1; break; @@ -1431,13 +1420,13 @@ create_hl_timer(ErtsSchedulerData *esdp, roflgs |= ERTS_TMR_ROFLG_BIF_TMR; if (is_internal_pid(rcvr)) { roflgs |= ERTS_TMR_ROFLG_PROC; - tmr->receiver.proc = (Process *) rcvrp; + tmr->head.receiver.proc = (Process *) rcvrp; refc = 2; } else { ERTS_HLT_ASSERT(is_atom(rcvr)); roflgs |= ERTS_TMR_ROFLG_REG_NAME; - tmr->receiver.name = rcvr; + tmr->head.receiver.name = rcvr; refc = 1; } @@ -1486,14 +1475,14 @@ create_hl_timer(ErtsSchedulerData *esdp, static ERTS_INLINE void hlt_proc_timeout(ErtsHLTimer *tmr) { - if (proc_timeout_common(tmr->receiver.proc, (void *) tmr)) + if (proc_timeout_common(tmr->head.receiver.proc, (void *) tmr)) hl_timer_dec_refc(tmr, tmr->head.roflgs); } static ERTS_INLINE void hlt_port_timeout(ErtsHLTimer *tmr) { - if (port_timeout_common(tmr->receiver.port, (void *) tmr)) + if (port_timeout_common(tmr->head.receiver.port, (void *) tmr)) hl_timer_dec_refc(tmr, tmr->head.roflgs); } @@ -1508,14 +1497,14 @@ static void hlt_timeout(ErtsHLTimer *tmr, void *vsrv) ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_HLT); if (roflgs & ERTS_TMR_ROFLG_BIF_TMR) - bif_timer_timeout(srv, (ErtsBifTimer *) tmr, roflgs, 1); + bif_timer_timeout(srv, (ErtsBifTimer *) tmr, roflgs); else if (roflgs & ERTS_TMR_ROFLG_PROC) hlt_proc_timeout(tmr); else if (roflgs & ERTS_TMR_ROFLG_PORT) hlt_port_timeout(tmr); else { ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_CALLBACK); - (*tmr->receiver.callback)(tmr->head.u.arg); + (*tmr->head.receiver.callback)(tmr->head.u.arg); } tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; @@ -2022,10 +2011,7 @@ setup_bif_timer(Process *c_p, int twheel, ErtsMonotonicTime timeout_pos, proc_btm_rbt_insert(&proc->bif_timers, tmr); #endif erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); - if (twheel) - tmr->type.twt.receiver.proc = proc; - else - tmr->type.hlt.receiver.proc = proc; + tmr->type.head.receiver.proc = proc; } } @@ -2064,9 +2050,7 @@ cancel_bif_timer(ErtsBifTimer *tmr) if (roflgs & ERTS_TMR_ROFLG_PROC) { Process *proc; - proc = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT) - ? tmr->type.hlt.receiver.proc - : tmr->type.twt.receiver.proc); + proc = tmr->type.head.receiver.proc; ERTS_HLT_ASSERT(!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME)); erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM); @@ -2108,7 +2092,7 @@ access_btm(ErtsBifTimer *tmr, Uint32 sid, ErtsSchedulerData *esdp, int cancel) is_hlt = !!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT); timeout = (is_hlt ? tmr->type.hlt.timeout - : tmr->type.twt.tw_tmr.timeout_pos); + : erts_tweel_read_timeout(&tmr->type.twt.u.tw_tmr)); if (!cancel) { erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->btm.state); @@ -3147,7 +3131,7 @@ erts_read_port_timer(Port *c_prt) if (tmr->head.roflgs & ERTS_TMR_ROFLG_HLT) timeout_pos = tmr->hlt.timeout; else - timeout_pos = tmr->twt.tw_tmr.timeout_pos; + timeout_pos = erts_tweel_read_timeout(&tmr->twt.u.tw_tmr); return get_time_left(NULL, timeout_pos); } @@ -3179,10 +3163,6 @@ btm_print(ErtsBifTimer *tmr, void *vbtmp, ErtsMonotonicTime tpos, int is_hlt) left = 0; else left = ERTS_CLKTCKS_TO_MSEC(tmr->type.hlt.timeout - btmp->now); - - receiver = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) - ? tmr->type.hlt.receiver.name - : tmr->type.hlt.receiver.proc->common.id); } else { ERTS_HLT_ASSERT(!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT)); @@ -3190,12 +3170,12 @@ btm_print(ErtsBifTimer *tmr, void *vbtmp, ErtsMonotonicTime tpos, int is_hlt) left = 0; else left = ERTS_CLKTCKS_TO_MSEC(tpos - btmp->now); - - receiver = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) - ? tmr->type.twt.receiver.name - : tmr->type.twt.receiver.proc->common.id); } + receiver = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) + ? tmr->type.head.receiver.name + : tmr->type.head.receiver.proc->common.id); + erts_print(btmp->to, btmp->to_arg, "=timer:%T\n" "Message: %T\n" @@ -3229,7 +3209,7 @@ btm_tree_print(ErtsBifTimer *tmr, void *vbtmp) if (is_hlt) tpos = 0; else - tpos = tmr->type.twt.tw_tmr.timeout_pos; + tpos = erts_tweel_read_timeout(&tmr->type.twt.u.tw_tmr); btm_print(tmr, vbtmp, tpos, is_hlt); } @@ -3273,7 +3253,7 @@ typedef struct { } ErtsBTMForeachDebug; static void -debug_btm_foreach(ErtsBifTimer *tmr, void *vbtmfd, int is_hlt) +debug_btm_foreach(ErtsBifTimer *tmr, void *vbtmfd) { #ifdef ERTS_MAGIC_REF_BIF_TIMERS if (!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) @@ -3281,15 +3261,9 @@ debug_btm_foreach(ErtsBifTimer *tmr, void *vbtmfd, int is_hlt) #endif if (erts_smp_atomic32_read_nob(&tmr->btm.state) == ERTS_TMR_STATE_ACTIVE) { ErtsBTMForeachDebug *btmfd = (ErtsBTMForeachDebug *) vbtmfd; - Eterm id; - if (is_hlt) - id = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) - ? tmr->type.hlt.receiver.name - : tmr->type.hlt.receiver.proc->common.id); - else - id = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) - ? tmr->type.twt.receiver.name - : tmr->type.twt.receiver.proc->common.id); + Eterm id = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) + ? tmr->type.head.receiver.name + : tmr->type.head.receiver.proc->common.id); (*btmfd->func)(id, tmr->btm.message, tmr->btm.bp, btmfd->arg); } } @@ -3299,22 +3273,13 @@ debug_btm_foreach(ErtsBifTimer *tmr, void *vbtmfd, int is_hlt) static void hlt_debug_btm_foreach(ErtsHLTimer *tmr, void *vbtmfd) { - debug_btm_foreach((ErtsBifTimer *) tmr, vbtmfd, 1); + debug_btm_foreach((ErtsBifTimer *) tmr, vbtmfd); } static void twt_debug_btm_foreach(void *vbtmfd, ErtsMonotonicTime tpos, void *vtwtp) { - debug_btm_foreach((ErtsBifTimer *) vtwtp, vbtmfd, 0); -} - -#else - -static void -debug_btm_tree_foreach(ErtsBifTimer *tmr, void *vbtmfd) -{ - debug_btm_foreach(tmr, vbtmfd, !!(tmr->type.head.roflgs - & ERTS_TMR_ROFLG_HLT)); + debug_btm_foreach((ErtsBifTimer *) vtwtp, vbtmfd); } #endif @@ -3349,7 +3314,7 @@ erts_debug_bif_timer_foreach(void (*func)(Eterm, (void *) &btmfd); #else btm_rbt_foreach(srv->btm_tree, - debug_btm_tree_foreach, + debug_btm_foreach, (void *) &btmfd); #endif } @@ -3370,7 +3335,7 @@ debug_callback_timer_foreach_list(ErtsHLTimer *tmr, void *vdfct) = (ErtsDebugForeachCallbackTimer *) vdfct; if ((tmr->head.roflgs & ERTS_TMR_ROFLG_CALLBACK) - && (tmr->receiver.callback == dfct->tclbk)) + && (tmr->head.receiver.callback == dfct->tclbk)) (*dfct->func)(dfct->arg, tmr->timeout, tmr->head.u.arg); @@ -3388,7 +3353,7 @@ debug_callback_timer_foreach(ErtsHLTimer *tmr, void *vdfct) vdfct); if ((tmr->head.roflgs & ERTS_TMR_ROFLG_CALLBACK) - && (tmr->receiver.callback == dfct->tclbk)) + && (tmr->head.receiver.callback == dfct->tclbk)) (*dfct->func)(dfct->arg, tmr->timeout, tmr->head.u.arg); @@ -3403,7 +3368,7 @@ debug_tw_callback_timer(void *vdfct, ErtsDebugForeachCallbackTimer *dfct = (ErtsDebugForeachCallbackTimer *) vdfct; - if (twtp->receiver.callback == dfct->tclbk) + if (twtp->head.receiver.callback == dfct->tclbk) (*dfct->func)(dfct->arg, timeout_pos, twtp->head.u.arg); diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index a1c4220633..2fb4b754d7 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -419,17 +419,12 @@ void erts_sched_init_time_sup(ErtsSchedulerData *esdp); ** Timer entry: */ typedef struct erl_timer { + ErtsMonotonicTime timeout_pos; /* Timeout in absolute clock ticks */ struct erl_timer* next; /* next entry tiw slot or chain */ struct erl_timer* prev; /* prev entry tiw slot or chain */ - union { - struct { - void (*timeout)(void*); /* called when timeout */ - void (*cancel)(void*); /* called when cancel (may be NULL) */ - void* arg; /* argument to timeout/cancel procs */ - } func; - ErtsThrPrgrLaterOp cleanup; - } u; - ErtsMonotonicTime timeout_pos; /* Timeout in absolute clock ticks */ + void (*timeout)(void*); /* called when timeout */ + void (*cancel)(void*); /* called when cancel (may be NULL) */ + void* arg; /* argument to timeout/cancel procs */ int slot; } ErtsTWheelTimer; @@ -447,6 +442,7 @@ ErtsMonotonicTime erts_check_next_timeout_time(ErtsSchedulerData *); ERTS_GLB_INLINE void erts_twheel_init_timer(ErtsTWheelTimer *p); ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef); +ERTS_GLB_INLINE ErtsMonotonicTime erts_tweel_read_timeout(ErtsTWheelTimer *twt); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -460,6 +456,12 @@ ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef nxt_ return *((ErtsMonotonicTime *) nxt_tmo_ref); } +ERTS_GLB_INLINE ErtsMonotonicTime +erts_tweel_read_timeout(ErtsTWheelTimer *twt) +{ + return twt->timeout_pos; +} + #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ void diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 6f15082130..fdf74498c9 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -330,8 +330,8 @@ timeout_timer(ErtsTWheelTimer *p) ErlTimeoutProc timeout; void *arg; p->slot = ERTS_TWHEEL_SLOT_INACTIVE; - timeout = p->u.func.timeout; - arg = p->u.func.arg; + timeout = p->timeout; + arg = p->arg; (*timeout)(arg); ASSERT_NO_LOCKED_LOCKS; } @@ -542,9 +542,9 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp) tiw->next_timeout_time = mtime + ERTS_MONOTONIC_DAY; tiw->sentinel.next = &tiw->sentinel; tiw->sentinel.prev = &tiw->sentinel; - tiw->sentinel.u.func.timeout = NULL; - tiw->sentinel.u.func.cancel = NULL; - tiw->sentinel.u.func.arg = NULL; + tiw->sentinel.timeout = NULL; + tiw->sentinel.cancel = NULL; + tiw->sentinel.arg = NULL; return tiw; } @@ -583,9 +583,9 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw, ErtsMonotonicTime timeout_time; ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); - p->u.func.timeout = timeout; - p->u.func.cancel = cancel; - p->u.func.arg = arg; + p->timeout = timeout; + p->cancel = cancel; + p->arg = arg; ERTS_TW_ASSERT(p->slot == ERTS_TWHEEL_SLOT_INACTIVE); @@ -636,8 +636,8 @@ erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) void *arg; ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); remove_timer(tiw, p); - cancel = p->u.func.cancel; - arg = p->u.func.arg; + cancel = p->cancel; + arg = p->arg; if (cancel) (*cancel)(arg); ERTS_MSACC_POP_STATE_M_X(); @@ -657,22 +657,22 @@ erts_twheel_debug_foreach(ErtsTimerWheel *tiw, tmr = tiw->sentinel.next; while (tmr != &tiw->sentinel) { - if (tmr->u.func.timeout == tclbk) - (*func)(arg, tmr->timeout_pos, tmr->u.func.arg); + if (tmr->timeout == tclbk) + (*func)(arg, tmr->timeout_pos, tmr->arg); tmr = tmr->next; } for (tmr = tiw->at_once.head; tmr; tmr = tmr->next) { - if (tmr->u.func.timeout == tclbk) - (*func)(arg, tmr->timeout_pos, tmr->u.func.arg); + if (tmr->timeout == tclbk) + (*func)(arg, tmr->timeout_pos, tmr->arg); } for (ix = 0; ix < ERTS_TIW_SIZE; ix++) { tmr = tiw->w[ix]; if (tmr) { do { - if (tmr->u.func.timeout == tclbk) - (*func)(arg, tmr->timeout_pos, tmr->u.func.arg); + if (tmr->timeout == tclbk) + (*func)(arg, tmr->timeout_pos, tmr->arg); tmr = tmr->next; } while (tmr != tiw->w[ix]); } -- cgit v1.2.3 From d37df80764c9c2e517d7c9a90291d75975a5e9bb Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 3 Feb 2017 16:08:12 +0100 Subject: Remove unnecessary cancel callback from timer-wheel timers --- erts/emulator/beam/erl_hl_timer.c | 16 +--------------- erts/emulator/beam/erl_process.c | 1 - erts/emulator/beam/erl_time.h | 7 ++----- erts/emulator/beam/erl_time_sup.c | 3 --- erts/emulator/beam/time.c | 11 +---------- 5 files changed, 4 insertions(+), 34 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index 8e47fdd674..13d6136672 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -995,18 +995,13 @@ tw_port_timeout(void *vtwtp) tw_timer_dec_refc(twtp); } -static void -tw_ptimer_cancel(void *vtwtp) -{ - tw_timer_dec_refc((ErtsTWTimer *) vtwtp); -} - static void cancel_tw_timer(ErtsSchedulerData *esdp, ErtsTWTimer *tmr) { ERTS_HLT_ASSERT((tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK) == (Uint32) esdp->no); erts_twheel_cancel_timer(esdp->timer_wheel, &tmr->u.tw_tmr); + tw_timer_dec_refc(tmr); } static void @@ -1034,7 +1029,6 @@ create_tw_timer(ErtsSchedulerData *esdp, { ErtsTWTimer *tmr; void (*timeout_func)(void *); - void (*cancel_func)(void *); erts_aint32_t refc; if (type != ERTS_TMR_BIF) { @@ -1068,7 +1062,6 @@ create_tw_timer(ErtsSchedulerData *esdp, tmr->head.receiver.proc = (Process *) rcvrp; tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC; timeout_func = tw_proc_timeout; - cancel_func = tw_ptimer_cancel; erts_proc_inc_refc((Process *) rcvrp); refc = 2; break; @@ -1077,7 +1070,6 @@ create_tw_timer(ErtsSchedulerData *esdp, tmr->head.receiver.port = (Port *) rcvrp; tmr->head.roflgs |= ERTS_TMR_ROFLG_PORT; timeout_func = tw_port_timeout; - cancel_func = tw_ptimer_cancel; erts_port_inc_refc((Port *) rcvrp); refc = 2; break; @@ -1088,14 +1080,12 @@ create_tw_timer(ErtsSchedulerData *esdp, tmr->head.roflgs |= ERTS_TMR_ROFLG_CALLBACK; timeout_func = tw_callback_timeout; - cancel_func = NULL; refc = 1; break; case ERTS_TMR_BIF: timeout_func = tw_bif_timer_timeout; - cancel_func = NULL; if (is_internal_pid(rcvr)) { tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC; tmr->head.receiver.proc = (Process *) rcvrp; @@ -1129,7 +1119,6 @@ create_tw_timer(ErtsSchedulerData *esdp, erts_twheel_set_timer(esdp->timer_wheel, &tmr->u.tw_tmr, timeout_func, - cancel_func, tmr, timeout_pos); @@ -1452,7 +1441,6 @@ create_hl_timer(ErtsSchedulerData *esdp, erts_twheel_set_timer(esdp->timer_wheel, &srv->service_timer, hlt_service_timeout, - NULL, (void *) esdp, tmr->timeout); srv->next_timeout = tmr; @@ -1574,7 +1562,6 @@ hlt_service_timeout(void *vesdp) erts_twheel_set_timer(esdp->timer_wheel, &srv->service_timer, hlt_service_timeout, - NULL, vesdp, tmr->timeout); } @@ -1631,7 +1618,6 @@ hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr) erts_twheel_set_timer(esdp->timer_wheel, &srv->service_timer, hlt_service_timeout, - NULL, (void *) esdp, smlst->timeout); } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 5baf533acd..267ca7e8a6 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -2815,7 +2815,6 @@ start_aux_work_timer(ErtsSchedulerData *esdp) erts_twheel_set_timer(esdp->timer_wheel, &aux_work_tmo->timer.data, aux_work_timeout, - NULL, (void *) esdp, tmo); } diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 2fb4b754d7..00b4a84b4a 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -420,21 +420,18 @@ void erts_sched_init_time_sup(ErtsSchedulerData *esdp); */ typedef struct erl_timer { ErtsMonotonicTime timeout_pos; /* Timeout in absolute clock ticks */ - struct erl_timer* next; /* next entry tiw slot or chain */ + struct erl_timer* next; /* next entry tiw slot or chain */ struct erl_timer* prev; /* prev entry tiw slot or chain */ void (*timeout)(void*); /* called when timeout */ - void (*cancel)(void*); /* called when cancel (may be NULL) */ void* arg; /* argument to timeout/cancel procs */ int slot; } ErtsTWheelTimer; typedef void (*ErlTimeoutProc)(void*); -typedef void (*ErlCancelProc)(void*); void erts_twheel_set_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p, ErlTimeoutProc timeout, - ErlCancelProc cancel, void *arg, - ErtsMonotonicTime timeout_pos); + void *arg, ErtsMonotonicTime timeout_pos); void erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p); ErtsTimerWheel *erts_create_timer_wheel(ErtsSchedulerData *esdp); diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index cf9d3adc86..c69fec3c80 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -678,7 +678,6 @@ check_time_correction(void *vesdp) erts_twheel_set_timer(esdp->timer_wheel, &time_sup.inf.c.parmon.timer, check_time_correction, - NULL, (void *) esdp, timeout_pos); } @@ -729,7 +728,6 @@ check_time_offset(void *vesdp) erts_twheel_set_timer(esdp->timer_wheel, &time_sup.inf.c.parmon.timer, check_time_offset, - NULL, vesdp, timeout_pos); } @@ -836,7 +834,6 @@ late_init_time_correction(ErtsSchedulerData *esdp) erts_twheel_set_timer(esdp->timer_wheel, &time_sup.inf.c.parmon.timer, check_func, - NULL, (quick_init_drift_adj ? NULL : esdp), diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index fdf74498c9..53896be19e 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -543,7 +543,6 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp) tiw->sentinel.next = &tiw->sentinel; tiw->sentinel.prev = &tiw->sentinel; tiw->sentinel.timeout = NULL; - tiw->sentinel.cancel = NULL; tiw->sentinel.arg = NULL; return tiw; } @@ -577,14 +576,12 @@ erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode) void erts_twheel_set_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p, ErlTimeoutProc timeout, - ErlCancelProc cancel, void *arg, - ErtsMonotonicTime timeout_pos) + void *arg, ErtsMonotonicTime timeout_pos) { ErtsMonotonicTime timeout_time; ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); p->timeout = timeout; - p->cancel = cancel; p->arg = arg; ERTS_TW_ASSERT(p->slot == ERTS_TWHEEL_SLOT_INACTIVE); @@ -632,14 +629,8 @@ void erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) { if (p->slot != ERTS_TWHEEL_SLOT_INACTIVE) { - ErlCancelProc cancel; - void *arg; ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); remove_timer(tiw, p); - cancel = p->cancel; - arg = p->arg; - if (cancel) - (*cancel)(arg); ERTS_MSACC_POP_STATE_M_X(); } } -- cgit v1.2.3 From 0981bfd7afee64c3c1f48db134d81e57c30e31c7 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 6 Feb 2017 20:39:02 +0100 Subject: Timer wheel divided into a "soon wheel" and a "later wheel" The old single wheel implementation handled about 65 seconds. The new dual wheel implementation handles more than 37 hours. The dual wheels have also been shrunk compared to the single wheel, so the total memory consumption for timer wheels have been cut in half. --- erts/emulator/beam/break.c | 2 +- erts/emulator/beam/erl_time.h | 50 ++- erts/emulator/beam/time.c | 810 ++++++++++++++++++++++++++++++++---------- 3 files changed, 671 insertions(+), 191 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 0b40d70cb7..538ab0d947 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -590,7 +590,7 @@ do_break(void) #endif #ifdef DEBUG case 't': - erts_p_slpq(); + /* erts_p_slpq(); */ return; case 'b': bin_check(); diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 00b4a84b4a..699e69e991 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -21,19 +21,52 @@ #ifndef ERL_TIME_H__ #define ERL_TIME_H__ -/* timer wheel size NEED to be a power of 2 */ -#ifdef SMALL_MEMORY -#define ERTS_TIW_SIZE (1 << 13) -#else -#define ERTS_TIW_SIZE (1 << 16) +#if 0 +# define ERTS_TW_DEBUG +#endif +#if defined(DEBUG) && !defined(ERTS_TW_DEBUG) +# define ERTS_TW_DEBUG #endif -#if defined(DEBUG) || 0 +#if defined(ERTS_TW_DEBUG) #define ERTS_TIME_ASSERT(B) ERTS_ASSERT(B) #else #define ERTS_TIME_ASSERT(B) ((void) 1) #endif +#ifdef ERTS_TW_DEBUG +/* + * Soon wheel will handle about 1 seconds + * Later wheel will handle about 8 minutes + */ +# define ERTS_TW_SOON_WHEEL_BITS 10 +# define ERTS_TW_LATER_WHEEL_BITS 10 +#else +# ifdef SMALL_MEMORY +/* + * Soon wheel will handle about 4 seconds + * Later wheel will handle about 2 hours and 19 minutes + */ +# define ERTS_TW_SOON_WHEEL_BITS 12 +# define ERTS_TW_LATER_WHEEL_BITS 12 +# else +/* + * Soon wheel will handle about 16 seconds + * Later wheel will handle about 37 hours and 16 minutes + */ +# define ERTS_TW_SOON_WHEEL_BITS 14 +# define ERTS_TW_LATER_WHEEL_BITS 14 +# endif +#endif + +/* + * Number of slots in each timer wheel... + * + * These *need* to be a power of 2 + */ +#define ERTS_TW_SOON_WHEEL_SIZE (1 << ERTS_TW_SOON_WHEEL_BITS) +#define ERTS_TW_LATER_WHEEL_SIZE (1 << ERTS_TW_LATER_WHEEL_BITS) + typedef enum { ERTS_NO_TIME_WARP_MODE, ERTS_SINGLE_TIME_WARP_MODE, @@ -103,7 +136,10 @@ Eterm erts_system_time_source(struct process*c_p); #define ERTS_CLKTCK_RESOLUTION (erts_time_sup__.r.o.clktck_resolution) #endif -#define ERTS_TIMER_WHEEL_MSEC (ERTS_TIW_SIZE/(ERTS_CLKTCK_RESOLUTION/1000)) +#define ERTS_TW_SOON_WHEEL_MSEC (ERTS_TW_SOON_WHEEL_SIZE/(ERTS_CLKTCK_RESOLUTION/1000)) +#define ERTS_TW_LATER_WHEEL_MSEC (ERTS_TW_LATER_WHEEL_SIZE*ERTS_TW_SOON_WHEEL_MSEC/2) + +#define ERTS_TIMER_WHEEL_MSEC ERTS_TW_LATER_WHEEL_MSEC struct erts_time_sup_read_only__ { ErtsMonotonicTime monotonic_time_unit; diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 53896be19e..7b5ff7b992 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -17,57 +17,124 @@ * * %CopyrightEnd% */ + /* - * TIMING WHEEL + * TIMER WHEEL + * + * + * The time scale used for timers is Erlang monotonic time. The + * time unit used is ERTS specific clock ticks. A clock tick is + * currently defined to 1 millisecond. That is, the resolution of + * timers triggered by the runtime system is 1 millisecond. * - * Timeouts kept in an wheel. A timeout is measured relative to the - * current slot (tiw_pos) in the wheel, and inserted at slot - * (tiw_pos + timeout) % TIW_SIZE. Each timeout also has a count - * equal to timeout/TIW_SIZE, which is needed since the time axis - * is wrapped arount the wheel. + * When a timer is set, it is determined at what Erlang monotonic + * time, in clock ticks, it should be triggered. * - * Several slots may be processed in one operation. If the number of - * slots is greater that the wheel size, the wheel is only traversed - * once, + * The 'pos' field of the wheel corresponds to current time of + * the wheel. That is, it corresponds to Erlang monotonic time in + * clock tick time unit. The 'pos' field of the wheel is + * monotonically increased when erts_bump_timers() is called. All + * timers in the wheel that have a time less than or equal to + * 'pos' are triggered by the bump operation. The bump operation + * may however be spread over multiple calls to erts_bump_timers() + * if there are a lots of timers to trigger. + * + * Each scheduler thread maintains its own timer wheel. The timer + * wheel of a scheduler, however, actually consists of two wheels. + * A soon wheel and a later wheel. + * + * + * -- The Soon Wheel -- + * + * The soon wheel contain timers that should be triggered soon. + * That is, they are soon to be triggered. Each slot in the soon + * wheel is 1 clock tick wide. The number of slots in the soon + * wheel is currently 2¹⁴. That is, it contains timers in the + * range ('pos', 'pos' + 2¹⁴] which corresponds to a bit more + * than 16 seconds. + * + * When the bump operation is started, 'pos' is moved forward to a + * position that corresponds to current Erlang monotonic time. Then + * all timers that are in the range (old 'pos', new 'pos'] are + * triggered. During a bump operation, the soon wheel may contain + * timers in the two, possibly overlapping, ranges (old 'pos', + * old 'pos' + 2¹⁴], and (new 'pos', new 'pos' + 2¹⁴]. This may + * occur even if the bump operation doesn't yield, due to timeout + * callbacks inserting new timers. + * + * + * -- The Later Wheel -- + * + * The later wheel contain timers that are further away from 'pos' + * than the width of the soon timer wheel. That is, currently + * timers further away from 'pos' than 2¹⁴ clock ticks. The width + * of each slot in the later wheel is half the width of the soon + * wheel. That is, each slot is currently 2¹³ clock ticks wide + * which corresponds to about 8 seconds. If three timers of the + * times 'pos' + 17000, 'pos' + 18000, and 'pos' + 19000 are + * inserted, they will all end up in the same slot in the later + * wheel. + * + * The number of slots in the later wheel is currently the same as + * in the soon wheel, i.e. 2¹⁴. That is, one revolution of the later + * wheel currently corresponds to 2¹⁴×2¹³ clock ticks which is + * almost 37 ½ hour. Timers even further away than that are put in + * the later slot identified by their time modulo the size of the later + * wheel. Such timers are however very uncommon. Most timers used + * by the runtime system will utilize the high level timer API. + * The high level timer implementation will not insert timers + * further away then one revolution into the later wheel. It will + * instead keep such timers in a tree of very long timers. The + * high level timer implementation utilize one timer wheel timer + * for the management of this tree of timers. This timer is set to + * the closest timeout in the tree. This timer may however be + * further away than one revolution in the later wheel. + * + * The 'later.pos' field identifies next position in the later wheel. + * 'later.pos' is always increased by the width of a later wheel slot. + * That is, currently 2¹³ clock ticks. When 'pos' is moved (during + * a bump operation) closer to 'later.pos' than the width of a later + * wheel slot, i.e. currently when 'pos' + 2¹³ ≥ 'later.pos', we + * inspect the slot identified by 'later.pos' and then move 'later.pos' + * forward. When inspecting the later slot we move all timers in the + * slot, that are in the soon wheel range, from the later wheel to + * the soon wheel. Timers one or more revolutions of the later wheel + * away are kept in the slot. + * + * During normal operation, timers originally located in the later + * wheel will currently be moved into the soon wheel about 8 to + * 16 seconds before they should be triggered. During extremely + * heavy load, the scheduler might however be heavily delayed, so + * the code must be prepared for situations where time for + * triggering the timer has passed when we inspect the later wheel + * slot, and then trigger the timer immediately. We must also be + * prepared to inspect multiple later wheel slots at once due to the + * delay. + * + * + * -- Slot Management -- + * + * All timers of a slot are placed in a circular double linked + * list. This makes insertion and removal of a timer O(1). + * + * While bumping timers in a slot, we move the circular list + * away from the slot, and refer to it from the 'sentinel' + * field. The list will stay there until we are done with it + * even if the bump operation should yield. The cancel operation + * can remove the timer from this position as well as from the + * slot position by just removing it from the circular double + * linked list that it is in. + * + * -- At Once List -- + * + * If a timer is set that has a time earlier or equal to 'pos', + * it is not inserted into the wheel. It is instead inserted, + * into a list referred to by the 'at_once' field. When the + * bump operation is performed thise timers will be triggered + * at once. * - * The following example shows a time axis where there is one timeout - * at each "tick", and where 1, 2, 3 ... wheel slots are released in - * one operation. The notation "pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE >= (TIW)->later.pos) + +static int bump_later_wheel(ErtsTimerWheel *tiw, int *yield_count_p); + +static ERTS_INLINE int +soon_slot(ErtsMonotonicTime soon_pos) +{ + ErtsMonotonicTime slot = soon_pos; + slot &= ERTS_TW_SOON_WHEEL_MASK; + + ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT <= slot); + ERTS_TW_ASSERT(slot < ERTS_TW_SOON_WHEEL_END_SLOT); + + return (int) slot; +} + +static ERTS_INLINE int +later_slot(ErtsMonotonicTime later_pos) +{ + ErtsMonotonicTime slot = later_pos; + slot >>= ERTS_TW_LATER_WHEEL_SHIFT; + slot &= ERTS_TW_LATER_WHEEL_MASK; + slot += ERTS_TW_LATER_WHEEL_FIRST_SLOT; + + ERTS_TW_ASSERT(ERTS_TW_LATER_WHEEL_FIRST_SLOT <= slot); + ERTS_TW_ASSERT(slot < ERTS_TW_LATER_WHEEL_END_SLOT); + + return (int) slot; +} + +#ifdef ERTS_TW_HARD_DEBUG +#define ERTS_HARD_DBG_CHK_WHEELS(TIW, CHK_MIN_TPOS) \ + hrd_dbg_check_wheels((TIW), (CHK_MIN_TPOS)) +static void hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos); +#else +#define ERTS_HARD_DBG_CHK_WHEELS(TIW, CHK_MIN_TPOS) +#endif + static ERTS_INLINE ErtsMonotonicTime find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw, @@ -143,11 +289,14 @@ find_next_timeout(ErtsSchedulerData *esdp, ErtsMonotonicTime curr_time, /* When !search_all */ ErtsMonotonicTime max_search_time) /* When !search_all */ { - int start_ix, tiw_pos_ix; - ErtsTWheelTimer *p; + int start_ix, tiw_pos_ix, pos_inc, wheel_start_ix, wheel_end_ix; int true_min_timeout = 0; ErtsMonotonicTime min_timeout, min_timeout_pos, slot_timeout_pos; + ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); + + ERTS_TW_ASSERT(tiw->yield_slot == ERTS_TWHEEL_SLOT_INACTIVE); + if (tiw->nto == 0) { /* no timeouts in wheel */ if (!search_all) min_timeout_pos = tiw->pos; @@ -156,58 +305,115 @@ find_next_timeout(ErtsSchedulerData *esdp, tiw->pos = min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); } min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY); - goto found_next; + goto done; } - slot_timeout_pos = min_timeout_pos = tiw->pos; if (search_all) - min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY); + min_timeout_pos = tiw->pos + ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY); else - min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + max_search_time); + min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + max_search_time); + + if (!tiw->soon.nto) { + /* Select later wheel... */ + slot_timeout_pos = tiw->later.pos; + start_ix = tiw_pos_ix = later_slot(slot_timeout_pos); + pos_inc = ERTS_TW_LATER_WHEEL_SLOT_SIZE; + wheel_start_ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT; + wheel_end_ix = ERTS_TW_LATER_WHEEL_END_SLOT; + /* Pre-timeout for move from later to soon wheel... */ + slot_timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + } + else { + /* Select soon wheel... */ + + /* + * Besides inspecting the soon wheel we + * may also have to inspect two slots in the + * later wheel which potentially can trigger + * timeouts before timeouts in soon wheel... + */ + slot_timeout_pos = tiw->later.pos; + slot_timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + if (slot_timeout_pos < min_timeout_pos) { + int fslot = later_slot(tiw->later.pos); + if (tiw->w[fslot]) { + min_timeout_pos = slot_timeout_pos; + true_min_timeout = 1; + } + else { + slot_timeout_pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE; + if (slot_timeout_pos < min_timeout_pos) { + fslot++; + if (fslot == ERTS_TW_LATER_WHEEL_END_SLOT) + fslot = ERTS_TW_LATER_WHEEL_FIRST_SLOT; + if (tiw->w[fslot]) { + min_timeout_pos = slot_timeout_pos; + true_min_timeout = 1; + } + } + } + } - start_ix = tiw_pos_ix = (int) (tiw->pos & (ERTS_TIW_SIZE-1)); + slot_timeout_pos = tiw->pos; + start_ix = tiw_pos_ix = soon_slot(tiw->pos); + pos_inc = 1; + wheel_start_ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; + wheel_end_ix = ERTS_TW_SOON_WHEEL_END_SLOT; + } + /* + * Scan selected wheel... + */ do { - if (++slot_timeout_pos >= min_timeout_pos) - break; - - p = tiw->w[tiw_pos_ix]; - - if (p) { - ErtsTWheelTimer *end = p; - - do { - ErtsMonotonicTime timeout_pos; - timeout_pos = p->timeout_pos; - if (min_timeout_pos > timeout_pos) { - true_min_timeout = 1; - min_timeout_pos = timeout_pos; - if (min_timeout_pos <= slot_timeout_pos) - goto found_next; - } - p = p->next; - } while (p != end); - } - - tiw_pos_ix++; - if (tiw_pos_ix == ERTS_TIW_SIZE) - tiw_pos_ix = 0; + if (tiw->w[tiw_pos_ix]) { + min_timeout_pos = slot_timeout_pos; + true_min_timeout = 1; + break; + } + + slot_timeout_pos += pos_inc; + if (slot_timeout_pos >= min_timeout_pos) + break; + + tiw_pos_ix++; + if (tiw_pos_ix == wheel_end_ix) + tiw_pos_ix = wheel_start_ix; } while (start_ix != tiw_pos_ix); -found_next: +done: min_timeout = ERTS_CLKTCKS_TO_MONOTONIC(min_timeout_pos); tiw->next_timeout_time = min_timeout; tiw->true_next_timeout_time = true_min_timeout; + ERTS_HARD_DBG_CHK_WHEELS(tiw, 1); + return min_timeout; } +static ERTS_INLINE void +insert_timer_into_at_once_list(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) +{ + tiw->at_once.nto++; + p->next = NULL; + p->prev = tiw->at_once.tail; + if (tiw->at_once.tail) { + ERTS_TW_ASSERT(tiw->at_once.head); + tiw->at_once.tail->next = p; + } + else { + ERTS_TW_ASSERT(!tiw->at_once.head); + tiw->at_once.head = p; + } + tiw->at_once.tail = p; + p->slot = ERTS_TWHEEL_SLOT_AT_ONCE; +} + static ERTS_INLINE void insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p) { - ERTS_TW_ASSERT(slot >= 0); - ERTS_TW_ASSERT(slot < ERTS_TIW_SIZE); + ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT <= slot); + ERTS_TW_ASSERT(slot < ERTS_TW_LATER_WHEEL_END_SLOT); p->slot = slot; if (!tiw->w[slot]) { tiw->w[slot] = p; @@ -223,6 +429,10 @@ insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p) prev->next = p; next->prev = p; } + if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) { + ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); + tiw->soon.nto++; + } } static ERTS_INLINE void @@ -231,13 +441,13 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) int slot = p->slot; ERTS_TW_ASSERT(slot != ERTS_TWHEEL_SLOT_INACTIVE); - if (slot >= 0) { + if (slot >= ERTS_TW_SOON_WHEEL_FIRST_SLOT) { /* * Timer in wheel or in circular * list of timers currently beeing * triggered (referred by sentinel). */ - ERTS_TW_ASSERT(slot < ERTS_TIW_SIZE); + ERTS_TW_ASSERT(slot < ERTS_TW_LATER_WHEEL_END_SLOT); if (p->next == p) { ERTS_TW_ASSERT(tiw->w[slot] == p); @@ -249,6 +459,8 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) p->prev->next = p->next; p->next->prev = p->prev; } + if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) + tiw->soon.nto--; } else { /* Timer in "at once" queue... */ @@ -270,8 +482,6 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) } p->slot = ERTS_TWHEEL_SLOT_INACTIVE; - - tiw->nto--; } ErtsMonotonicTime @@ -299,13 +509,13 @@ debug_check_safe_to_skip_to(ErtsTimerWheel *tiw, ErtsMonotonicTime skip_to_pos) ErtsTWheelTimer *tmr; ErtsMonotonicTime tmp; - ix = (int) (tiw->pos & (ERTS_TIW_SIZE-1)); + ix = soon_slot(tiw->pos); tmp = skip_to_pos - tiw->pos; ERTS_TW_ASSERT(tmp >= 0); - if (tmp < (ErtsMonotonicTime) ERTS_TIW_SIZE) + if (tmp < (ErtsMonotonicTime) ERTS_TW_SOON_WHEEL_SIZE) slots = (int) tmp; else - slots = ERTS_TIW_SIZE; + slots = ERTS_TW_SOON_WHEEL_SIZE; while (slots > 0) { tmr = tiw->w[ix]; @@ -317,10 +527,41 @@ debug_check_safe_to_skip_to(ErtsTimerWheel *tiw, ErtsMonotonicTime skip_to_pos) } while (tmr != end); } ix++; - if (ix == ERTS_TIW_SIZE) - ix = 0; + if (ix == ERTS_TW_SOON_WHEEL_END_SLOT) + ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; slots--; } + + ix = later_slot(tiw->later.pos); + tmp = skip_to_pos; + tmp &= ERTS_TW_LATER_WHEEL_POS_MASK; + if (tmp > tiw->later.pos) { + tmp -= tiw->later.pos; + tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + ERTS_TW_ASSERT(tmp >= 0); + if (tmp < (ErtsMonotonicTime) ERTS_TW_LATER_WHEEL_SIZE) + slots = (int) tmp; + else + slots = ERTS_TW_LATER_WHEEL_SIZE; + + while (slots > 0) { + tmr = tiw->w[ix]; + if (tmr) { + ErtsMonotonicTime tpos = tmr->timeout_pos; + ErtsTWheelTimer *end = tmr; + do { + tpos &= ERTS_TW_LATER_WHEEL_POS_MASK; + tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + ERTS_TW_ASSERT(tpos > skip_to_pos); + tmr = tmr->next; + } while (tmr != end); + } + ix++; + if (ix == ERTS_TW_LATER_WHEEL_END_SLOT) + ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT; + slots--; + } + } } #endif @@ -339,8 +580,8 @@ timeout_timer(ErtsTWheelTimer *p) void erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) { - int tiw_pos_ix, slots, yielded_slot_restarted, yield_count; - ErtsMonotonicTime bump_to, tmp_slots, old_pos; + int tiw_pos_ix, yielded_slot_restarted, yield_count, slots; + ErtsMonotonicTime bump_to; ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); yield_count = ERTS_TWHEEL_BUMP_YIELD_LIMIT; @@ -350,13 +591,16 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) * where we left off when restarting after a yield. */ - if (tiw->yield_slot >= 0) { + if (tiw->yield_slot >= ERTS_TW_SOON_WHEEL_FIRST_SLOT) { yielded_slot_restarted = 1; + bump_to = tiw->pos; + if (tiw->yield_slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) + goto restart_yielded_later_slot; tiw_pos_ix = tiw->yield_slot; slots = tiw->yield_slots_left; - bump_to = tiw->pos; - old_pos = tiw->yield_start_pos; - goto restart_yielded_slot; + ASSERT(0 <= slots && slots <= ERTS_TW_SOON_WHEEL_SIZE); + tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; + goto restart_yielded_soon_slot; } do { @@ -368,8 +612,6 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) while (1) { ErtsTWheelTimer *p; - old_pos = tiw->pos; - if (tiw->nto == 0) { empty_wheel: ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, bump_to); @@ -383,12 +625,12 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) p = tiw->at_once.head; while (p) { - if (--yield_count <= 0) { + if (yield_count <= 0) { ERTS_TW_ASSERT(tiw->nto > 0); ERTS_TW_ASSERT(tiw->at_once.nto > 0); tiw->yield_slot = ERTS_TWHEEL_SLOT_AT_ONCE; tiw->true_next_timeout_time = 1; - tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(old_pos); + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to); ERTS_MSACC_POP_STATE_M_X(); return; } @@ -405,6 +647,8 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) timeout_timer(p); + yield_count -= ERTS_TW_COST_TIMEOUT; + p = tiw->at_once.head; } @@ -433,22 +677,34 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, skip_until_pos); tiw->pos = skip_until_pos; + + skip_until_pos++; + skip_until_pos &= ERTS_TW_LATER_WHEEL_POS_MASK; + if (tiw->later.pos < skip_until_pos) + tiw->later.pos = skip_until_pos; } } - tiw_pos_ix = (int) ((tiw->pos+1) & (ERTS_TIW_SIZE-1)); - tmp_slots = (bump_to - tiw->pos); - if (tmp_slots < (ErtsMonotonicTime) ERTS_TIW_SIZE) - slots = (int) tmp_slots; - else - slots = ERTS_TIW_SIZE; + { + ErtsMonotonicTime tmp_slots = bump_to - tiw->pos; + tmp_slots = (bump_to - tiw->pos); + if (tmp_slots < ERTS_TW_SOON_WHEEL_SIZE) + slots = (int) tmp_slots; + else + slots = ERTS_TW_SOON_WHEEL_SIZE; + } + tiw_pos_ix = soon_slot(tiw->pos+1); tiw->pos = bump_to; - while (slots > 0) { + /* Timeout timers in soon wheel */ + while (slots) { + + yield_count -= ERTS_TW_COST_SLOT; p = tiw->w[tiw_pos_ix]; if (p) { + /* timeout callback need tiw->pos to be up to date */ if (p->next == p) { ERTS_TW_ASSERT(tiw->sentinel.next == &tiw->sentinel); ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel); @@ -463,18 +719,21 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) while (1) { - if (p->timeout_pos > bump_to) { - /* Very unusual case... */ - ++yield_count; - insert_timer_into_slot(tiw, tiw_pos_ix, p); - } - else { - /* Normal case... */ - timeout_timer(p); - tiw->nto--; - } - - restart_yielded_slot: + ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT <= p->slot + && p->slot < ERTS_TW_SOON_WHEEL_END_SLOT); + tiw->soon.nto--; + if (p->timeout_pos <= bump_to) { + timeout_timer(p); + tiw->nto--; + yield_count -= ERTS_TW_COST_TIMEOUT; + } + else { + /* uncommon case */ + insert_timer_into_slot(tiw, tiw_pos_ix, p); + yield_count -= ERTS_TW_COST_SLOT_MOVE; + } + + restart_yielded_soon_slot: p = tiw->sentinel.next; if (p == &tiw->sentinel) { @@ -482,12 +741,11 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) break; } - if (--yield_count <= 0) { + if (yield_count <= 0) { tiw->true_next_timeout_time = 1; - tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(old_pos); + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to); tiw->yield_slot = tiw_pos_ix; tiw->yield_slots_left = slots; - tiw->yield_start_pos = old_pos; ERTS_MSACC_POP_STATE_M_X(); return; /* Yield! */ } @@ -496,16 +754,22 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) p->next->prev = &tiw->sentinel; } } - tiw_pos_ix++; - if (tiw_pos_ix == ERTS_TIW_SIZE) - tiw_pos_ix = 0; - slots--; + + tiw_pos_ix++; + if (tiw_pos_ix == ERTS_TW_SOON_WHEEL_END_SLOT) + tiw_pos_ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; + slots--; } + + if (ERTS_TW_BUMP_LATER_WHEEL(tiw)) { + restart_yielded_later_slot: + if (bump_later_wheel(tiw, &yield_count)) + return; /* Yield! */ + } } } while (yielded_slot_restarted); - tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; tiw->true_next_timeout_time = 0; tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY; @@ -514,6 +778,119 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) ERTS_MSACC_POP_STATE_M_X(); } +static int +bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) +{ + ErtsMonotonicTime cpos = tiw->pos; + ErtsMonotonicTime later_pos = tiw->later.pos; + int ycount = *ycount_p; + int slots, fslot; + + ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); + + if (tiw->yield_slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) { + fslot = tiw->yield_slot; + slots = tiw->yield_slots_left; + ASSERT(0 <= slots && slots <= ERTS_TW_LATER_WHEEL_SIZE); + tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; + goto restart_yielded_slot; + } + else { + ErtsMonotonicTime tmp_slots = cpos; + tmp_slots += ERTS_TW_LATER_WHEEL_SLOT_SIZE; + tmp_slots -= later_pos; + tmp_slots /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + if (tmp_slots < ERTS_TW_LATER_WHEEL_SIZE) + slots = (int) tmp_slots; + else + slots = ERTS_TW_LATER_WHEEL_SIZE; + } + + while (slots >= 0) { + ErtsTWheelTimer *p; + + fslot = later_slot(later_pos); + + ycount -= ERTS_TW_COST_SLOT; + + p = tiw->w[fslot]; + + if (p) { + + if (p->next == p) { + ERTS_TW_ASSERT(tiw->sentinel.next == &tiw->sentinel); + ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel); + } + else { + tiw->sentinel.next = p->next; + tiw->sentinel.prev = p->prev; + tiw->sentinel.next->prev = &tiw->sentinel; + tiw->sentinel.prev->next = &tiw->sentinel; + } + tiw->w[fslot] = NULL; + + while (1) { + ErtsMonotonicTime tpos = p->timeout_pos; + + ERTS_TW_ASSERT(tpos >= later_pos); + ERTS_TW_ASSERT(p->slot == fslot); + + if (tpos >= later_pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE) { + /* keep in later slot; very uncommon... */ + insert_timer_into_slot(tiw, fslot, p); + ycount -= ERTS_TW_COST_SLOT_MOVE; + } + else { + ERTS_TW_ASSERT(tpos < cpos + ERTS_TW_SOON_WHEEL_SIZE); + if (tpos > cpos) { + /* move into soon wheel */ + insert_timer_into_slot(tiw, soon_slot(tpos), p); + ycount -= ERTS_TW_COST_SLOT_MOVE; + } + else { + /* trigger at once */ + timeout_timer(p); + tiw->nto--; + ycount -= ERTS_TW_COST_TIMEOUT; + } + } + + restart_yielded_slot: + + p = tiw->sentinel.next; + if (p == &tiw->sentinel) { + ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel); + break; + } + + if (ycount < 0) { + tiw->later.pos = later_pos; + tiw->true_next_timeout_time = 1; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(cpos); + tiw->yield_slot = fslot; + tiw->yield_slots_left = slots; + *ycount_p = 0; + ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); + return 1; /* Yield! */ + } + + tiw->sentinel.next = p->next; + p->next->prev = &tiw->sentinel; + } + } + slots--; + later_pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE; + } + + ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); + + tiw->later.pos = later_pos; + + *ycount_p = ycount; + + return 0; +} + Uint erts_timer_wheel_memory_size(void) { @@ -524,11 +901,11 @@ ErtsTimerWheel * erts_create_timer_wheel(ErtsSchedulerData *esdp) { ErtsMonotonicTime mtime; - int i; + int i = ERTS_TW_SOON_WHEEL_FIRST_SLOT; ErtsTimerWheel *tiw; tiw = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_TIMER_WHEEL, sizeof(ErtsTimerWheel)); - for(i = 0; i < ERTS_TIW_SIZE; i++) + for(; i < ERTS_TW_LATER_WHEEL_END_SLOT; i++) tiw->w[i] = NULL; mtime = erts_get_monotonic_time(esdp); @@ -537,6 +914,10 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp) tiw->at_once.head = NULL; tiw->at_once.tail = NULL; tiw->at_once.nto = 0; + tiw->soon.nto = 0; + tiw->later.pos = tiw->pos; + tiw->later.pos &= ERTS_TW_LATER_WHEEL_POS_MASK; + tiw->later.pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE; tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; tiw->true_next_timeout_time = 0; tiw->next_timeout_time = mtime + ERTS_MONOTONIC_DAY; @@ -586,36 +967,40 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw, ERTS_TW_ASSERT(p->slot == ERTS_TWHEEL_SLOT_INACTIVE); + tiw->nto++; if (timeout_pos <= tiw->pos) { - tiw->nto++; - tiw->at_once.nto++; - p->next = NULL; - p->prev = tiw->at_once.tail; - if (tiw->at_once.tail) { - ERTS_TW_ASSERT(tiw->at_once.head); - tiw->at_once.tail->next = p; - } - else { - ERTS_TW_ASSERT(!tiw->at_once.head); - tiw->at_once.head = p; - } - tiw->at_once.tail = p; - p->timeout_pos = tiw->pos; - p->slot = ERTS_TWHEEL_SLOT_AT_ONCE; + p->timeout_pos = tiw->pos; + insert_timer_into_at_once_list(tiw, p); timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(tiw->pos); } else { int slot; + p->timeout_pos = timeout_pos; + /* calculate slot */ - slot = (int) (timeout_pos & (ERTS_TIW_SIZE-1)); + if (timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE) { + /* soon wheel */ + slot = soon_slot(timeout_pos); + } + else { + /* later wheel */ + slot = later_slot(timeout_pos); + + /* + * Next timeout due to this timeout + * should be in good time before the + * actual timeout (one later wheel slot + * size). This, in order to move it + * from the later wheel to the soon + * wheel. + */ + timeout_pos &= ERTS_TW_LATER_WHEEL_POS_MASK; + timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + } insert_timer_into_slot(tiw, slot, p); - - tiw->nto++; - - timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos); - p->timeout_pos = timeout_pos; + timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos); } if (timeout_time < tiw->next_timeout_time) { @@ -631,6 +1016,7 @@ erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) if (p->slot != ERTS_TWHEEL_SLOT_INACTIVE) { ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); remove_timer(tiw, p); + tiw->nto--; ERTS_MSACC_POP_STATE_M_X(); } } @@ -658,7 +1044,9 @@ erts_twheel_debug_foreach(ErtsTimerWheel *tiw, (*func)(arg, tmr->timeout_pos, tmr->arg); } - for (ix = 0; ix < ERTS_TIW_SIZE; ix++) { + for (ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; + ix < ERTS_TW_LATER_WHEEL_END_SLOT; + ix++) { tmr = tiw->w[ix]; if (tmr) { do { @@ -670,36 +1058,92 @@ erts_twheel_debug_foreach(ErtsTimerWheel *tiw, } } -#ifdef ERTS_TW_DEBUG -void erts_p_slpq(void) +#ifdef ERTS_TW_HARD_DEBUG + +static void +hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos) { - erts_printf("Not yet implemented...\n"); -#if 0 - ErtsMonotonicTime current_time = erts_get_monotonic_time(NULL); - int i; - ErtsTWheelTimer* p; - - /* print the whole wheel, starting at the current position */ - erts_printf("\ncurrent time = %bps tiw_pos = %d tiw_nto %d\n", - current_time, tiw->pos, tiw->nto); - i = tiw->pos; - if (tiw->w[i] != NULL) { - erts_printf("%d:\n", i); - for(p = tiw->w[i]; p != NULL; p = p->next) { - erts_printf(" (timeout time %bps, slot %d)\n", - ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos), - p->slot); - } + int ix, soon_tmo, later_tmo, at_once_tmo; + ErtsMonotonicTime min_tpos; + + min_tpos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time); + + soon_tmo = 0; + for (ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; + ix < ERTS_TW_SOON_WHEEL_END_SLOT; + ix++) { + ErtsTWheelTimer *p; + + p = tiw->w[ix]; + if (p) { + ErtsTWheelTimer *first = p; + do { + ErtsMonotonicTime tpos = p->timeout_pos; + ERTS_TW_ASSERT(p->slot == ix); + soon_tmo++; + ERTS_TW_ASSERT(ix == soon_slot(tpos)); + ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); + ERTS_TW_ASSERT(!check_min_tpos || tpos >= min_tpos); + ERTS_TW_ASSERT(p->next->prev == p); + p = p->next; + } while (p != first); + } } - for(i = ((i+1) & (ERTS_TIW_SIZE-1)); i != (tiw->pos & (ERTS_TIW_SIZE-1)); i = ((i+1) & (ERTS_TIW_SIZE-1))) { - if (tiw->w[i] != NULL) { - erts_printf("%d:\n", i); - for(p = tiw->w[i]; p != NULL; p = p->next) { - erts_printf(" (timeout time %bps, slot %d)\n", - ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos), p->slot); - } - } + + later_tmo = 0; + for (ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT; + ix < ERTS_TW_LATER_WHEEL_END_SLOT; + ix++) { + ErtsTWheelTimer *p; + + p = tiw->w[ix]; + if (p) { + ErtsTWheelTimer *first = p; + do { + ErtsMonotonicTime tpos = p->timeout_pos; + ERTS_TW_ASSERT(p->slot == ix); + later_tmo++; + ERTS_TW_ASSERT(later_slot(tpos) == ix); + tpos &= ERTS_TW_LATER_WHEEL_POS_MASK; + tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + ERTS_TW_ASSERT(!check_min_tpos || tpos >= min_tpos); + ERTS_TW_ASSERT(p->next->prev == p); + p = p->next; + } while (p != first); + } } -#endif + + if (tiw->yield_slot >= 0) { + ErtsTWheelTimer *p = tiw->sentinel.next; + while (p != &tiw->sentinel) { + ErtsMonotonicTime tpos = p->timeout_pos; + if (tiw->yield_slot >= ERTS_TW_SOON_WHEEL_SIZE) { + later_tmo++; + ERTS_TW_ASSERT(p->slot == later_slot(tpos)); + } + else { + soon_tmo++; + ERTS_TW_ASSERT(p->slot == (tpos & ERTS_TW_SOON_WHEEL_MASK)); + ERTS_TW_ASSERT(tpos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); + } + p = p->next; + } + } + + at_once_tmo = 0; + if (tiw->at_once.head) { + ErtsTWheelTimer *p = tiw->at_once.head; + while (p) { + ErtsMonotonicTime tpos = p->timeout_pos; + at_once_tmo++; + ERTS_TW_ASSERT(tpos <= tiw->pos); + p = p->next; + } + } + + ERTS_TW_ASSERT(tiw->at_once.nto == at_once_tmo); + ERTS_TW_ASSERT(tiw->soon.nto == soon_tmo); + ERTS_TW_ASSERT(tiw->nto == soon_tmo + later_tmo + at_once_tmo); } -#endif /* ERTS_TW_DEBUG */ + +#endif -- cgit v1.2.3 From 2ed143ac11d858db728bfa69f14fb34d7f449305 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 10 Feb 2017 17:16:51 +0100 Subject: Introduce timer slot range counters Timer counters for ranges of slots in order to be able to avoid inspecting large ranges of slots without timers. --- erts/emulator/beam/time.c | 374 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 301 insertions(+), 73 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 7b5ff7b992..1b6ca2b269 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -18,7 +18,6 @@ * %CopyrightEnd% */ - /* * TIMER WHEEL * @@ -134,6 +133,32 @@ * bump operation is performed thise timers will be triggered * at once. * + * -- Searching for Next Timeout -- + * + * In order to limit the amount of work needed in order to find + * next timeout, we keep track of total amount of timers in the + * wheels, total amount of timers in soon wheel, and the total + * amount of timers in each range of slots. Each slot range + * currently contain 512 slots. + * + * When next timeout is less than half the soon wheel width away + * we determine the exact timeout. Due to the timer counts of + * slot ranges, we currently at most need to search 1024 slots + * in the soon wheel. This besides inspecting slot range counts + * and two slots in the later wheel which potentially might trigger + * timeouts for moving timers from the later wheel to the soon wheel + * earlier than timeouts in the soon wheel. + * + * When next timeout is further away than half the soon wheel + * width we settle for the earliest possible timeout in the first + * non-empty slot range. The further away the next timeout is, the + * more likely it is that the next timeout change before we + * actually get there. That is, a change due to another timer is + * set to an earlier time and/or the timer is cancelled. It is + * therefore in this case no point determining next timeout + * exactly. If the state should not change, we will wake up a bit + * early and do a recalculation of next timeout and eventually + * we will be so close to it that we determine it exactly. * */ @@ -147,8 +172,8 @@ #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" -#define ERTS_MONOTONIC_DAY ERTS_SEC_TO_MONOTONIC(60*60*24) -#define ERTS_CLKTCKS_DAY ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY) +#define ERTS_MONOTONIC_WEEK ERTS_SEC_TO_MONOTONIC(7*60*60*24) +#define ERTS_CLKTCKS_WEEK ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_WEEK) #ifdef ERTS_ENABLE_LOCK_CHECK #define ASSERT_NO_LOCKED_LOCKS erts_lc_check_exact(NULL, 0) @@ -211,6 +236,21 @@ #define ERTS_TW_LATER_WHEEL_MASK (ERTS_TW_LATER_WHEEL_SIZE-1) +#define ERTS_TW_SCNT_BITS 9 +#define ERTS_TW_SCNT_SHIFT +#define ERTS_TW_SCNT_SIZE \ + ((ERTS_TW_SOON_WHEEL_SIZE + ERTS_TW_LATER_WHEEL_SIZE) \ + >> ERTS_TW_SCNT_BITS) + +#ifdef __GNUC__ +#if ERTS_TW_SOON_WHEEL_BITS < ERTS_TW_SCNT_BITS +# warning Consider larger soon timer wheel +#endif +#if ERTS_TW_SOON_WHEEL_BITS < ERTS_TW_SCNT_BITS +# warning Consider larger later timer wheel +#endif +#endif + /* Actual interval time chosen by sys_init_time() */ #if SYS_CLOCK_RESOLUTION == 1 @@ -223,6 +263,8 @@ static int tiw_itime; /* Constant after init */ struct ErtsTimerWheel_ { ErtsTWheelTimer *w[ERTS_TW_SOON_WHEEL_SIZE + ERTS_TW_LATER_WHEEL_SIZE]; + Sint scnt[ERTS_TW_SCNT_SIZE]; + Sint bump_scnt[ERTS_TW_SCNT_SIZE]; ErtsMonotonicTime pos; Uint nto; struct { @@ -248,6 +290,108 @@ struct ErtsTimerWheel_ { static int bump_later_wheel(ErtsTimerWheel *tiw, int *yield_count_p); +static ERTS_INLINE int +scnt_get_ix(int slot) +{ + return slot >> ERTS_TW_SCNT_BITS; +} + +static ERTS_INLINE void +scnt_inc(Sint *scnt, int slot) +{ + scnt[slot >> ERTS_TW_SCNT_BITS]++; +} + +#ifdef ERTS_TW_HARD_DEBUG + +static ERTS_INLINE void +scnt_ix_inc(Sint *scnt, int six) +{ + scnt[six]++; +} + +#endif + +static ERTS_INLINE void +scnt_dec(Sint *scnt, int slot) +{ + scnt[slot >> ERTS_TW_SCNT_BITS]--; + ERTS_TW_ASSERT(scnt[slot >> ERTS_TW_SCNT_BITS] >= 0); +} + +static ERTS_INLINE void +scnt_ix_dec(Sint *scnt, int six) +{ + scnt[six]--; + ERTS_TW_ASSERT(scnt[six] >= 0); +} + +static ERTS_INLINE void +scnt_wheel_next(int *slotp, int *leftp, ErtsMonotonicTime *posp, + int *sixp, Sint *scnt, int first_slot, + int end_slot, ErtsMonotonicTime slot_sz) +{ + int slot = *slotp; + int left = *leftp; + int ix; + + ERTS_TW_ASSERT(*leftp >= 0); + + left--; + slot++; + if (slot == end_slot) + slot = first_slot; + ix = slot >> ERTS_TW_SCNT_BITS; + + while (!scnt[ix] && left > 0) { + int diff, old_slot = slot; + ix++; + slot = (ix << ERTS_TW_SCNT_BITS); + diff = slot - old_slot; + if (left < diff) { + slot = old_slot + left; + diff = left; + } + if (slot < end_slot) + left -= diff; + else { + left -= end_slot - old_slot; + slot = first_slot; + ix = slot >> ERTS_TW_SCNT_BITS; + } + } + + ERTS_TW_ASSERT(left >= -1); + + if (posp) + *posp += slot_sz * ((ErtsMonotonicTime) (*leftp - left)); + if (sixp) + *sixp = slot >> ERTS_TW_SCNT_BITS; + *leftp = left; + *slotp = slot; +} + + +static ERTS_INLINE void +scnt_soon_wheel_next(int *slotp, int *leftp, ErtsMonotonicTime *posp, + int *sixp, Sint *scnt) +{ + scnt_wheel_next(slotp, leftp, posp, sixp, scnt, + ERTS_TW_SOON_WHEEL_FIRST_SLOT, + ERTS_TW_SOON_WHEEL_END_SLOT, 1); +} + +static ERTS_INLINE void +scnt_later_wheel_next(int *slotp, int *leftp, ErtsMonotonicTime *posp, + int *sixp, Sint *scnt) +{ + scnt_wheel_next(slotp, leftp, posp, sixp, scnt, + ERTS_TW_LATER_WHEEL_FIRST_SLOT, + ERTS_TW_LATER_WHEEL_END_SLOT, + ERTS_TW_LATER_WHEEL_SLOT_SIZE); +} + + static ERTS_INLINE int soon_slot(ErtsMonotonicTime soon_pos) { @@ -282,14 +426,10 @@ static void hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos); #define ERTS_HARD_DBG_CHK_WHEELS(TIW, CHK_MIN_TPOS) #endif -static ERTS_INLINE ErtsMonotonicTime -find_next_timeout(ErtsSchedulerData *esdp, - ErtsTimerWheel *tiw, - int search_all, - ErtsMonotonicTime curr_time, /* When !search_all */ - ErtsMonotonicTime max_search_time) /* When !search_all */ +static ErtsMonotonicTime +find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw, int thorough) { - int start_ix, tiw_pos_ix, pos_inc, wheel_start_ix, wheel_end_ix; + int slot, slots; int true_min_timeout = 0; ErtsMonotonicTime min_timeout, min_timeout_pos, slot_timeout_pos; @@ -298,32 +438,51 @@ find_next_timeout(ErtsSchedulerData *esdp, ERTS_TW_ASSERT(tiw->yield_slot == ERTS_TWHEEL_SLOT_INACTIVE); if (tiw->nto == 0) { /* no timeouts in wheel */ - if (!search_all) + if (!thorough) min_timeout_pos = tiw->pos; else { - curr_time = erts_get_monotonic_time(esdp); + ErtsMonotonicTime curr_time = erts_get_monotonic_time(esdp); tiw->pos = min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); } - min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY); + min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_WEEK); goto done; } - if (search_all) - min_timeout_pos = tiw->pos + ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY); + min_timeout_pos = tiw->pos; + if (thorough) + min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_WEEK); else - min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + max_search_time); + min_timeout_pos += ERTS_TW_SOON_WHEEL_SIZE/2; if (!tiw->soon.nto) { + ErtsMonotonicTime tmp; /* Select later wheel... */ slot_timeout_pos = tiw->later.pos; - start_ix = tiw_pos_ix = later_slot(slot_timeout_pos); - pos_inc = ERTS_TW_LATER_WHEEL_SLOT_SIZE; - wheel_start_ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT; - wheel_end_ix = ERTS_TW_LATER_WHEEL_END_SLOT; + slot = later_slot(slot_timeout_pos); /* Pre-timeout for move from later to soon wheel... */ slot_timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + tmp = min_timeout_pos - slot_timeout_pos; + tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + tmp++; + if (tmp > ERTS_TW_LATER_WHEEL_SIZE) + slots = ERTS_TW_LATER_WHEEL_SIZE; + else + slots = (int) tmp; + + /* + * We never search for an exact timeout in the + * later wheel, but instead settle for the first + * scnt range used. + */ + if (tiw->w[slot]) + true_min_timeout = 1; + else + scnt_later_wheel_next(&slot, &slots, &slot_timeout_pos, + NULL, tiw->scnt); + min_timeout_pos = slot_timeout_pos; } else { + ErtsMonotonicTime tmp; /* Select soon wheel... */ /* @@ -355,30 +514,31 @@ find_next_timeout(ErtsSchedulerData *esdp, } slot_timeout_pos = tiw->pos; - start_ix = tiw_pos_ix = soon_slot(tiw->pos); - pos_inc = 1; - wheel_start_ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; - wheel_end_ix = ERTS_TW_SOON_WHEEL_END_SLOT; - } + slot = soon_slot(tiw->pos); + tmp = min_timeout_pos - slot_timeout_pos; + if (tmp > ERTS_TW_SOON_WHEEL_SIZE) + slots = ERTS_TW_SOON_WHEEL_SIZE; + else + slots = (int) tmp; - /* - * Scan selected wheel... - */ - do { - if (tiw->w[tiw_pos_ix]) { - min_timeout_pos = slot_timeout_pos; - true_min_timeout = 1; - break; + while (slot_timeout_pos < min_timeout_pos) { + if (tiw->w[slot]) { + ERTS_TW_ASSERT(tiw->w[slot]->timeout_pos == slot_timeout_pos); + min_timeout_pos = slot_timeout_pos; + true_min_timeout = 1; + break; + } + scnt_soon_wheel_next(&slot, &slots, &slot_timeout_pos, + NULL, tiw->scnt); } - slot_timeout_pos += pos_inc; - if (slot_timeout_pos >= min_timeout_pos) - break; - - tiw_pos_ix++; - if (tiw_pos_ix == wheel_end_ix) - tiw_pos_ix = wheel_start_ix; - } while (start_ix != tiw_pos_ix); + if (slots > 0 && !true_min_timeout) { + /* Find first non-empty range... */ + scnt_soon_wheel_next(&slot, &slots, &slot_timeout_pos, + NULL, tiw->scnt); + min_timeout_pos = slot_timeout_pos; + } + } done: @@ -433,6 +593,7 @@ insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p) ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); tiw->soon.nto++; } + scnt_inc(tiw->scnt, slot); } static ERTS_INLINE void @@ -461,6 +622,7 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) } if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) tiw->soon.nto--; + scnt_dec(tiw->scnt, slot); } else { /* Timer in "at once" queue... */ @@ -480,7 +642,6 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) ERTS_TW_ASSERT(tiw->at_once.nto > 0); tiw->at_once.nto--; } - p->slot = ERTS_TWHEEL_SLOT_INACTIVE; } @@ -493,7 +654,7 @@ erts_check_next_timeout_time(ErtsSchedulerData *esdp) if (tiw->true_next_timeout_time) return tiw->next_timeout_time; ERTS_MSACC_PUSH_AND_SET_STATE_CACHED_X(ERTS_MSACC_STATE_TIMERS); - time = find_next_timeout(esdp, tiw, 1, 0, 0); + time = find_next_timeout(esdp, tiw, 1); ERTS_MSACC_POP_STATE_M_X(); return time; } @@ -535,9 +696,10 @@ debug_check_safe_to_skip_to(ErtsTimerWheel *tiw, ErtsMonotonicTime skip_to_pos) ix = later_slot(tiw->later.pos); tmp = skip_to_pos; tmp &= ERTS_TW_LATER_WHEEL_POS_MASK; - if (tmp > tiw->later.pos) { + if (tmp >= tiw->later.pos) { tmp -= tiw->later.pos; tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + tmp++; ERTS_TW_ASSERT(tmp >= 0); if (tmp < (ErtsMonotonicTime) ERTS_TW_LATER_WHEEL_SIZE) slots = (int) tmp; @@ -580,12 +742,16 @@ timeout_timer(ErtsTWheelTimer *p) void erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) { - int tiw_pos_ix, yielded_slot_restarted, yield_count, slots; + int slot, yielded_slot_restarted, yield_count, slots, scnt_ix; ErtsMonotonicTime bump_to; + Sint *scnt, *bump_scnt; ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); yield_count = ERTS_TWHEEL_BUMP_YIELD_LIMIT; + scnt = &tiw->scnt[0]; + bump_scnt = &tiw->bump_scnt[0]; + /* * In order to be fair we always continue with work * where we left off when restarting after a yield. @@ -596,7 +762,8 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) bump_to = tiw->pos; if (tiw->yield_slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) goto restart_yielded_later_slot; - tiw_pos_ix = tiw->yield_slot; + slot = tiw->yield_slot; + scnt_ix = scnt_get_ix(slot); slots = tiw->yield_slots_left; ASSERT(0 <= slots && slots <= ERTS_TW_SOON_WHEEL_SIZE); tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; @@ -616,7 +783,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) empty_wheel: ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, bump_to); tiw->true_next_timeout_time = 0; - tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY; + tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_WEEK; tiw->pos = bump_to; tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; ERTS_MSACC_POP_STATE_M_X(); @@ -660,6 +827,22 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) if (tiw->nto == 0) goto empty_wheel; + /* + * Save slot counts in bump operation local + * array. + * + * The amount of timers to trigger (or move) + * will only decrease from now until we have + * completed this bump operation (even if we + * yield in the middle of it). + * + * The amount of timers in the wheels may + * however increase due to timers being set + * by timeout callbacks. + */ + sys_memcpy((void *) bump_scnt, (void *) scnt, + sizeof(Sint) * ERTS_TW_SCNT_SIZE); + if (tiw->true_next_timeout_time) { ErtsMonotonicTime skip_until_pos; /* @@ -694,15 +877,17 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) slots = ERTS_TW_SOON_WHEEL_SIZE; } - tiw_pos_ix = soon_slot(tiw->pos+1); + slot = soon_slot(tiw->pos+1); tiw->pos = bump_to; + scnt_ix = scnt_get_ix(slot); + /* Timeout timers in soon wheel */ - while (slots) { + while (slots > 0) { yield_count -= ERTS_TW_COST_SLOT; - p = tiw->w[tiw_pos_ix]; + p = tiw->w[slot]; if (p) { /* timeout callback need tiw->pos to be up to date */ if (p->next == p) { @@ -715,21 +900,23 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) tiw->sentinel.next->prev = &tiw->sentinel; tiw->sentinel.prev->next = &tiw->sentinel; } - tiw->w[tiw_pos_ix] = NULL; + tiw->w[slot] = NULL; while (1) { ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT <= p->slot && p->slot < ERTS_TW_SOON_WHEEL_END_SLOT); tiw->soon.nto--; + scnt_ix_dec(scnt, scnt_ix); if (p->timeout_pos <= bump_to) { timeout_timer(p); tiw->nto--; + scnt_ix_dec(bump_scnt, scnt_ix); yield_count -= ERTS_TW_COST_TIMEOUT; } else { /* uncommon case */ - insert_timer_into_slot(tiw, tiw_pos_ix, p); + insert_timer_into_slot(tiw, slot, p); yield_count -= ERTS_TW_COST_SLOT_MOVE; } @@ -744,7 +931,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) if (yield_count <= 0) { tiw->true_next_timeout_time = 1; tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to); - tiw->yield_slot = tiw_pos_ix; + tiw->yield_slot = slot; tiw->yield_slots_left = slots; ERTS_MSACC_POP_STATE_M_X(); return; /* Yield! */ @@ -755,10 +942,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) } } - tiw_pos_ix++; - if (tiw_pos_ix == ERTS_TW_SOON_WHEEL_END_SLOT) - tiw_pos_ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; - slots--; + scnt_soon_wheel_next(&slot, &slots, NULL, &scnt_ix, bump_scnt); } if (ERTS_TW_BUMP_LATER_WHEEL(tiw)) { @@ -771,10 +955,9 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) } while (yielded_slot_restarted); tiw->true_next_timeout_time = 0; - tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY; + tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_WEEK; - /* Search at most two seconds ahead... */ - (void) find_next_timeout(NULL, tiw, 0, curr_time, ERTS_SEC_TO_MONOTONIC(2)); + (void) find_next_timeout(NULL, tiw, 0); ERTS_MSACC_POP_STATE_M_X(); } @@ -784,12 +967,17 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) ErtsMonotonicTime cpos = tiw->pos; ErtsMonotonicTime later_pos = tiw->later.pos; int ycount = *ycount_p; - int slots, fslot; + int slots, fslot, scnt_ix; + Sint *scnt, *bump_scnt; + + scnt = &tiw->scnt[0]; + bump_scnt = &tiw->bump_scnt[0]; ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); if (tiw->yield_slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) { fslot = tiw->yield_slot; + scnt_ix = scnt_get_ix(fslot); slots = tiw->yield_slots_left; ASSERT(0 <= slots && slots <= ERTS_TW_LATER_WHEEL_SIZE); tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; @@ -800,17 +988,18 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) tmp_slots += ERTS_TW_LATER_WHEEL_SLOT_SIZE; tmp_slots -= later_pos; tmp_slots /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + tmp_slots++; if (tmp_slots < ERTS_TW_LATER_WHEEL_SIZE) slots = (int) tmp_slots; else slots = ERTS_TW_LATER_WHEEL_SIZE; + fslot = later_slot(later_pos); + scnt_ix = scnt_get_ix(fslot); } - while (slots >= 0) { + while (slots > 0) { ErtsTWheelTimer *p; - fslot = later_slot(later_pos); - ycount -= ERTS_TW_COST_SLOT; p = tiw->w[fslot]; @@ -835,12 +1024,15 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) ERTS_TW_ASSERT(tpos >= later_pos); ERTS_TW_ASSERT(p->slot == fslot); + scnt_ix_dec(scnt, scnt_ix); + if (tpos >= later_pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE) { /* keep in later slot; very uncommon... */ insert_timer_into_slot(tiw, fslot, p); ycount -= ERTS_TW_COST_SLOT_MOVE; } else { + scnt_ix_dec(bump_scnt, scnt_ix); ERTS_TW_ASSERT(tpos < cpos + ERTS_TW_SOON_WHEEL_SIZE); if (tpos > cpos) { /* move into soon wheel */ @@ -878,8 +1070,8 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) p->next->prev = &tiw->sentinel; } } - slots--; - later_pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE; + + scnt_later_wheel_next(&fslot, &slots, &later_pos, &scnt_ix, bump_scnt); } ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); @@ -908,6 +1100,9 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp) for(; i < ERTS_TW_LATER_WHEEL_END_SLOT; i++) tiw->w[i] = NULL; + for (i = 0; i < ERTS_TW_SCNT_SIZE; i++) + tiw->scnt[i] = 0; + mtime = erts_get_monotonic_time(esdp); tiw->pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime); tiw->nto = 0; @@ -920,7 +1115,7 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp) tiw->later.pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE; tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; tiw->true_next_timeout_time = 0; - tiw->next_timeout_time = mtime + ERTS_MONOTONIC_DAY; + tiw->next_timeout_time = mtime + ERTS_MONOTONIC_WEEK; tiw->sentinel.next = &tiw->sentinel; tiw->sentinel.prev = &tiw->sentinel; tiw->sentinel.timeout = NULL; @@ -1063,24 +1258,37 @@ erts_twheel_debug_foreach(ErtsTimerWheel *tiw, static void hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos) { - int ix, soon_tmo, later_tmo, at_once_tmo; + int ix, six, soon_tmo, later_tmo, at_once_tmo, + scnt_slot, scnt_slots, scnt_six; ErtsMonotonicTime min_tpos; + Sint scnt[ERTS_TW_SCNT_SIZE]; + + for (six = 0; six < ERTS_TW_SCNT_SIZE; six++) + scnt[six] = 0; min_tpos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time); soon_tmo = 0; + scnt_slot = ERTS_TW_SOON_WHEEL_END_SLOT-1; + scnt_slots = ERTS_TW_SOON_WHEEL_SIZE; + scnt_six = 0; + scnt_soon_wheel_next(&scnt_slot, &scnt_slots, + NULL, &scnt_six, tiw->scnt); for (ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; ix < ERTS_TW_SOON_WHEEL_END_SLOT; ix++) { ErtsTWheelTimer *p; p = tiw->w[ix]; + six = scnt_get_ix(ix); + ERTS_TW_ASSERT(!p || six == scnt_six); if (p) { ErtsTWheelTimer *first = p; do { ErtsMonotonicTime tpos = p->timeout_pos; - ERTS_TW_ASSERT(p->slot == ix); soon_tmo++; + scnt_ix_inc(scnt, six); + ERTS_TW_ASSERT(p->slot == ix); ERTS_TW_ASSERT(ix == soon_slot(tpos)); ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); ERTS_TW_ASSERT(!check_min_tpos || tpos >= min_tpos); @@ -1088,21 +1296,33 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos) p = p->next; } while (p != first); } + if (ix == scnt_slot) + scnt_soon_wheel_next(&scnt_slot, &scnt_slots, + NULL, &scnt_six, tiw->scnt); } later_tmo = 0; + scnt_slot = ERTS_TW_SOON_WHEEL_END_SLOT-1; + scnt_slots = ERTS_TW_SOON_WHEEL_SIZE; + scnt_six = 0; + scnt_later_wheel_next(&scnt_slot, &scnt_slots, + NULL, &scnt_six, tiw->scnt); for (ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT; ix < ERTS_TW_LATER_WHEEL_END_SLOT; ix++) { ErtsTWheelTimer *p; p = tiw->w[ix]; + six = scnt_get_ix(ix); + ERTS_TW_ASSERT(!p || six == scnt_six); if (p) { ErtsTWheelTimer *first = p; + six = scnt_get_ix(ix); do { ErtsMonotonicTime tpos = p->timeout_pos; - ERTS_TW_ASSERT(p->slot == ix); later_tmo++; + scnt_ix_inc(scnt, six); + ERTS_TW_ASSERT(p->slot == ix); ERTS_TW_ASSERT(later_slot(tpos) == ix); tpos &= ERTS_TW_LATER_WHEEL_POS_MASK; tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; @@ -1111,19 +1331,24 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos) p = p->next; } while (p != first); } + if (ix == scnt_slot) + scnt_later_wheel_next(&scnt_slot, &scnt_slots, + NULL, &scnt_six, tiw->scnt); } if (tiw->yield_slot >= 0) { ErtsTWheelTimer *p = tiw->sentinel.next; + ix = tiw->yield_slot; while (p != &tiw->sentinel) { ErtsMonotonicTime tpos = p->timeout_pos; - if (tiw->yield_slot >= ERTS_TW_SOON_WHEEL_SIZE) { + scnt_inc(scnt, ix); + if (ix >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) { later_tmo++; - ERTS_TW_ASSERT(p->slot == later_slot(tpos)); + ERTS_TW_ASSERT(ix == later_slot(tpos)); } else { soon_tmo++; - ERTS_TW_ASSERT(p->slot == (tpos & ERTS_TW_SOON_WHEEL_MASK)); + ERTS_TW_ASSERT(ix == (tpos & ERTS_TW_SOON_WHEEL_MASK)); ERTS_TW_ASSERT(tpos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); } p = p->next; @@ -1144,6 +1369,9 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos) ERTS_TW_ASSERT(tiw->at_once.nto == at_once_tmo); ERTS_TW_ASSERT(tiw->soon.nto == soon_tmo); ERTS_TW_ASSERT(tiw->nto == soon_tmo + later_tmo + at_once_tmo); + + for (six = 0; six < ERTS_TW_SCNT_SIZE; six++) + ERTS_TW_ASSERT(scnt[six] == tiw->scnt[six]); } #endif -- cgit v1.2.3 From 25e9f3abd395c26eccb7962acc47dffe90beec81 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Sun, 12 Feb 2017 04:03:55 +0100 Subject: Manage timers to trigger at once in a slot similar to other timers --- erts/emulator/beam/erl_time.h | 5 +- erts/emulator/beam/time.c | 365 ++++++++++++++++++++++-------------------- 2 files changed, 193 insertions(+), 177 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 699e69e991..46d6da6448 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -448,8 +448,7 @@ erts_time_unit_conversion(Uint64 value, void erts_sched_init_time_sup(ErtsSchedulerData *esdp); -#define ERTS_TWHEEL_SLOT_AT_ONCE -1 -#define ERTS_TWHEEL_SLOT_INACTIVE -2 +#define ERTS_TW_SLOT_INACTIVE (-2) /* ** Timer entry: @@ -481,7 +480,7 @@ ERTS_GLB_INLINE ErtsMonotonicTime erts_tweel_read_timeout(ErtsTWheelTimer *twt); ERTS_GLB_INLINE void erts_twheel_init_timer(ErtsTWheelTimer *p) { - p->slot = ERTS_TWHEEL_SLOT_INACTIVE; + p->slot = ERTS_TW_SLOT_INACTIVE; } ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef nxt_tmo_ref) diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 1b6ca2b269..16e4328a05 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -125,13 +125,17 @@ * slot position by just removing it from the circular double * linked list that it is in. * - * -- At Once List -- + * -- At Once Slot -- * * If a timer is set that has a time earlier or equal to 'pos', * it is not inserted into the wheel. It is instead inserted, - * into a list referred to by the 'at_once' field. When the - * bump operation is performed thise timers will be triggered - * at once. + * into a circular double linked list referred to by the "at + * once" slot. When the bump operation is performed these timers + * will be triggered at once. The circular list of the slot will + * be moved to the 'sentinel' field while bumping these timers + * as when bumping an ordinary wheel slot. A yielding bump + * operation and cancelation of timers is handled the same way + * as if the timer was in a wheel slot. * * -- Searching for Next Timeout -- * @@ -262,14 +266,15 @@ static int tiw_itime; /* Constant after init */ #endif struct ErtsTimerWheel_ { - ErtsTWheelTimer *w[ERTS_TW_SOON_WHEEL_SIZE + ERTS_TW_LATER_WHEEL_SIZE]; + ErtsTWheelTimer *slots[1 /* At Once Slot */ + + ERTS_TW_SOON_WHEEL_SIZE /* Soon Wheel Slots */ + + ERTS_TW_LATER_WHEEL_SIZE]; /* Later Wheel Slots */ + ErtsTWheelTimer **w; Sint scnt[ERTS_TW_SCNT_SIZE]; Sint bump_scnt[ERTS_TW_SCNT_SIZE]; ErtsMonotonicTime pos; Uint nto; struct { - ErtsTWheelTimer *head; - ErtsTWheelTimer *tail; Uint nto; } at_once; struct { @@ -285,6 +290,8 @@ struct ErtsTimerWheel_ { ErtsMonotonicTime next_timeout_time; }; +#define ERTS_TW_SLOT_AT_ONCE (-1) + #define ERTS_TW_BUMP_LATER_WHEEL(TIW) \ ((tiw)->pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE >= (TIW)->later.pos) @@ -435,7 +442,7 @@ find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw, int thorough) ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); - ERTS_TW_ASSERT(tiw->yield_slot == ERTS_TWHEEL_SLOT_INACTIVE); + ERTS_TW_ASSERT(tiw->yield_slot == ERTS_TW_SLOT_INACTIVE); if (tiw->nto == 0) { /* no timeouts in wheel */ if (!thorough) @@ -551,29 +558,11 @@ done: return min_timeout; } -static ERTS_INLINE void -insert_timer_into_at_once_list(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) -{ - tiw->at_once.nto++; - p->next = NULL; - p->prev = tiw->at_once.tail; - if (tiw->at_once.tail) { - ERTS_TW_ASSERT(tiw->at_once.head); - tiw->at_once.tail->next = p; - } - else { - ERTS_TW_ASSERT(!tiw->at_once.head); - tiw->at_once.head = p; - } - tiw->at_once.tail = p; - p->slot = ERTS_TWHEEL_SLOT_AT_ONCE; -} - static ERTS_INLINE void insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p) { - ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT <= slot); - ERTS_TW_ASSERT(slot < ERTS_TW_LATER_WHEEL_END_SLOT); + ERTS_TW_ASSERT(ERTS_TW_SLOT_AT_ONCE <= slot + && slot < ERTS_TW_LATER_WHEEL_END_SLOT); p->slot = slot; if (!tiw->w[slot]) { tiw->w[slot] = p; @@ -589,60 +578,53 @@ insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p) prev->next = p; next->prev = p; } - if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) { - ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); - tiw->soon.nto++; + if (slot == ERTS_TW_SLOT_AT_ONCE) + tiw->at_once.nto++; + else { + if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) { + ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); + tiw->soon.nto++; + } + scnt_inc(tiw->scnt, slot); } - scnt_inc(tiw->scnt, slot); } static ERTS_INLINE void remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) { int slot = p->slot; - ERTS_TW_ASSERT(slot != ERTS_TWHEEL_SLOT_INACTIVE); - - if (slot >= ERTS_TW_SOON_WHEEL_FIRST_SLOT) { - /* - * Timer in wheel or in circular - * list of timers currently beeing - * triggered (referred by sentinel). - */ - ERTS_TW_ASSERT(slot < ERTS_TW_LATER_WHEEL_END_SLOT); - - if (p->next == p) { - ERTS_TW_ASSERT(tiw->w[slot] == p); - tiw->w[slot] = NULL; - } - else { - if (tiw->w[slot] == p) - tiw->w[slot] = p->next; - p->prev->next = p->next; - p->next->prev = p->prev; - } - if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) - tiw->soon.nto--; - scnt_dec(tiw->scnt, slot); + ERTS_TW_ASSERT(slot != ERTS_TW_SLOT_INACTIVE); + + /* + * Timer is in circular list either referred to + * by at once slot, slot in soon wheel, slot + * in later wheel, or by sentinel (timers currently + * being triggered). + */ + ERTS_TW_ASSERT(ERTS_TW_SLOT_AT_ONCE <= slot + && slot < ERTS_TW_LATER_WHEEL_END_SLOT); + + if (p->next == p) { + /* Cannot be referred by sentinel, i.e. must be referred by slot... */ + ERTS_TW_ASSERT(tiw->w[slot] == p); + tiw->w[slot] = NULL; } else { - /* Timer in "at once" queue... */ - ERTS_TW_ASSERT(slot == ERTS_TWHEEL_SLOT_AT_ONCE); - if (p->prev) - p->prev->next = p->next; - else { - ERTS_TW_ASSERT(tiw->at_once.head == p); - tiw->at_once.head = p->next; - } - if (p->next) - p->next->prev = p->prev; - else { - ERTS_TW_ASSERT(tiw->at_once.tail == p); - tiw->at_once.tail = p->prev; - } + if (tiw->w[slot] == p) + tiw->w[slot] = p->next; + p->prev->next = p->next; + p->next->prev = p->prev; + } + if (slot == ERTS_TW_SLOT_AT_ONCE) { ERTS_TW_ASSERT(tiw->at_once.nto > 0); tiw->at_once.nto--; } - p->slot = ERTS_TWHEEL_SLOT_INACTIVE; + else { + if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) + tiw->soon.nto--; + scnt_dec(tiw->scnt, slot); + } + p->slot = ERTS_TW_SLOT_INACTIVE; } ErtsMonotonicTime @@ -732,7 +714,7 @@ timeout_timer(ErtsTWheelTimer *p) { ErlTimeoutProc timeout; void *arg; - p->slot = ERTS_TWHEEL_SLOT_INACTIVE; + p->slot = ERTS_TW_SLOT_INACTIVE; timeout = p->timeout; arg = p->arg; (*timeout)(arg); @@ -742,7 +724,7 @@ timeout_timer(ErtsTWheelTimer *p) void erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) { - int slot, yielded_slot_restarted, yield_count, slots, scnt_ix; + int slot, restarted, yield_count, slots, scnt_ix; ErtsMonotonicTime bump_to; Sint *scnt, *bump_scnt; ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); @@ -757,23 +739,24 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) * where we left off when restarting after a yield. */ - if (tiw->yield_slot >= ERTS_TW_SOON_WHEEL_FIRST_SLOT) { - yielded_slot_restarted = 1; + slot = tiw->yield_slot; + restarted = slot != ERTS_TW_SLOT_INACTIVE; + if (restarted) { bump_to = tiw->pos; - if (tiw->yield_slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) + if (slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) goto restart_yielded_later_slot; - slot = tiw->yield_slot; + tiw->yield_slot = ERTS_TW_SLOT_INACTIVE; + if (slot == ERTS_TW_SLOT_AT_ONCE) + goto restart_yielded_at_once_slot; scnt_ix = scnt_get_ix(slot); slots = tiw->yield_slots_left; ASSERT(0 <= slots && slots <= ERTS_TW_SOON_WHEEL_SIZE); - tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; goto restart_yielded_soon_slot; } do { - yielded_slot_restarted = 0; - + restarted = 0; bump_to = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); while (1) { @@ -785,38 +768,59 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) tiw->true_next_timeout_time = 0; tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_WEEK; tiw->pos = bump_to; - tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; + tiw->yield_slot = ERTS_TW_SLOT_INACTIVE; ERTS_MSACC_POP_STATE_M_X(); return; } - p = tiw->at_once.head; - while (p) { - if (yield_count <= 0) { - ERTS_TW_ASSERT(tiw->nto > 0); - ERTS_TW_ASSERT(tiw->at_once.nto > 0); - tiw->yield_slot = ERTS_TWHEEL_SLOT_AT_ONCE; - tiw->true_next_timeout_time = 1; - tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to); - ERTS_MSACC_POP_STATE_M_X(); - return; - } + p = tiw->w[ERTS_TW_SLOT_AT_ONCE]; - ERTS_TW_ASSERT(tiw->nto > 0); - ERTS_TW_ASSERT(tiw->at_once.nto > 0); - tiw->nto--; - tiw->at_once.nto--; - tiw->at_once.head = p->next; - if (p->next) - p->next->prev = NULL; - else - tiw->at_once.tail = NULL; + if (p) { - timeout_timer(p); + if (p->next == p) { + ERTS_TW_ASSERT(tiw->sentinel.next == &tiw->sentinel); + ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel); + } + else { + tiw->sentinel.next = p->next; + tiw->sentinel.prev = p->prev; + tiw->sentinel.next->prev = &tiw->sentinel; + tiw->sentinel.prev->next = &tiw->sentinel; + } + tiw->w[ERTS_TW_SLOT_AT_ONCE] = NULL; + + while (1) { + ERTS_TW_ASSERT(tiw->nto > 0); + ERTS_TW_ASSERT(tiw->at_once.nto > 0); + tiw->nto--; + tiw->at_once.nto--; + + timeout_timer(p); + + yield_count -= ERTS_TW_COST_TIMEOUT; + + restart_yielded_at_once_slot: - yield_count -= ERTS_TW_COST_TIMEOUT; + p = tiw->sentinel.next; + if (p == &tiw->sentinel) { + ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel); + break; + } + + if (yield_count <= 0) { + ERTS_TW_ASSERT(tiw->nto > 0); + ERTS_TW_ASSERT(tiw->at_once.nto > 0); + tiw->yield_slot = ERTS_TW_SLOT_AT_ONCE; + tiw->true_next_timeout_time = 1; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to); + ERTS_MSACC_POP_STATE_M_X(); + return; /* Yield! */ + } + + tiw->sentinel.next = p->next; + p->next->prev = &tiw->sentinel; + } - p = tiw->at_once.head; } if (tiw->pos >= bump_to) { @@ -952,7 +956,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) } } - } while (yielded_slot_restarted); + } while (restarted); tiw->true_next_timeout_time = 0; tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_WEEK; @@ -980,7 +984,7 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) scnt_ix = scnt_get_ix(fslot); slots = tiw->yield_slots_left; ASSERT(0 <= slots && slots <= ERTS_TW_LATER_WHEEL_SIZE); - tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; + tiw->yield_slot = ERTS_TW_SLOT_INACTIVE; goto restart_yielded_slot; } else { @@ -1093,11 +1097,29 @@ ErtsTimerWheel * erts_create_timer_wheel(ErtsSchedulerData *esdp) { ErtsMonotonicTime mtime; - int i = ERTS_TW_SOON_WHEEL_FIRST_SLOT; + int i; ErtsTimerWheel *tiw; + + /* Some compile time sanity checks... */ + + /* Slots... */ + ERTS_CT_ASSERT(ERTS_TW_SLOT_AT_ONCE == -1); + ERTS_CT_ASSERT(ERTS_TW_SLOT_INACTIVE < ERTS_TW_SLOT_AT_ONCE); + ERTS_CT_ASSERT(ERTS_TW_SLOT_AT_ONCE + 1 == ERTS_TW_SOON_WHEEL_FIRST_SLOT); + ERTS_CT_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT < ERTS_TW_SOON_WHEEL_END_SLOT); + ERTS_CT_ASSERT(ERTS_TW_SOON_WHEEL_END_SLOT == ERTS_TW_LATER_WHEEL_FIRST_SLOT); + ERTS_CT_ASSERT(ERTS_TW_LATER_WHEEL_FIRST_SLOT < ERTS_TW_LATER_WHEEL_END_SLOT); + + /* Both wheel sizes should be a powers of 2 */ + ERTS_CT_ASSERT(ERTS_TW_SOON_WHEEL_SIZE + && !(ERTS_TW_SOON_WHEEL_SIZE & (ERTS_TW_SOON_WHEEL_SIZE-1))); + ERTS_CT_ASSERT(ERTS_TW_LATER_WHEEL_SIZE + && !(ERTS_TW_LATER_WHEEL_SIZE & (ERTS_TW_LATER_WHEEL_SIZE-1))); + tiw = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_TIMER_WHEEL, sizeof(ErtsTimerWheel)); - for(; i < ERTS_TW_LATER_WHEEL_END_SLOT; i++) + tiw->w = &tiw->slots[1]; + for(i = ERTS_TW_SLOT_AT_ONCE; i < ERTS_TW_LATER_WHEEL_END_SLOT; i++) tiw->w[i] = NULL; for (i = 0; i < ERTS_TW_SCNT_SIZE; i++) @@ -1106,14 +1128,12 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp) mtime = erts_get_monotonic_time(esdp); tiw->pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime); tiw->nto = 0; - tiw->at_once.head = NULL; - tiw->at_once.tail = NULL; tiw->at_once.nto = 0; tiw->soon.nto = 0; tiw->later.pos = tiw->pos; tiw->later.pos &= ERTS_TW_LATER_WHEEL_POS_MASK; tiw->later.pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE; - tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; + tiw->yield_slot = ERTS_TW_SLOT_INACTIVE; tiw->true_next_timeout_time = 0; tiw->next_timeout_time = mtime + ERTS_MONOTONIC_WEEK; tiw->sentinel.next = &tiw->sentinel; @@ -1154,50 +1174,48 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p, ErlTimeoutProc timeout, void *arg, ErtsMonotonicTime timeout_pos) { + int slot; ErtsMonotonicTime timeout_time; ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); p->timeout = timeout; p->arg = arg; - ERTS_TW_ASSERT(p->slot == ERTS_TWHEEL_SLOT_INACTIVE); + ERTS_TW_ASSERT(p->slot == ERTS_TW_SLOT_INACTIVE); tiw->nto++; + + /* calculate slot */ if (timeout_pos <= tiw->pos) { - p->timeout_pos = tiw->pos; - insert_timer_into_at_once_list(tiw, p); - timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(tiw->pos); + /* at once */ + p->timeout_pos = timeout_pos = tiw->pos; + slot = ERTS_TW_SLOT_AT_ONCE; + } + else if (timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE) { + /* soon wheel */ + p->timeout_pos = timeout_pos; + slot = soon_slot(timeout_pos); } else { - int slot; - - p->timeout_pos = timeout_pos; - - /* calculate slot */ - if (timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE) { - /* soon wheel */ - slot = soon_slot(timeout_pos); - } - else { - /* later wheel */ - slot = later_slot(timeout_pos); - - /* - * Next timeout due to this timeout - * should be in good time before the - * actual timeout (one later wheel slot - * size). This, in order to move it - * from the later wheel to the soon - * wheel. - */ - timeout_pos &= ERTS_TW_LATER_WHEEL_POS_MASK; - timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; - } + /* later wheel */ + p->timeout_pos = timeout_pos; + slot = later_slot(timeout_pos); - insert_timer_into_slot(tiw, slot, p); - timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos); + /* + * Next timeout due to this timeout + * should be in good time before the + * actual timeout (one later wheel slot + * size). This, in order to move it + * from the later wheel to the soon + * wheel. + */ + timeout_pos &= ERTS_TW_LATER_WHEEL_POS_MASK; + timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; } + insert_timer_into_slot(tiw, slot, p); + timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos); + if (timeout_time < tiw->next_timeout_time) { tiw->true_next_timeout_time = 1; tiw->next_timeout_time = timeout_time; @@ -1208,7 +1226,7 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw, void erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) { - if (p->slot != ERTS_TWHEEL_SLOT_INACTIVE) { + if (p->slot != ERTS_TW_SLOT_INACTIVE) { ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); remove_timer(tiw, p); tiw->nto--; @@ -1234,14 +1252,7 @@ erts_twheel_debug_foreach(ErtsTimerWheel *tiw, tmr = tmr->next; } - for (tmr = tiw->at_once.head; tmr; tmr = tmr->next) { - if (tmr->timeout == tclbk) - (*func)(arg, tmr->timeout_pos, tmr->arg); - } - - for (ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; - ix < ERTS_TW_LATER_WHEEL_END_SLOT; - ix++) { + for (ix = ERTS_TW_SLOT_AT_ONCE; ix < ERTS_TW_LATER_WHEEL_END_SLOT; ix++) { tmr = tiw->w[ix]; if (tmr) { do { @@ -1262,12 +1273,27 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos) scnt_slot, scnt_slots, scnt_six; ErtsMonotonicTime min_tpos; Sint scnt[ERTS_TW_SCNT_SIZE]; + ErtsTWheelTimer *p; for (six = 0; six < ERTS_TW_SCNT_SIZE; six++) scnt[six] = 0; min_tpos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time); + at_once_tmo = 0; + p = tiw->w[ERTS_TW_SLOT_AT_ONCE]; + if (p) { + ErtsTWheelTimer *first = p; + do { + at_once_tmo++; + ERTS_TW_ASSERT(p->slot == ERTS_TW_SLOT_AT_ONCE); + ERTS_TW_ASSERT(p->timeout_pos <= tiw->pos); + ERTS_TW_ASSERT(!check_min_tpos || tiw->pos >= min_tpos); + ERTS_TW_ASSERT(p->next->prev == p); + p = p->next; + } while (p != first); + } + soon_tmo = 0; scnt_slot = ERTS_TW_SOON_WHEEL_END_SLOT-1; scnt_slots = ERTS_TW_SOON_WHEEL_SIZE; @@ -1277,8 +1303,6 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos) for (ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; ix < ERTS_TW_SOON_WHEEL_END_SLOT; ix++) { - ErtsTWheelTimer *p; - p = tiw->w[ix]; six = scnt_get_ix(ix); ERTS_TW_ASSERT(!p || six == scnt_six); @@ -1310,8 +1334,6 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos) for (ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT; ix < ERTS_TW_LATER_WHEEL_END_SLOT; ix++) { - ErtsTWheelTimer *p; - p = tiw->w[ix]; six = scnt_get_ix(ix); ERTS_TW_ASSERT(!p || six == scnt_six); @@ -1336,35 +1358,30 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos) NULL, &scnt_six, tiw->scnt); } - if (tiw->yield_slot >= 0) { - ErtsTWheelTimer *p = tiw->sentinel.next; + if (tiw->yield_slot != ERTS_TW_SLOT_INACTIVE) { + p = tiw->sentinel.next; ix = tiw->yield_slot; while (p != &tiw->sentinel) { ErtsMonotonicTime tpos = p->timeout_pos; - scnt_inc(scnt, ix); - if (ix >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) { - later_tmo++; - ERTS_TW_ASSERT(ix == later_slot(tpos)); - } + ERTS_TW_ASSERT(ix == p->slot); + if (ix == ERTS_TW_SLOT_AT_ONCE) + at_once_tmo++; else { - soon_tmo++; - ERTS_TW_ASSERT(ix == (tpos & ERTS_TW_SOON_WHEEL_MASK)); - ERTS_TW_ASSERT(tpos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); + scnt_inc(scnt, ix); + if (ix >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) { + later_tmo++; + ERTS_TW_ASSERT(ix == later_slot(tpos)); + } + else { + soon_tmo++; + ERTS_TW_ASSERT(ix == (tpos & ERTS_TW_SOON_WHEEL_MASK)); + ERTS_TW_ASSERT(tpos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); + } + p = p->next; } - p = p->next; } } - at_once_tmo = 0; - if (tiw->at_once.head) { - ErtsTWheelTimer *p = tiw->at_once.head; - while (p) { - ErtsMonotonicTime tpos = p->timeout_pos; - at_once_tmo++; - ERTS_TW_ASSERT(tpos <= tiw->pos); - p = p->next; - } - } ERTS_TW_ASSERT(tiw->at_once.nto == at_once_tmo); ERTS_TW_ASSERT(tiw->soon.nto == soon_tmo); -- cgit v1.2.3 From cfd009221d53d2f19dfa2852962f48b85b34d485 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Sun, 12 Feb 2017 23:25:39 +0100 Subject: Minimum timeout position in each timer wheel Minimum known timeout position is saved in bot far and near wheel. This information is used to avoid scanning from current position in the cases were we know the minimum timeout position. --- erts/emulator/beam/time.c | 499 +++++++++++++++++++++++++++++----------------- 1 file changed, 315 insertions(+), 184 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 16e4328a05..f530ee5de0 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -141,20 +141,24 @@ * * In order to limit the amount of work needed in order to find * next timeout, we keep track of total amount of timers in the - * wheels, total amount of timers in soon wheel, and the total - * amount of timers in each range of slots. Each slot range - * currently contain 512 slots. + * wheels, total amount of timers in the later wheel, total amount + * of timers in soon wheel, and the total amount of timers in + * each range of slots. Each slot range currently contain 512 + * slots. * - * When next timeout is less than half the soon wheel width away - * we determine the exact timeout. Due to the timer counts of + * When next timeout is less than the soon wheel width away we + * determine the exact timeout. Due to the timer counts of * slot ranges, we currently at most need to search 1024 slots * in the soon wheel. This besides inspecting slot range counts * and two slots in the later wheel which potentially might trigger * timeouts for moving timers from the later wheel to the soon wheel - * earlier than timeouts in the soon wheel. + * earlier than timeouts in the soon wheel. We also keep track + * of latest known minimum timeout position in each wheel which + * makes it possible to avoid scanning from current position + * each time. * - * When next timeout is further away than half the soon wheel - * width we settle for the earliest possible timeout in the first + * When next timeout is further away than the soon wheel width + * we settle for the earliest possible timeout in the first * non-empty slot range. The further away the next timeout is, the * more likely it is that the next timeout change before we * actually get there. That is, a change due to another timer is @@ -176,8 +180,11 @@ #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" -#define ERTS_MONOTONIC_WEEK ERTS_SEC_TO_MONOTONIC(7*60*60*24) -#define ERTS_CLKTCKS_WEEK ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_WEEK) +#define ERTS_MAX_CLKTCKS \ + ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_TIME_MAX) + +#define ERTS_CLKTCKS_WEEK \ + ERTS_MONOTONIC_TO_CLKTCKS(ERTS_SEC_TO_MONOTONIC(7*60*60*24)) #ifdef ERTS_ENABLE_LOCK_CHECK #define ASSERT_NO_LOCKED_LOCKS erts_lc_check_exact(NULL, 0) @@ -265,10 +272,21 @@ static int tiw_itime; /* Constant after init */ # define TIW_ITIME tiw_itime #endif +const int etp_tw_soon_wheel_size = ERTS_TW_SOON_WHEEL_SIZE; +const ErtsMonotonicTime etp_tw_soon_wheel_mask = ERTS_TW_SOON_WHEEL_MASK; +const int etp_tw_soon_wheel_first_slot = ERTS_TW_SOON_WHEEL_FIRST_SLOT; + +const int etp_tw_later_wheel_size = ERTS_TW_LATER_WHEEL_SIZE; +const ErtsMonotonicTime etp_tw_later_wheel_slot_size = ERTS_TW_LATER_WHEEL_SLOT_SIZE; +const int etp_tw_later_wheel_shift = ERTS_TW_LATER_WHEEL_SHIFT; +const ErtsMonotonicTime etp_tw_later_wheel_mask = ERTS_TW_LATER_WHEEL_MASK; +const ErtsMonotonicTime etp_tw_later_wheel_pos_mask = ERTS_TW_LATER_WHEEL_POS_MASK; +const int etp_tw_later_wheel_first_slot = ERTS_TW_LATER_WHEEL_FIRST_SLOT; + struct ErtsTimerWheel_ { ErtsTWheelTimer *slots[1 /* At Once Slot */ + ERTS_TW_SOON_WHEEL_SIZE /* Soon Wheel Slots */ - + ERTS_TW_LATER_WHEEL_SIZE]; /* Later Wheel Slots */ + + ERTS_TW_LATER_WHEEL_SIZE]; /* Later Wheel Slots */ ErtsTWheelTimer **w; Sint scnt[ERTS_TW_SCNT_SIZE]; Sint bump_scnt[ERTS_TW_SCNT_SIZE]; @@ -278,15 +296,20 @@ struct ErtsTimerWheel_ { Uint nto; } at_once; struct { + ErtsMonotonicTime min_tpos; Uint nto; } soon; struct { + ErtsMonotonicTime min_tpos; + int min_tpos_slot; ErtsMonotonicTime pos; + Uint nto; } later; int yield_slot; int yield_slots_left; ErtsTWheelTimer sentinel; int true_next_timeout_time; + ErtsMonotonicTime next_timeout_pos; ErtsMonotonicTime next_timeout_time; }; @@ -297,6 +320,18 @@ struct ErtsTimerWheel_ { static int bump_later_wheel(ErtsTimerWheel *tiw, int *yield_count_p); +#ifdef ERTS_TW_DEBUG +#define ERTS_TW_DBG_VERIFY_EMPTY_SOON_SLOTS(TIW, TO_POS) \ + dbg_verify_empty_soon_slots((TIW), (TO_POS)) +#define ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(TIW, TO_POS) \ + dbg_verify_empty_later_slots((TIW), (TO_POS)) +void dbg_verify_empty_soon_slots(ErtsTimerWheel *, ErtsMonotonicTime); +void dbg_verify_empty_later_slots(ErtsTimerWheel *, ErtsMonotonicTime); +#else +#define ERTS_TW_DBG_VERIFY_EMPTY_SOON_SLOTS(TIW, TO_POS) +#define ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(TIW, TO_POS) +#endif + static ERTS_INLINE int scnt_get_ix(int slot) { @@ -434,47 +469,59 @@ static void hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos); #endif static ErtsMonotonicTime -find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw, int thorough) +find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw) { int slot, slots; int true_min_timeout = 0; - ErtsMonotonicTime min_timeout, min_timeout_pos, slot_timeout_pos; + ErtsMonotonicTime min_timeout_pos; + + ERTS_TW_ASSERT(tiw->pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE < tiw->later.pos + && tiw->later.pos <= tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); ERTS_TW_ASSERT(tiw->yield_slot == ERTS_TW_SLOT_INACTIVE); if (tiw->nto == 0) { /* no timeouts in wheel */ - if (!thorough) - min_timeout_pos = tiw->pos; - else { - ErtsMonotonicTime curr_time = erts_get_monotonic_time(esdp); - tiw->pos = min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); - } - min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_WEEK); + ErtsMonotonicTime curr_time = erts_get_monotonic_time(esdp); + tiw->pos = min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); + tiw->later.pos = min_timeout_pos + ERTS_TW_SOON_WHEEL_SIZE; + tiw->later.pos &= ERTS_TW_LATER_WHEEL_POS_MASK; + min_timeout_pos += ERTS_CLKTCKS_WEEK; goto done; } - min_timeout_pos = tiw->pos; - if (thorough) - min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_WEEK); - else - min_timeout_pos += ERTS_TW_SOON_WHEEL_SIZE/2; + ERTS_TW_ASSERT(tiw->soon.nto || tiw->later.nto); if (!tiw->soon.nto) { - ErtsMonotonicTime tmp; - /* Select later wheel... */ - slot_timeout_pos = tiw->later.pos; - slot = later_slot(slot_timeout_pos); - /* Pre-timeout for move from later to soon wheel... */ - slot_timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; - tmp = min_timeout_pos - slot_timeout_pos; - tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; - tmp++; - if (tmp > ERTS_TW_LATER_WHEEL_SIZE) + ErtsMonotonicTime tpos, min_tpos; + + /* Search later wheel... */ + + min_tpos = tiw->later.min_tpos & ERTS_TW_LATER_WHEEL_POS_MASK; + + if (min_tpos <= tiw->later.pos) { + tpos = tiw->later.pos; slots = ERTS_TW_LATER_WHEEL_SIZE; - else - slots = (int) tmp; + } + else { + ErtsMonotonicTime tmp; + /* Don't inspect slots we know are empty... */ + tmp = min_tpos - tiw->later.pos; + tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + if (tmp >= ERTS_TW_LATER_WHEEL_SIZE) { + /* Timeout more than one revolution ahead... */ + + /* Pre-timeout for move from later to soon wheel... */ + min_timeout_pos = min_tpos - ERTS_TW_LATER_WHEEL_SLOT_SIZE; + goto done; + } + tpos = min_tpos; + ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, min_tpos); + slots = ERTS_TW_LATER_WHEEL_SIZE - ((int) tmp); + } + + slot = later_slot(tpos); /* * We never search for an exact timeout in the @@ -484,13 +531,21 @@ find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw, int thorough) if (tiw->w[slot]) true_min_timeout = 1; else - scnt_later_wheel_next(&slot, &slots, &slot_timeout_pos, - NULL, tiw->scnt); - min_timeout_pos = slot_timeout_pos; + scnt_later_wheel_next(&slot, &slots, &tpos, NULL, tiw->scnt); + + tiw->later.min_tpos = tpos; + tiw->later.min_tpos_slot = slot; + ERTS_TW_ASSERT(slot == later_slot(tpos)); + + /* Pre-timeout for move from later to soon wheel... */ + tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + min_timeout_pos = tpos; } else { - ErtsMonotonicTime tmp; - /* Select soon wheel... */ + ErtsMonotonicTime tpos; + /* Search soon wheel... */ + + min_timeout_pos = tiw->pos + ERTS_TW_SOON_WHEEL_SIZE; /* * Besides inspecting the soon wheel we @@ -498,64 +553,72 @@ find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw, int thorough) * later wheel which potentially can trigger * timeouts before timeouts in soon wheel... */ - slot_timeout_pos = tiw->later.pos; - slot_timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; - if (slot_timeout_pos < min_timeout_pos) { - int fslot = later_slot(tiw->later.pos); - if (tiw->w[fslot]) { - min_timeout_pos = slot_timeout_pos; - true_min_timeout = 1; - } + if (tiw->later.min_tpos > (tiw->later.pos + + 2*ERTS_TW_LATER_WHEEL_SLOT_SIZE)) { + ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS( + tiw, 2*ERTS_TW_LATER_WHEEL_SLOT_SIZE); + } + else { + int fslot; + tpos = tiw->later.pos; + tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + fslot = later_slot(tiw->later.pos); + if (tiw->w[fslot]) + min_timeout_pos = tpos; else { - slot_timeout_pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE; - if (slot_timeout_pos < min_timeout_pos) { + tpos += ERTS_TW_LATER_WHEEL_SLOT_SIZE; + if (tpos < min_timeout_pos) { fslot++; if (fslot == ERTS_TW_LATER_WHEEL_END_SLOT) fslot = ERTS_TW_LATER_WHEEL_FIRST_SLOT; - if (tiw->w[fslot]) { - min_timeout_pos = slot_timeout_pos; - true_min_timeout = 1; - } + if (tiw->w[fslot]) + min_timeout_pos = tpos; } } } - slot_timeout_pos = tiw->pos; - slot = soon_slot(tiw->pos); - tmp = min_timeout_pos - slot_timeout_pos; - if (tmp > ERTS_TW_SOON_WHEEL_SIZE) + if (tiw->soon.min_tpos <= tiw->pos) { + tpos = tiw->pos; slots = ERTS_TW_SOON_WHEEL_SIZE; - else - slots = (int) tmp; + } + else { + ErtsMonotonicTime tmp; + /* Don't inspect slots we know are empty... */ + tmp = tiw->soon.min_tpos - tiw->pos; + ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_SIZE > tmp); + ERTS_TW_DBG_VERIFY_EMPTY_SOON_SLOTS(tiw, tiw->soon.min_tpos); + slots = ERTS_TW_SOON_WHEEL_SIZE - ((int) tmp); + tpos = tiw->soon.min_tpos; + } - while (slot_timeout_pos < min_timeout_pos) { + slot = soon_slot(tpos); + + /* find next non-empty slot */ + while (tpos < min_timeout_pos) { if (tiw->w[slot]) { - ERTS_TW_ASSERT(tiw->w[slot]->timeout_pos == slot_timeout_pos); - min_timeout_pos = slot_timeout_pos; - true_min_timeout = 1; + ERTS_TW_ASSERT(tiw->w[slot]->timeout_pos == tpos); + min_timeout_pos = tpos; break; } - scnt_soon_wheel_next(&slot, &slots, &slot_timeout_pos, - NULL, tiw->scnt); + scnt_soon_wheel_next(&slot, &slots, &tpos, NULL, tiw->scnt); } - if (slots > 0 && !true_min_timeout) { - /* Find first non-empty range... */ - scnt_soon_wheel_next(&slot, &slots, &slot_timeout_pos, - NULL, tiw->scnt); - min_timeout_pos = slot_timeout_pos; - } + tiw->soon.min_tpos = min_timeout_pos; + true_min_timeout = 1; } -done: +done: { + ErtsMonotonicTime min_timeout; - min_timeout = ERTS_CLKTCKS_TO_MONOTONIC(min_timeout_pos); - tiw->next_timeout_time = min_timeout; - tiw->true_next_timeout_time = true_min_timeout; + min_timeout = ERTS_CLKTCKS_TO_MONOTONIC(min_timeout_pos); + tiw->next_timeout_pos = min_timeout_pos; + tiw->next_timeout_time = min_timeout; + tiw->true_next_timeout_time = true_min_timeout; - ERTS_HARD_DBG_CHK_WHEELS(tiw, 1); + ERTS_HARD_DBG_CHK_WHEELS(tiw, 1); - return min_timeout; + return min_timeout; + } } static ERTS_INLINE void @@ -581,9 +644,20 @@ insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p) if (slot == ERTS_TW_SLOT_AT_ONCE) tiw->at_once.nto++; else { + ErtsMonotonicTime tpos = p->timeout_pos; if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) { ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); tiw->soon.nto++; + if (tiw->soon.min_tpos > tpos) + tiw->soon.min_tpos = tpos; + } + else { + ERTS_TW_ASSERT(p->timeout_pos >= tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); + tiw->later.nto++; + if (tiw->later.min_tpos > tpos) { + tiw->later.min_tpos = tpos; + tiw->later.min_tpos_slot = slot; + } } scnt_inc(tiw->scnt, slot); } @@ -593,6 +667,7 @@ static ERTS_INLINE void remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) { int slot = p->slot; + int empty_slot; ERTS_TW_ASSERT(slot != ERTS_TW_SLOT_INACTIVE); /* @@ -608,21 +683,45 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) /* Cannot be referred by sentinel, i.e. must be referred by slot... */ ERTS_TW_ASSERT(tiw->w[slot] == p); tiw->w[slot] = NULL; + empty_slot = 1; } else { if (tiw->w[slot] == p) tiw->w[slot] = p->next; p->prev->next = p->next; p->next->prev = p->prev; + empty_slot = 0; } if (slot == ERTS_TW_SLOT_AT_ONCE) { ERTS_TW_ASSERT(tiw->at_once.nto > 0); tiw->at_once.nto--; } else { - if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) - tiw->soon.nto--; scnt_dec(tiw->scnt, slot); + if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) { + if (empty_slot + && tiw->true_next_timeout_time + && p->timeout_pos == tiw->next_timeout_pos) { + tiw->true_next_timeout_time = 0; + } + if (--tiw->soon.nto == 0) + tiw->soon.min_tpos = ERTS_MAX_CLKTCKS; + } + else { + if (empty_slot + && tiw->true_next_timeout_time + && tiw->later.min_tpos_slot == slot) { + ErtsMonotonicTime tpos = tiw->later.min_tpos; + tpos &= ERTS_TW_LATER_WHEEL_POS_MASK; + tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + if (tpos == tiw->next_timeout_pos) + tiw->true_next_timeout_time = 0; + } + if (--tiw->later.nto == 0) { + tiw->later.min_tpos = ERTS_MAX_CLKTCKS; + tiw->later.min_tpos_slot = ERTS_TW_LATER_WHEEL_END_SLOT; + } + } } p->slot = ERTS_TW_SLOT_INACTIVE; } @@ -633,82 +732,18 @@ erts_check_next_timeout_time(ErtsSchedulerData *esdp) ErtsTimerWheel *tiw = esdp->timer_wheel; ErtsMonotonicTime time; ERTS_MSACC_DECLARE_CACHE_X(); + ERTS_TW_ASSERT(tiw->next_timeout_time + == ERTS_CLKTCKS_TO_MONOTONIC(tiw->next_timeout_pos)); if (tiw->true_next_timeout_time) - return tiw->next_timeout_time; + return tiw->next_timeout_time; /* known timeout... */ + if (tiw->next_timeout_pos > tiw->pos + ERTS_TW_SOON_WHEEL_SIZE) + return tiw->next_timeout_time; /* sufficiently later away... */ ERTS_MSACC_PUSH_AND_SET_STATE_CACHED_X(ERTS_MSACC_STATE_TIMERS); - time = find_next_timeout(esdp, tiw, 1); + time = find_next_timeout(esdp, tiw); ERTS_MSACC_POP_STATE_M_X(); return time; } -#ifndef ERTS_TW_DEBUG -#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) ((void) 0) -#else -#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) debug_check_safe_to_skip_to((TIW), (TO)) -static void -debug_check_safe_to_skip_to(ErtsTimerWheel *tiw, ErtsMonotonicTime skip_to_pos) -{ - int slots, ix; - ErtsTWheelTimer *tmr; - ErtsMonotonicTime tmp; - - ix = soon_slot(tiw->pos); - tmp = skip_to_pos - tiw->pos; - ERTS_TW_ASSERT(tmp >= 0); - if (tmp < (ErtsMonotonicTime) ERTS_TW_SOON_WHEEL_SIZE) - slots = (int) tmp; - else - slots = ERTS_TW_SOON_WHEEL_SIZE; - - while (slots > 0) { - tmr = tiw->w[ix]; - if (tmr) { - ErtsTWheelTimer *end = tmr; - do { - ERTS_TW_ASSERT(tmr->timeout_pos > skip_to_pos); - tmr = tmr->next; - } while (tmr != end); - } - ix++; - if (ix == ERTS_TW_SOON_WHEEL_END_SLOT) - ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; - slots--; - } - - ix = later_slot(tiw->later.pos); - tmp = skip_to_pos; - tmp &= ERTS_TW_LATER_WHEEL_POS_MASK; - if (tmp >= tiw->later.pos) { - tmp -= tiw->later.pos; - tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; - tmp++; - ERTS_TW_ASSERT(tmp >= 0); - if (tmp < (ErtsMonotonicTime) ERTS_TW_LATER_WHEEL_SIZE) - slots = (int) tmp; - else - slots = ERTS_TW_LATER_WHEEL_SIZE; - - while (slots > 0) { - tmr = tiw->w[ix]; - if (tmr) { - ErtsMonotonicTime tpos = tmr->timeout_pos; - ErtsTWheelTimer *end = tmr; - do { - tpos &= ERTS_TW_LATER_WHEEL_POS_MASK; - tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; - ERTS_TW_ASSERT(tpos > skip_to_pos); - tmr = tmr->next; - } while (tmr != end); - } - ix++; - if (ix == ERTS_TW_LATER_WHEEL_END_SLOT) - ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT; - slots--; - } - } -} -#endif - static ERTS_INLINE void timeout_timer(ErtsTWheelTimer *p) { @@ -758,16 +793,23 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) restarted = 0; bump_to = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); + tiw->true_next_timeout_time = 1; + tiw->next_timeout_pos = bump_to; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to); while (1) { ErtsTWheelTimer *p; if (tiw->nto == 0) { empty_wheel: - ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, bump_to); + ERTS_TW_DBG_VERIFY_EMPTY_SOON_SLOTS(tiw, bump_to); + ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, bump_to); tiw->true_next_timeout_time = 0; - tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_WEEK; + tiw->next_timeout_pos = bump_to + ERTS_CLKTCKS_WEEK; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(tiw->next_timeout_pos);; tiw->pos = bump_to; + tiw->later.pos = bump_to + ERTS_TW_SOON_WHEEL_SIZE; + tiw->later.pos &= ERTS_TW_LATER_WHEEL_POS_MASK; tiw->yield_slot = ERTS_TW_SLOT_INACTIVE; ERTS_MSACC_POP_STATE_M_X(); return; @@ -811,8 +853,6 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) ERTS_TW_ASSERT(tiw->nto > 0); ERTS_TW_ASSERT(tiw->at_once.nto > 0); tiw->yield_slot = ERTS_TW_SLOT_AT_ONCE; - tiw->true_next_timeout_time = 1; - tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to); ERTS_MSACC_POP_STATE_M_X(); return; /* Yield! */ } @@ -847,28 +887,22 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) sys_memcpy((void *) bump_scnt, (void *) scnt, sizeof(Sint) * ERTS_TW_SCNT_SIZE); - if (tiw->true_next_timeout_time) { - ErtsMonotonicTime skip_until_pos; + if (tiw->soon.min_tpos > tiw->pos) { + ErtsMonotonicTime skip_until_pos = tiw->soon.min_tpos; + /* * No need inspecting slots where we know no timeouts * to trigger should reside. */ - skip_until_pos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time); if (skip_until_pos > bump_to) skip_until_pos = bump_to; skip_until_pos--; if (skip_until_pos > tiw->pos) { - ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, skip_until_pos); - + ERTS_TW_DBG_VERIFY_EMPTY_SOON_SLOTS(tiw, skip_until_pos); tiw->pos = skip_until_pos; - - skip_until_pos++; - skip_until_pos &= ERTS_TW_LATER_WHEEL_POS_MASK; - if (tiw->later.pos < skip_until_pos) - tiw->later.pos = skip_until_pos; } } @@ -884,6 +918,9 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) slot = soon_slot(tiw->pos+1); tiw->pos = bump_to; + tiw->next_timeout_pos = bump_to; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to); + scnt_ix = scnt_get_ix(slot); /* Timeout timers in soon wheel */ @@ -910,7 +947,8 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT <= p->slot && p->slot < ERTS_TW_SOON_WHEEL_END_SLOT); - tiw->soon.nto--; + if (--tiw->soon.nto == 0) + tiw->soon.min_tpos = ERTS_MAX_CLKTCKS; scnt_ix_dec(scnt, scnt_ix); if (p->timeout_pos <= bump_to) { timeout_timer(p); @@ -933,8 +971,6 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) } if (yield_count <= 0) { - tiw->true_next_timeout_time = 1; - tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to); tiw->yield_slot = slot; tiw->yield_slots_left = slots; ERTS_MSACC_POP_STATE_M_X(); @@ -959,9 +995,9 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) } while (restarted); tiw->true_next_timeout_time = 0; - tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_WEEK; + ERTS_TW_ASSERT(tiw->next_timeout_pos == bump_to); - (void) find_next_timeout(NULL, tiw, 0); + (void) find_next_timeout(NULL, tiw); ERTS_MSACC_POP_STATE_M_X(); } @@ -988,15 +1024,31 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) goto restart_yielded_slot; } else { - ErtsMonotonicTime tmp_slots = cpos; - tmp_slots += ERTS_TW_LATER_WHEEL_SLOT_SIZE; + ErtsMonotonicTime end_later_pos, tmp_slots, min_tpos; + + min_tpos = tiw->later.min_tpos & ERTS_TW_LATER_WHEEL_POS_MASK; + end_later_pos = cpos + ERTS_TW_SOON_WHEEL_SIZE; + end_later_pos &= ERTS_TW_LATER_WHEEL_POS_MASK; + + /* Skip known empty slots... */ + if (min_tpos > later_pos) { + if (min_tpos > end_later_pos) { + later_pos = end_later_pos; + ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, later_pos); + goto done; + } + later_pos = min_tpos; + ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, later_pos); + } + + tmp_slots = end_later_pos; tmp_slots -= later_pos; tmp_slots /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; - tmp_slots++; if (tmp_slots < ERTS_TW_LATER_WHEEL_SIZE) slots = (int) tmp_slots; else slots = ERTS_TW_LATER_WHEEL_SIZE; + fslot = later_slot(later_pos); scnt_ix = scnt_get_ix(fslot); } @@ -1028,6 +1080,10 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) ERTS_TW_ASSERT(tpos >= later_pos); ERTS_TW_ASSERT(p->slot == fslot); + if (--tiw->later.nto == 0) { + tiw->later.min_tpos = ERTS_MAX_CLKTCKS; + tiw->later.min_tpos_slot = ERTS_TW_LATER_WHEEL_END_SLOT; + } scnt_ix_dec(scnt, scnt_ix); if (tpos >= later_pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE) { @@ -1061,8 +1117,6 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) if (ycount < 0) { tiw->later.pos = later_pos; - tiw->true_next_timeout_time = 1; - tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(cpos); tiw->yield_slot = fslot; tiw->yield_slots_left = slots; *ycount_p = 0; @@ -1078,6 +1132,8 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) scnt_later_wheel_next(&fslot, &slots, &later_pos, &scnt_ix, bump_scnt); } +done: + ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); tiw->later.pos = later_pos; @@ -1129,13 +1185,17 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp) tiw->pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime); tiw->nto = 0; tiw->at_once.nto = 0; + tiw->soon.min_tpos = ERTS_MAX_CLKTCKS; tiw->soon.nto = 0; - tiw->later.pos = tiw->pos; + tiw->later.min_tpos = ERTS_MAX_CLKTCKS; + tiw->later.min_tpos_slot = ERTS_TW_LATER_WHEEL_END_SLOT; + tiw->later.pos = tiw->pos + ERTS_TW_SOON_WHEEL_SIZE; tiw->later.pos &= ERTS_TW_LATER_WHEEL_POS_MASK; - tiw->later.pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE; + tiw->later.nto = 0; tiw->yield_slot = ERTS_TW_SLOT_INACTIVE; tiw->true_next_timeout_time = 0; - tiw->next_timeout_time = mtime + ERTS_MONOTONIC_WEEK; + tiw->next_timeout_pos = tiw->pos + ERTS_CLKTCKS_WEEK; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(tiw->next_timeout_pos); tiw->sentinel.next = &tiw->sentinel; tiw->sentinel.prev = &tiw->sentinel; tiw->sentinel.timeout = NULL; @@ -1175,7 +1235,6 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw, void *arg, ErtsMonotonicTime timeout_pos) { int slot; - ErtsMonotonicTime timeout_time; ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); p->timeout = timeout; @@ -1195,6 +1254,8 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw, /* soon wheel */ p->timeout_pos = timeout_pos; slot = soon_slot(timeout_pos); + if (tiw->soon.min_tpos > timeout_pos) + tiw->soon.min_tpos = timeout_pos; } else { /* later wheel */ @@ -1214,11 +1275,13 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw, } insert_timer_into_slot(tiw, slot, p); - timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos); - if (timeout_time < tiw->next_timeout_time) { + if (timeout_pos <= tiw->next_timeout_pos) { tiw->true_next_timeout_time = 1; - tiw->next_timeout_time = timeout_time; + if (timeout_pos < tiw->next_timeout_pos) { + tiw->next_timeout_pos = timeout_pos; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos); + } } ERTS_MSACC_POP_STATE_M_X(); } @@ -1264,6 +1327,73 @@ erts_twheel_debug_foreach(ErtsTimerWheel *tiw, } } +#ifdef ERTS_TW_DEBUG + +void +dbg_verify_empty_soon_slots(ErtsTimerWheel *tiw, ErtsMonotonicTime to_pos) +{ + int ix; + ErtsMonotonicTime tmp; + + ix = soon_slot(tiw->pos); + tmp = to_pos; + if (tmp > tiw->pos) { + int slots; + tmp -= tiw->pos; + ERTS_TW_ASSERT(tmp > 0); + if (tmp < (ErtsMonotonicTime) ERTS_TW_SOON_WHEEL_SIZE) + slots = (int) tmp; + else + slots = ERTS_TW_SOON_WHEEL_SIZE; + + while (slots > 0) { + ERTS_TW_ASSERT(!tiw->w[ix]); + ix++; + if (ix == ERTS_TW_SOON_WHEEL_END_SLOT) + ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; + slots--; + } + } +} + +void +dbg_verify_empty_later_slots(ErtsTimerWheel *tiw, ErtsMonotonicTime to_pos) +{ + int ix; + ErtsMonotonicTime tmp; + + ix = later_slot(tiw->later.pos); + tmp = to_pos; + tmp &= ERTS_TW_LATER_WHEEL_POS_MASK; + if (tmp > tiw->later.pos) { + int slots; + tmp -= tiw->later.pos; + tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + ERTS_TW_ASSERT(tmp > 0); + if (tmp < (ErtsMonotonicTime) ERTS_TW_LATER_WHEEL_SIZE) + slots = (int) tmp; + else + slots = ERTS_TW_LATER_WHEEL_SIZE; + + while (slots > 0) { + ErtsTWheelTimer *tmr = tiw->w[ix]; + if (tmr) { + ErtsTWheelTimer *end = tmr; + do { + ERTS_TW_ASSERT(tmr->timeout_pos > to_pos); + tmr = tmr->next; + } while (tmr != end); + } + ix++; + if (ix == ERTS_TW_LATER_WHEEL_END_SLOT) + ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT; + slots--; + } + } +} + +#endif /* ERTS_TW_DEBUG */ + #ifdef ERTS_TW_HARD_DEBUG static void @@ -1385,10 +1515,11 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos) ERTS_TW_ASSERT(tiw->at_once.nto == at_once_tmo); ERTS_TW_ASSERT(tiw->soon.nto == soon_tmo); + ERTS_TW_ASSERT(tiw->later.nto == later_tmo); ERTS_TW_ASSERT(tiw->nto == soon_tmo + later_tmo + at_once_tmo); for (six = 0; six < ERTS_TW_SCNT_SIZE; six++) ERTS_TW_ASSERT(scnt[six] == tiw->scnt[six]); } -#endif +#endif /* ERTS_TW_HARD_DEBUG */ -- cgit v1.2.3 From 02da78a8cce79a07f00ff0816967e84ead8a3bfa Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 18 Apr 2017 16:37:44 +0200 Subject: Fix of later timer wheel --- erts/emulator/beam/time.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index f530ee5de0..cee3cb619f 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -1033,8 +1033,8 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) /* Skip known empty slots... */ if (min_tpos > later_pos) { if (min_tpos > end_later_pos) { - later_pos = end_later_pos; - ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, later_pos); + ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, end_later_pos); + tiw->later.pos = end_later_pos; goto done; } later_pos = min_tpos; @@ -1051,6 +1051,8 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) fslot = later_slot(later_pos); scnt_ix = scnt_get_ix(fslot); + + tiw->later.pos = end_later_pos; } while (slots > 0) { @@ -1077,7 +1079,6 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) while (1) { ErtsMonotonicTime tpos = p->timeout_pos; - ERTS_TW_ASSERT(tpos >= later_pos); ERTS_TW_ASSERT(p->slot == fslot); if (--tiw->later.nto == 0) { @@ -1086,7 +1087,7 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) } scnt_ix_dec(scnt, scnt_ix); - if (tpos >= later_pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE) { + if (tpos >= tiw->later.pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE) { /* keep in later slot; very uncommon... */ insert_timer_into_slot(tiw, fslot, p); ycount -= ERTS_TW_COST_SLOT_MOVE; @@ -1116,7 +1117,6 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) } if (ycount < 0) { - tiw->later.pos = later_pos; tiw->yield_slot = fslot; tiw->yield_slots_left = slots; *ycount_p = 0; @@ -1129,15 +1129,13 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) } } - scnt_later_wheel_next(&fslot, &slots, &later_pos, &scnt_ix, bump_scnt); + scnt_later_wheel_next(&fslot, &slots, NULL, &scnt_ix, bump_scnt); } done: ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); - tiw->later.pos = later_pos; - *ycount_p = ycount; return 0; -- cgit v1.2.3 From 0835f40ae25f97360dc393928796387d3cd6b16e Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Wed, 19 Apr 2017 00:43:37 +0100 Subject: erts: Add enif_phash2 and enif_phash2_ranged These allow one to hash VM terms from NIF code. --- erts/emulator/beam/bif.c | 17 ++++++----------- erts/emulator/beam/erl_nif.c | 20 ++++++++++++++++++++ erts/emulator/beam/erl_nif_api_funcs.h | 4 ++++ erts/emulator/beam/erl_utils.h | 1 + erts/emulator/beam/utils.c | 10 ++++++++++ 5 files changed, 41 insertions(+), 11 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 214de3652f..d59adc18d6 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -4888,7 +4888,6 @@ BIF_RETTYPE phash2_1(BIF_ALIST_1) BIF_RETTYPE phash2_2(BIF_ALIST_2) { Uint32 hash; - Uint32 final_hash; Uint32 range; /* Check for special case 2^32 */ @@ -4901,23 +4900,19 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2) } range = (Uint32) u; } - hash = make_hash2(BIF_ARG_1); - if (range) { - final_hash = hash % range; /* [0..range-1] */ - } else { - final_hash = hash; - } + hash = make_hash2_within_range(BIF_ARG_1, range); + /* * Return either a small or a big. Use the heap for bigs if there is room. */ #if defined(ARCH_64) - BIF_RET(make_small(final_hash)); + BIF_RET(make_small(hash)); #else - if (IS_USMALL(0, final_hash)) { - BIF_RET(make_small(final_hash)); + if (IS_USMALL(0, hash)) { + BIF_RET(make_small(hash)); } else { Eterm* hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE); - BIF_RET(uint_to_big(final_hash, hp)); + BIF_RET(uint_to_big(hash, hp)); } #endif } diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index f86b9739fa..14abf461a3 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -55,6 +55,7 @@ #include "dtrace-wrapper.h" #include "erl_process.h" #include "erl_bif_unique.h" +#include "erl_utils.h" #undef ERTS_WANT_NFUNC_SCHED_INTERNALS__ #define ERTS_WANT_NFUNC_SCHED_INTERNALS__ #include "erl_nfunc_sched.h" @@ -1213,6 +1214,25 @@ int enif_compare(Eterm lhs, Eterm rhs) return result; } +#if SIZEOF_LONG < 4 +/* This *really* shouldn't happen */ +# error Incompatible long word size +#endif + +unsigned long enif_phash2(Eterm term) +{ + return make_hash2(term) & ((1 << 27) - 1); +} + +unsigned long enif_phash2_ranged(Eterm term, unsigned long range) +{ +#if SIZEOF_LONG > 4 + if (range > (unsigned long) UINT32_MAX) + range = 0; +#endif + return make_hash2_within_range(term, range); +} + int enif_get_tuple(ErlNifEnv* env, Eterm tpl, int* arity, const Eterm** array) { Eterm* ptr; diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index 01d9e386ed..8a3f9165d9 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -47,6 +47,8 @@ ERL_NIF_API_FUNC_DECL(int,enif_get_list_cell,(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int* arity, const ERL_NIF_TERM** array)); ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); ERL_NIF_API_FUNC_DECL(int,enif_compare,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); +ERL_NIF_API_FUNC_DECL(unsigned long,enif_phash2,(ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(unsigned long,enif_phash2_ranged,(ERL_NIF_TERM term, unsigned long)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i)); @@ -208,6 +210,8 @@ ERL_NIF_API_FUNC_DECL(int, enif_compare_monitors,(const ErlNifMonitor*,const Erl # define enif_get_list_cell ERL_NIF_API_FUNC_MACRO(enif_get_list_cell) # define enif_is_identical ERL_NIF_API_FUNC_MACRO(enif_is_identical) # define enif_compare ERL_NIF_API_FUNC_MACRO(enif_compare) +# define enif_phash2 ERL_NIF_API_FUNC_MACRO(enif_phash2) +# define enif_phash2_ranged ERL_NIF_API_FUNC_MACRO(enif_phash2_ranged) # define enif_make_binary ERL_NIF_API_FUNC_MACRO(enif_make_binary) # define enif_make_badarg ERL_NIF_API_FUNC_MACRO(enif_make_badarg) diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index 47289a0af1..75578c26d6 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -119,6 +119,7 @@ Sint erts_list_length(Eterm); int erts_is_builtin(Eterm, Eterm, int); Uint32 block_hash(byte *, unsigned, Uint32); Uint32 make_hash2(Eterm); +Uint32 make_hash2_within_range(Eterm, Uint32); Uint32 make_hash(Eterm); Uint32 make_internal_hash(Eterm); diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 8f3f48f38f..72c59b33c6 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1552,6 +1552,16 @@ make_hash2(Eterm term) } } +Uint32 +make_hash2_within_range(Eterm term, Uint32 range) +{ + Uint32 hash = make_hash2(term); + if (range) + return hash % range; /* [0..range-1] */ + else + return hash; /* Special case: 2**32 */ +} + /* Term hash function for internal use. * * Limitation #1: Is not "portable" in any way between different VM instances. -- cgit v1.2.3 From f8622f94739bb4d240cc3965cc67b8096feab503 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Thu, 20 Apr 2017 14:32:36 +0200 Subject: erts: Initialize esdp->type in non-smp to normal The previous code worked through luck because the memory returned from erts_alloc was zero:ed and SCHED_NORMAL is 0. But if you run with +Meamin that is not always the case which is how this error was detected. --- erts/emulator/beam/erl_process.c | 1 + 1 file changed, 1 insertion(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 894e0ee582..c8e217d114 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -6073,6 +6073,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, runq->scheduler = esdp; esdp->run_queue = runq; esdp->no = (Uint) num; + esdp->type = ERTS_SCHED_NORMAL; #endif esdp->ssi = ssi; -- cgit v1.2.3 From 1b37d0b010ea31b04b9d0a15d0bec9c75a013dc9 Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Thu, 20 Apr 2017 21:35:54 +0100 Subject: erts: Remove enif_phash2_ranged --- erts/emulator/beam/bif.c | 17 +++++++++++------ erts/emulator/beam/erl_nif.c | 14 ++------------ erts/emulator/beam/erl_nif_api_funcs.h | 2 -- erts/emulator/beam/erl_utils.h | 1 - erts/emulator/beam/utils.c | 10 ---------- 5 files changed, 13 insertions(+), 31 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index d59adc18d6..214de3652f 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -4888,6 +4888,7 @@ BIF_RETTYPE phash2_1(BIF_ALIST_1) BIF_RETTYPE phash2_2(BIF_ALIST_2) { Uint32 hash; + Uint32 final_hash; Uint32 range; /* Check for special case 2^32 */ @@ -4900,19 +4901,23 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2) } range = (Uint32) u; } - hash = make_hash2_within_range(BIF_ARG_1, range); - + hash = make_hash2(BIF_ARG_1); + if (range) { + final_hash = hash % range; /* [0..range-1] */ + } else { + final_hash = hash; + } /* * Return either a small or a big. Use the heap for bigs if there is room. */ #if defined(ARCH_64) - BIF_RET(make_small(hash)); + BIF_RET(make_small(final_hash)); #else - if (IS_USMALL(0, hash)) { - BIF_RET(make_small(hash)); + if (IS_USMALL(0, final_hash)) { + BIF_RET(make_small(final_hash)); } else { Eterm* hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE); - BIF_RET(uint_to_big(hash, hp)); + BIF_RET(uint_to_big(final_hash, hp)); } #endif } diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 14abf461a3..f7777c8b92 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1214,25 +1214,15 @@ int enif_compare(Eterm lhs, Eterm rhs) return result; } +unsigned long enif_phash2(Eterm term) +{ #if SIZEOF_LONG < 4 /* This *really* shouldn't happen */ # error Incompatible long word size #endif - -unsigned long enif_phash2(Eterm term) -{ return make_hash2(term) & ((1 << 27) - 1); } -unsigned long enif_phash2_ranged(Eterm term, unsigned long range) -{ -#if SIZEOF_LONG > 4 - if (range > (unsigned long) UINT32_MAX) - range = 0; -#endif - return make_hash2_within_range(term, range); -} - int enif_get_tuple(ErlNifEnv* env, Eterm tpl, int* arity, const Eterm** array) { Eterm* ptr; diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index 8a3f9165d9..c096b18f91 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -48,7 +48,6 @@ ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int* ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); ERL_NIF_API_FUNC_DECL(int,enif_compare,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); ERL_NIF_API_FUNC_DECL(unsigned long,enif_phash2,(ERL_NIF_TERM term)); -ERL_NIF_API_FUNC_DECL(unsigned long,enif_phash2_ranged,(ERL_NIF_TERM term, unsigned long)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i)); @@ -211,7 +210,6 @@ ERL_NIF_API_FUNC_DECL(int, enif_compare_monitors,(const ErlNifMonitor*,const Erl # define enif_is_identical ERL_NIF_API_FUNC_MACRO(enif_is_identical) # define enif_compare ERL_NIF_API_FUNC_MACRO(enif_compare) # define enif_phash2 ERL_NIF_API_FUNC_MACRO(enif_phash2) -# define enif_phash2_ranged ERL_NIF_API_FUNC_MACRO(enif_phash2_ranged) # define enif_make_binary ERL_NIF_API_FUNC_MACRO(enif_make_binary) # define enif_make_badarg ERL_NIF_API_FUNC_MACRO(enif_make_badarg) diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index 75578c26d6..47289a0af1 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -119,7 +119,6 @@ Sint erts_list_length(Eterm); int erts_is_builtin(Eterm, Eterm, int); Uint32 block_hash(byte *, unsigned, Uint32); Uint32 make_hash2(Eterm); -Uint32 make_hash2_within_range(Eterm, Uint32); Uint32 make_hash(Eterm); Uint32 make_internal_hash(Eterm); diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 72c59b33c6..8f3f48f38f 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1552,16 +1552,6 @@ make_hash2(Eterm term) } } -Uint32 -make_hash2_within_range(Eterm term, Uint32 range) -{ - Uint32 hash = make_hash2(term); - if (range) - return hash % range; /* [0..range-1] */ - else - return hash; /* Special case: 2**32 */ -} - /* Term hash function for internal use. * * Limitation #1: Is not "portable" in any way between different VM instances. -- cgit v1.2.3 From d8756f8665a42effa2f3515369058cff4441abeb Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Thu, 20 Apr 2017 22:44:04 +0100 Subject: erts: Refactor enif_phash2 into enif_hash A more generic hashing function which can also hash terms based on `make_internal_hash'. --- erts/emulator/beam/erl_nif.c | 11 +++++++++-- erts/emulator/beam/erl_nif.h | 5 +++++ erts/emulator/beam/erl_nif_api_funcs.h | 4 ++-- 3 files changed, 16 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index f7777c8b92..3470cf3fd5 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1214,13 +1214,20 @@ int enif_compare(Eterm lhs, Eterm rhs) return result; } -unsigned long enif_phash2(Eterm term) +unsigned long enif_hash(ErlNifHash type, Eterm term) { #if SIZEOF_LONG < 4 /* This *really* shouldn't happen */ # error Incompatible long word size #endif - return make_hash2(term) & ((1 << 27) - 1); + switch (type) { + case ERL_NIF_INTERNAL_HASH: + return make_internal_hash(term); + case ERL_NIF_PHASH2: + return make_hash2(term) & ((1 << 27) - 1); + default: + return 0; + } } int enif_get_tuple(ErlNifEnv* env, Eterm tpl, int* arity, const Eterm** array) diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index ac45f3ac81..5a81f5fbbb 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -236,6 +236,11 @@ typedef enum { ERL_NIF_BIN2TERM_SAFE = 0x20000000 } ErlNifBinaryToTerm; +typedef enum { + ERL_NIF_INTERNAL_HASH = 1, + ERL_NIF_PHASH2 = 2 +} ErlNifHash; + /* * Return values from enif_thread_type(). Negative values * reserved for specific types of non-scheduler threads. diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index c096b18f91..f52c841ed7 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -47,7 +47,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_get_list_cell,(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int* arity, const ERL_NIF_TERM** array)); ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); ERL_NIF_API_FUNC_DECL(int,enif_compare,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); -ERL_NIF_API_FUNC_DECL(unsigned long,enif_phash2,(ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(unsigned long,enif_hash,(ErlNifHash type, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i)); @@ -209,7 +209,7 @@ ERL_NIF_API_FUNC_DECL(int, enif_compare_monitors,(const ErlNifMonitor*,const Erl # define enif_get_list_cell ERL_NIF_API_FUNC_MACRO(enif_get_list_cell) # define enif_is_identical ERL_NIF_API_FUNC_MACRO(enif_is_identical) # define enif_compare ERL_NIF_API_FUNC_MACRO(enif_compare) -# define enif_phash2 ERL_NIF_API_FUNC_MACRO(enif_phash2) +# define enif_hash ERL_NIF_API_FUNC_MACRO(enif_hash) # define enif_make_binary ERL_NIF_API_FUNC_MACRO(enif_make_binary) # define enif_make_badarg ERL_NIF_API_FUNC_MACRO(enif_make_badarg) -- cgit v1.2.3 From 21e1d879faaae2278a68200fe1085d7e73791fa0 Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Sat, 22 Apr 2017 17:22:32 +0100 Subject: erts: Support custom salt in enif_hash --- erts/emulator/beam/bif.c | 4 ++-- erts/emulator/beam/erl_bif_info.c | 2 +- erts/emulator/beam/erl_db_hash.c | 2 +- erts/emulator/beam/erl_map.h | 2 +- erts/emulator/beam/erl_nif.c | 6 ++--- erts/emulator/beam/erl_nif_api_funcs.h | 2 +- erts/emulator/beam/erl_process_dict.c | 2 +- erts/emulator/beam/erl_trace.c | 2 +- erts/emulator/beam/erl_utils.h | 4 ++-- erts/emulator/beam/utils.c | 40 +++++++++++++++++----------------- 10 files changed, 33 insertions(+), 33 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 214de3652f..7473a8e43f 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -4881,7 +4881,7 @@ BIF_RETTYPE phash2_1(BIF_ALIST_1) { Uint32 hash; - hash = make_hash2(BIF_ARG_1); + hash = make_hash2(BIF_ARG_1, 0); BIF_RET(make_small(hash & ((1L << 27) - 1))); } @@ -4901,7 +4901,7 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2) } range = (Uint32) u; } - hash = make_hash2(BIF_ARG_1); + hash = make_hash2(BIF_ARG_1, 0); if (range) { final_hash = hash % range; /* [0..range-1] */ } else { diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 3a8687dc59..f2863f5d5b 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -3950,7 +3950,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(erts_debug_reader_groups_map(BIF_P, (int) groups)); } else if (ERTS_IS_ATOM_STR("internal_hash", tp[1])) { - Uint hash = (Uint) make_internal_hash(tp[2]); + Uint hash = (Uint) make_internal_hash(tp[2], 0); Uint hsz = 0; Eterm* hp; erts_bld_uint(NULL, &hsz, hash); diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 7ab27df00c..07f6da7b6d 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -189,7 +189,7 @@ static ERTS_INLINE int add_fixed_deletion(DbTableHash* tb, int ix, /* optimised version of make_hash (normal case? atomic key) */ #define MAKE_HASH(term) \ ((is_atom(term) ? (atom_tab(atom_val(term))->slot.bucket.hvalue) : \ - make_internal_hash(term)) % MAX_HASH) + make_internal_hash(term, 0)) % MAX_HASH) #ifdef ERTS_SMP # define DB_HASH_LOCK_MASK (DB_HASH_LOCK_CNT-1) diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h index 61a841f7f0..f7d0413685 100644 --- a/erts/emulator/beam/erl_map.h +++ b/erts/emulator/beam/erl_map.h @@ -57,7 +57,7 @@ typedef struct flatmap_s { #define hashmap_size(x) (((hashmap_head_t*) hashmap_val(x))->size) -#define hashmap_make_hash(Key) make_internal_hash(Key) +#define hashmap_make_hash(Key) make_internal_hash(Key, 0) #define hashmap_restore_hash(Heap,Lvl,Key) \ (((Lvl) < 8) ? hashmap_make_hash(Key) >> (4*(Lvl)) : hashmap_make_hash(CONS(Heap, make_small((Lvl)>>3), (Key))) >> (4*((Lvl) & 7))) diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 3470cf3fd5..bdfb7f3b8e 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1214,7 +1214,7 @@ int enif_compare(Eterm lhs, Eterm rhs) return result; } -unsigned long enif_hash(ErlNifHash type, Eterm term) +unsigned long enif_hash(ErlNifHash type, Eterm term, unsigned long salt) { #if SIZEOF_LONG < 4 /* This *really* shouldn't happen */ @@ -1222,9 +1222,9 @@ unsigned long enif_hash(ErlNifHash type, Eterm term) #endif switch (type) { case ERL_NIF_INTERNAL_HASH: - return make_internal_hash(term); + return make_internal_hash(term, salt); case ERL_NIF_PHASH2: - return make_hash2(term) & ((1 << 27) - 1); + return make_hash2(term, salt) & ((1 << 27) - 1); default: return 0; } diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index f52c841ed7..6b9f78d46b 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -47,7 +47,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_get_list_cell,(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int* arity, const ERL_NIF_TERM** array)); ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); ERL_NIF_API_FUNC_DECL(int,enif_compare,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); -ERL_NIF_API_FUNC_DECL(unsigned long,enif_hash,(ErlNifHash type, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(unsigned long,enif_hash,(ErlNifHash type, ERL_NIF_TERM term, unsigned long salt)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i)); diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c index 7cfdf20341..01e240c65d 100644 --- a/erts/emulator/beam/erl_process_dict.c +++ b/erts/emulator/beam/erl_process_dict.c @@ -57,7 +57,7 @@ ((is_small(Term)) ? (Uint32) unsigned_val(Term) : \ ((is_atom(Term)) ? \ (Uint32) atom_val(Term) : \ - make_internal_hash(Term))) + make_internal_hash(Term, 0))) #define PD_SZ2BYTES(Sz) (sizeof(ProcDict) + ((Sz) - 1)*sizeof(Eterm)) diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 04f3160d42..8855449ca1 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -3230,7 +3230,7 @@ static int tracer_cmp_fun(void* a, void* b) static HashValue tracer_hash_fun(void* obj) { - return make_internal_hash(((ErtsTracerNif*)obj)->module); + return make_internal_hash(((ErtsTracerNif*)obj)->module, 0); } static void *tracer_alloc_fun(void* tmpl) diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index 47289a0af1..b38beaf93d 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -118,9 +118,9 @@ int erts_fit_in_bits_uint(Uint); Sint erts_list_length(Eterm); int erts_is_builtin(Eterm, Eterm, int); Uint32 block_hash(byte *, unsigned, Uint32); -Uint32 make_hash2(Eterm); +Uint32 make_hash2(Eterm, Uint32 salt); Uint32 make_hash(Eterm); -Uint32 make_internal_hash(Eterm); +Uint32 make_internal_hash(Eterm, Uint32 salt); void erts_save_emu_args(int argc, char **argv); Eterm erts_get_emu_args(struct process *c_p); diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 8f3f48f38f..2378c057dd 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1003,7 +1003,7 @@ tail_recur: break; } case MAP_DEF: - hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term); + hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term, 0); break; case TUPLE_DEF: { @@ -1109,7 +1109,7 @@ block_hash(byte *k, unsigned length, Uint32 initval) } Uint32 -make_hash2(Eterm term) +make_hash2(Eterm term, Uint32 salt) { Uint32 hash; Uint32 hash_xor_pairs; @@ -1179,7 +1179,7 @@ make_hash2(Eterm term) switch (term & _TAG_IMMED2_MASK) { case _TAG_IMMED2_ATOM: /* Fast, but the poor hash value should be mixed. */ - return atom_tab(atom_val(term))->slot.bucket.hvalue; + return atom_tab(atom_val(term))->slot.bucket.hvalue ^ salt; } break; case _TAG_IMMED1_SMALL: @@ -1190,7 +1190,7 @@ make_hash2(Eterm term) term = small_to_big(x, tmp_big); break; } - hash = 0; + hash = salt; SINT32_HASH(x, HCONST); return hash; } @@ -1201,7 +1201,7 @@ make_hash2(Eterm term) DECLARE_ESTACK(s); UseTmpHeapNoproc(2); - hash = 0; + hash = salt; for (;;) { switch (primary_tag(term)) { case TAG_PRIMARY_LIST: @@ -1277,7 +1277,7 @@ make_hash2(Eterm term) ESTACK_PUSH(s, hash_xor_pairs); ESTACK_PUSH(s, hash); ESTACK_PUSH(s, HASH_MAP_TAIL); - hash = 0; + hash = salt; hash_xor_pairs = 0; for (i = size - 1; i >= 0; i--) { ESTACK_PUSH(s, HASH_MAP_PAIR); @@ -1296,7 +1296,7 @@ make_hash2(Eterm term) ESTACK_PUSH(s, hash_xor_pairs); ESTACK_PUSH(s, hash); ESTACK_PUSH(s, HASH_MAP_TAIL); - hash = 0; + hash = salt; hash_xor_pairs = 0; } switch (hdr & _HEADER_MAP_SUBTAG_MASK) { @@ -1471,7 +1471,7 @@ make_hash2(Eterm term) break; default: - erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X, %lu)\n", term, salt); } } break; @@ -1488,21 +1488,21 @@ make_hash2(Eterm term) case _TAG_IMMED1_IMMED2: switch (term & _TAG_IMMED2_MASK) { case _TAG_IMMED2_ATOM: - if (hash == 0) + if (hash == salt) /* Fast, but the poor hash value should be mixed. */ - hash = atom_tab(atom_val(term))->slot.bucket.hvalue; + hash = atom_tab(atom_val(term))->slot.bucket.hvalue ^ salt; else UINT32_HASH(atom_tab(atom_val(term))->slot.bucket.hvalue, HCONST_3); goto hash2_common; case _TAG_IMMED2_NIL: - if (hash == 0) - hash = 3468870702UL; + if (hash == salt) + hash = 3468870702UL ^ salt; else UINT32_HASH(NIL_DEF, HCONST_2); goto hash2_common; default: - erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X, %lu)\n", term, salt); } case _TAG_IMMED1_SMALL: { @@ -1518,7 +1518,7 @@ make_hash2(Eterm term) } break; default: - erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X, %lu)\n", term, salt); hash2_common: /* Uint32 hash always has the hash value of the previous term, @@ -1542,7 +1542,7 @@ make_hash2(Eterm term) } case HASH_MAP_PAIR: hash_xor_pairs ^= hash; - hash = 0; + hash = salt; goto hash2_common; default: break; @@ -1578,7 +1578,7 @@ do { /* Lightweight mixing of constant (type info) */ \ } while (0) Uint32 -make_internal_hash(Eterm term) +make_internal_hash(Eterm term, Uint32 salt) { Uint32 hash; Uint32 hash_xor_pairs; @@ -1587,7 +1587,7 @@ make_internal_hash(Eterm term) /* Optimization. Simple cases before declaration of estack. */ if (primary_tag(term) == TAG_PRIMARY_IMMED1) { - hash = 0; + hash = salt; #if ERTS_SIZEOF_ETERM == 8 UINT32_HASH_2((Uint32)term, (Uint32)(term >> 32), HCONST); #elif ERTS_SIZEOF_ETERM == 4 @@ -1601,7 +1601,7 @@ make_internal_hash(Eterm term) Eterm tmp; DECLARE_ESTACK(s); - hash = 0; + hash = salt; for (;;) { switch (primary_tag(term)) { case TAG_PRIMARY_LIST: @@ -1894,7 +1894,7 @@ make_internal_hash(Eterm term) goto pop_next; } default: - erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_internal_hash(0x%X, %lu)\n", term, salt); } } break; @@ -1907,7 +1907,7 @@ make_internal_hash(Eterm term) goto pop_next; default: - erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_internal_hash(0x%X, %lu)\n", term, salt); pop_next: if (ESTACK_ISEMPTY(s)) { -- cgit v1.2.3 From da9abd24a93ae8fe174cdd38fc9699bbc45fdf56 Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Sat, 22 Apr 2017 20:00:28 +0100 Subject: erts: Allow for easier future enif_hash expansion Allow for expanding support to 64-bit hashes without breaking the interface. --- erts/emulator/beam/erl_nif.c | 10 +++------- erts/emulator/beam/erl_nif_api_funcs.h | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index bdfb7f3b8e..3ff6c8d3ed 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1214,17 +1214,13 @@ int enif_compare(Eterm lhs, Eterm rhs) return result; } -unsigned long enif_hash(ErlNifHash type, Eterm term, unsigned long salt) +ErlNifUInt64 enif_hash(ErlNifHash type, Eterm term, ErlNifUInt64 salt) { -#if SIZEOF_LONG < 4 -/* This *really* shouldn't happen */ -# error Incompatible long word size -#endif switch (type) { case ERL_NIF_INTERNAL_HASH: - return make_internal_hash(term, salt); + return make_internal_hash(term, (Uint32) salt); case ERL_NIF_PHASH2: - return make_hash2(term, salt) & ((1 << 27) - 1); + return make_hash2(term, (Uint32) salt) & ((1 << 27) - 1); default: return 0; } diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index 6b9f78d46b..1a91a04f32 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -47,7 +47,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_get_list_cell,(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int* arity, const ERL_NIF_TERM** array)); ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); ERL_NIF_API_FUNC_DECL(int,enif_compare,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); -ERL_NIF_API_FUNC_DECL(unsigned long,enif_hash,(ErlNifHash type, ERL_NIF_TERM term, unsigned long salt)); +ERL_NIF_API_FUNC_DECL(ErlNifUInt64,enif_hash,(ErlNifHash type, ERL_NIF_TERM term, ErlNifUInt64 salt)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i)); -- cgit v1.2.3 From a9d0d119837fb0bc52d2bb3d48a47568de9100b4 Mon Sep 17 00:00:00 2001 From: Dan Gudmundsson Date: Tue, 28 Feb 2017 15:23:11 +0100 Subject: Handle chardata in string:to_float and string:to_list --- erts/emulator/beam/bif.c | 4 ++-- erts/emulator/beam/bif.tab | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 214de3652f..075909a881 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -3109,7 +3109,7 @@ BIF_RETTYPE integer_to_list_1(BIF_ALIST_1) * On error returns: {error,not_a_list}, or {error, no_integer} */ -BIF_RETTYPE string_to_integer_1(BIF_ALIST_1) +BIF_RETTYPE string_list_to_integer_1(BIF_ALIST_1) { Eterm res; Eterm tail; @@ -3295,7 +3295,7 @@ BIF_RETTYPE float_to_binary_2(BIF_ALIST_2) #define LOAD_E(xi,xim,xl,xlm) ((xi)=(xim), (xl)=(xlm)) #define STRING_TO_FLOAT_BUF_INC_SZ (128) -BIF_RETTYPE string_to_float_1(BIF_ALIST_1) +BIF_RETTYPE string_list_to_float_1(BIF_ALIST_1) { Eterm orig = BIF_ARG_1; Eterm list = orig; diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 66e5dc2988..5aee795088 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -460,8 +460,8 @@ bif error_logger:warning_map/0 bif erlang:get_module_info/1 bif erlang:get_module_info/2 ubif erlang:is_boolean/1 -bif string:to_integer/1 -bif string:to_float/1 +bif string:list_to_integer/1 +bif string:list_to_float/1 bif erlang:make_fun/3 bif erlang:iolist_size/1 bif erlang:iolist_to_binary/1 -- cgit v1.2.3 From bf2e5951dbd6398b71e32fd1cfb72b2870280280 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 24 Apr 2017 17:18:53 +0200 Subject: Fix assert in dbg_verify_empty_later_slots() --- erts/emulator/beam/time.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index cee3cb619f..a7e5a64b22 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -1364,21 +1364,29 @@ dbg_verify_empty_later_slots(ErtsTimerWheel *tiw, ErtsMonotonicTime to_pos) tmp = to_pos; tmp &= ERTS_TW_LATER_WHEEL_POS_MASK; if (tmp > tiw->later.pos) { + ErtsMonotonicTime pos_min; int slots; tmp -= tiw->later.pos; tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; ERTS_TW_ASSERT(tmp > 0); + + pos_min = tiw->later.pos; + if (tmp < (ErtsMonotonicTime) ERTS_TW_LATER_WHEEL_SIZE) slots = (int) tmp; - else + else { + pos_min += ((tmp / ERTS_TW_LATER_WHEEL_SIZE) + * ERTS_TW_LATER_WHEEL_SLOT_SIZE); slots = ERTS_TW_LATER_WHEEL_SIZE; + } while (slots > 0) { ErtsTWheelTimer *tmr = tiw->w[ix]; + pos_min += ERTS_TW_LATER_WHEEL_SLOT_SIZE; if (tmr) { ErtsTWheelTimer *end = tmr; do { - ERTS_TW_ASSERT(tmr->timeout_pos > to_pos); + ERTS_TW_ASSERT(tmr->timeout_pos >= pos_min); tmr = tmr->next; } while (tmr != end); } -- cgit v1.2.3 From 3d6b8e7fb68543713f1620a45b8a590ef4ed88a5 Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Mon, 24 Apr 2017 22:51:49 +0100 Subject: erts: Discontinue salted use of enif_hash/phash2 --- erts/emulator/beam/bif.c | 4 ++-- erts/emulator/beam/erl_nif.c | 6 +++++- erts/emulator/beam/erl_utils.h | 2 +- erts/emulator/beam/utils.c | 30 +++++++++++++++--------------- 4 files changed, 23 insertions(+), 19 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 7473a8e43f..214de3652f 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -4881,7 +4881,7 @@ BIF_RETTYPE phash2_1(BIF_ALIST_1) { Uint32 hash; - hash = make_hash2(BIF_ARG_1, 0); + hash = make_hash2(BIF_ARG_1); BIF_RET(make_small(hash & ((1L << 27) - 1))); } @@ -4901,7 +4901,7 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2) } range = (Uint32) u; } - hash = make_hash2(BIF_ARG_1, 0); + hash = make_hash2(BIF_ARG_1); if (range) { final_hash = hash % range; /* [0..range-1] */ } else { diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 3ff6c8d3ed..e101747011 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1220,7 +1220,11 @@ ErlNifUInt64 enif_hash(ErlNifHash type, Eterm term, ErlNifUInt64 salt) case ERL_NIF_INTERNAL_HASH: return make_internal_hash(term, (Uint32) salt); case ERL_NIF_PHASH2: - return make_hash2(term, (Uint32) salt) & ((1 << 27) - 1); + /* It appears that make_hash2 doesn't always react to seasoning + * as well as it should. Therefore, let's make it ignore the salt + * value and declare salted uses of phash2 as unsupported. + */ + return make_hash2(term) & ((1 << 27) - 1); default: return 0; } diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index b38beaf93d..75d7e47239 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -118,7 +118,7 @@ int erts_fit_in_bits_uint(Uint); Sint erts_list_length(Eterm); int erts_is_builtin(Eterm, Eterm, int); Uint32 block_hash(byte *, unsigned, Uint32); -Uint32 make_hash2(Eterm, Uint32 salt); +Uint32 make_hash2(Eterm); Uint32 make_hash(Eterm); Uint32 make_internal_hash(Eterm, Uint32 salt); diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 2378c057dd..9d140470ac 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1003,7 +1003,7 @@ tail_recur: break; } case MAP_DEF: - hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term, 0); + hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term); break; case TUPLE_DEF: { @@ -1109,7 +1109,7 @@ block_hash(byte *k, unsigned length, Uint32 initval) } Uint32 -make_hash2(Eterm term, Uint32 salt) +make_hash2(Eterm term) { Uint32 hash; Uint32 hash_xor_pairs; @@ -1179,7 +1179,7 @@ make_hash2(Eterm term, Uint32 salt) switch (term & _TAG_IMMED2_MASK) { case _TAG_IMMED2_ATOM: /* Fast, but the poor hash value should be mixed. */ - return atom_tab(atom_val(term))->slot.bucket.hvalue ^ salt; + return atom_tab(atom_val(term))->slot.bucket.hvalue; } break; case _TAG_IMMED1_SMALL: @@ -1190,7 +1190,7 @@ make_hash2(Eterm term, Uint32 salt) term = small_to_big(x, tmp_big); break; } - hash = salt; + hash = 0; SINT32_HASH(x, HCONST); return hash; } @@ -1201,7 +1201,7 @@ make_hash2(Eterm term, Uint32 salt) DECLARE_ESTACK(s); UseTmpHeapNoproc(2); - hash = salt; + hash = 0; for (;;) { switch (primary_tag(term)) { case TAG_PRIMARY_LIST: @@ -1277,7 +1277,7 @@ make_hash2(Eterm term, Uint32 salt) ESTACK_PUSH(s, hash_xor_pairs); ESTACK_PUSH(s, hash); ESTACK_PUSH(s, HASH_MAP_TAIL); - hash = salt; + hash = 0; hash_xor_pairs = 0; for (i = size - 1; i >= 0; i--) { ESTACK_PUSH(s, HASH_MAP_PAIR); @@ -1296,7 +1296,7 @@ make_hash2(Eterm term, Uint32 salt) ESTACK_PUSH(s, hash_xor_pairs); ESTACK_PUSH(s, hash); ESTACK_PUSH(s, HASH_MAP_TAIL); - hash = salt; + hash = 0; hash_xor_pairs = 0; } switch (hdr & _HEADER_MAP_SUBTAG_MASK) { @@ -1471,7 +1471,7 @@ make_hash2(Eterm term, Uint32 salt) break; default: - erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X, %lu)\n", term, salt); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); } } break; @@ -1488,21 +1488,21 @@ make_hash2(Eterm term, Uint32 salt) case _TAG_IMMED1_IMMED2: switch (term & _TAG_IMMED2_MASK) { case _TAG_IMMED2_ATOM: - if (hash == salt) + if (hash == 0) /* Fast, but the poor hash value should be mixed. */ - hash = atom_tab(atom_val(term))->slot.bucket.hvalue ^ salt; + hash = atom_tab(atom_val(term))->slot.bucket.hvalue; else UINT32_HASH(atom_tab(atom_val(term))->slot.bucket.hvalue, HCONST_3); goto hash2_common; case _TAG_IMMED2_NIL: - if (hash == salt) - hash = 3468870702UL ^ salt; + if (hash == 0) + hash = 3468870702UL; else UINT32_HASH(NIL_DEF, HCONST_2); goto hash2_common; default: - erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X, %lu)\n", term, salt); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); } case _TAG_IMMED1_SMALL: { @@ -1518,7 +1518,7 @@ make_hash2(Eterm term, Uint32 salt) } break; default: - erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X, %lu)\n", term, salt); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); hash2_common: /* Uint32 hash always has the hash value of the previous term, @@ -1542,7 +1542,7 @@ make_hash2(Eterm term, Uint32 salt) } case HASH_MAP_PAIR: hash_xor_pairs ^= hash; - hash = salt; + hash = 0; goto hash2_common; default: break; -- cgit v1.2.3 From eec41eeed17682a4e1f8ca15d92a2b469f0c39b8 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Fri, 21 Apr 2017 14:54:33 +0200 Subject: erts: Add erlang:list_to_port/1 debug bif --- erts/emulator/beam/bif.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++ erts/emulator/beam/bif.tab | 1 + 2 files changed, 70 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 214de3652f..d7a18e8d88 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -4254,6 +4254,75 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } +BIF_RETTYPE list_to_port_1(BIF_ALIST_1) +{ + /* + * A valid port identifier is on the format + * "#Port" where N is node and P is + * the port id. Both N and P are of type Uint32. + */ + Uint32 n, p; + char* cp; + int i; + DistEntry *dep = NULL; + char buf[6 /* #Port< */ + + (2)*(10 + 1) /* N.P> */ + + 1 /* \0 */]; + + /* walk down the list and create a C string */ + if ((i = intlist_to_buf(BIF_ARG_1, buf, sizeof(buf)-1)) < 0) + goto bad; + + buf[i] = '\0'; /* null terminal */ + + cp = &buf[0]; + if (strncmp("#Port<", cp, 6) != 0) + goto bad; + + cp += 6; /* strlen("#Port<") */ + + if (sscanf(cp, "%u.%u>", &n, &p) < 2) + goto bad; + + if (p > ERTS_MAX_PORT_NUMBER) + goto bad; + + dep = erts_channel_no_to_dist_entry(n); + + if (!dep) + goto bad; + + if(dep == erts_this_dist_entry) { + erts_deref_dist_entry(dep); + BIF_RET(make_internal_port(p)); + } + else { + ExternalThing *etp; + ErlNode *enp; + + if (is_nil(dep->cid)) + goto bad; + + enp = erts_find_or_insert_node(dep->sysname, dep->creation); + ASSERT(enp != erts_this_node); + + etp = (ExternalThing *) HAlloc(BIF_P, EXTERNAL_THING_HEAD_SIZE + 1); + etp->header = make_external_port_header(1); + etp->next = MSO(BIF_P).first; + etp->node = enp; + etp->data.ui[0] = p; + + MSO(BIF_P).first = (struct erl_off_heap_header*) etp; + erts_deref_dist_entry(dep); + BIF_RET(make_external_port(etp)); + } + + bad: + if (dep) + erts_deref_dist_entry(dep); + BIF_ERROR(BIF_P, BADARG); +} + BIF_RETTYPE list_to_ref_1(BIF_ALIST_1) { /* diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 66e5dc2988..56d656a173 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -88,6 +88,7 @@ bif erlang:list_to_binary/1 bif erlang:list_to_float/1 bif erlang:list_to_integer/1 bif erlang:list_to_pid/1 +bif erlang:list_to_port/1 bif erlang:list_to_ref/1 bif erlang:list_to_tuple/1 bif erlang:loaded/0 -- cgit v1.2.3 From e40ec046a2a1037b2f87b657503c5f21c5de4e2a Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 27 Apr 2017 12:18:57 +0200 Subject: Move enif_hash last to not break Windows --- erts/emulator/beam/erl_nif_api_funcs.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index 1a91a04f32..b34058f303 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -47,7 +47,6 @@ ERL_NIF_API_FUNC_DECL(int,enif_get_list_cell,(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int* arity, const ERL_NIF_TERM** array)); ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); ERL_NIF_API_FUNC_DECL(int,enif_compare,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); -ERL_NIF_API_FUNC_DECL(ErlNifUInt64,enif_hash,(ErlNifHash type, ERL_NIF_TERM term, ErlNifUInt64 salt)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i)); @@ -181,6 +180,7 @@ ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type_x,(ErlNifEnv*, ERL_NIF_API_FUNC_DECL(int, enif_monitor_process,(ErlNifEnv*,void* obj,const ErlNifPid*,ErlDrvMonitor *monitor)); ERL_NIF_API_FUNC_DECL(int, enif_demonitor_process,(ErlNifEnv*,void* obj,const ErlDrvMonitor *monitor)); ERL_NIF_API_FUNC_DECL(int, enif_compare_monitors,(const ErlNifMonitor*,const ErlNifMonitor*)); +ERL_NIF_API_FUNC_DECL(ErlNifUInt64,enif_hash,(ErlNifHash type, ERL_NIF_TERM term, ErlNifUInt64 salt)); /* ** ADD NEW ENTRIES HERE (before this comment) !!! @@ -209,7 +209,6 @@ ERL_NIF_API_FUNC_DECL(int, enif_compare_monitors,(const ErlNifMonitor*,const Erl # define enif_get_list_cell ERL_NIF_API_FUNC_MACRO(enif_get_list_cell) # define enif_is_identical ERL_NIF_API_FUNC_MACRO(enif_is_identical) # define enif_compare ERL_NIF_API_FUNC_MACRO(enif_compare) -# define enif_hash ERL_NIF_API_FUNC_MACRO(enif_hash) # define enif_make_binary ERL_NIF_API_FUNC_MACRO(enif_make_binary) # define enif_make_badarg ERL_NIF_API_FUNC_MACRO(enif_make_badarg) @@ -344,6 +343,7 @@ ERL_NIF_API_FUNC_DECL(int, enif_compare_monitors,(const ErlNifMonitor*,const Erl # define enif_monitor_process ERL_NIF_API_FUNC_MACRO(enif_monitor_process) # define enif_demonitor_process ERL_NIF_API_FUNC_MACRO(enif_demonitor_process) # define enif_compare_monitors ERL_NIF_API_FUNC_MACRO(enif_compare_monitors) +# define enif_hash ERL_NIF_API_FUNC_MACRO(enif_hash) /* ** ADD NEW ENTRIES HERE (before this comment) -- cgit v1.2.3 From 3684673aab9006d114db4298ab882805bf9bf657 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 27 Apr 2017 19:27:36 +0200 Subject: erts: Optimize make_internal_hash for maps No need for the xor-trick, to be order independent, as both flatmaps and hashmaps have a constant key-value order internally. --- erts/emulator/beam/utils.c | 43 ++++++------------------------------------- 1 file changed, 6 insertions(+), 37 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 9263798a28..cdf766b206 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1581,9 +1581,6 @@ Uint32 make_internal_hash(Eterm term) { Uint32 hash; - Uint32 hash_xor_pairs; - - ERTS_UNDEF(hash_xor_pairs, 0); /* Optimization. Simple cases before declaration of estack. */ if (primary_tag(term) == TAG_PRIMARY_IMMED1) { @@ -1664,6 +1661,11 @@ make_internal_hash(Eterm term) Eterm* ptr = boxed_val(term) + 1; Uint size; int i; + + /* + * We rely on key-value iteration order being constant + * for identical maps (in this VM instance). + */ switch (hdr & _HEADER_MAP_SUBTAG_MASK) { case HAMT_SUBTAG_HEAD_FLATMAP: { @@ -1675,26 +1677,10 @@ make_internal_hash(Eterm term) if (size == 0) goto pop_next; - /* We want a hash function that is *independent* of - * the order in which keys and values are encountered. - * We therefore calculate context independent hashes for all . - * key-value pairs and then xor them together. - * - * We *do* need to use an initial seed for each pair, i.e. the - * hash value, so the hash value is reset for each pair with 'hash'. - * If we don't, no additional entropy is given to the system and the - * hash collision resolution in maps:from_list/1 would fail. - */ - ESTACK_PUSH(s, hash_xor_pairs); - ESTACK_PUSH(s, hash); - ESTACK_PUSH(s, HASH_MAP_TAIL); for (i = size - 1; i >= 0; i--) { - ESTACK_PUSH(s, hash); /* initial seed for all pairs */ - ESTACK_PUSH(s, HASH_MAP_PAIR); ESTACK_PUSH(s, vs[i]); ESTACK_PUSH(s, ks[i]); } - hash_xor_pairs = 0; goto pop_next; } case HAMT_SUBTAG_HEAD_ARRAY: @@ -1703,10 +1689,6 @@ make_internal_hash(Eterm term) UINT32_HASH(size, HCONST_16); if (size == 0) goto pop_next; - ESTACK_PUSH(s, hash_xor_pairs); - ESTACK_PUSH(s, hash); - ESTACK_PUSH(s, HASH_MAP_TAIL); - hash_xor_pairs = 0; } switch (hdr & _HEADER_MAP_SUBTAG_MASK) { case HAMT_SUBTAG_HEAD_ARRAY: @@ -1722,8 +1704,6 @@ make_internal_hash(Eterm term) while (i) { if (is_list(*ptr)) { Eterm* cons = list_val(*ptr); - ESTACK_PUSH(s, hash); /* initial seed for all pairs */ - ESTACK_PUSH(s, HASH_MAP_PAIR); ESTACK_PUSH(s, CDR(cons)); ESTACK_PUSH(s, CAR(cons)); } @@ -1739,7 +1719,7 @@ make_internal_hash(Eterm term) case EXPORT_SUBTAG: { Export* ep = *((Export **) (export_val(term) + 1)); - /* Assumes Export entries never moves */ + /* Assumes Export entries never move */ POINTER_HASH(ep, HCONST_14); goto pop_next; } @@ -1919,17 +1899,6 @@ make_internal_hash(Eterm term) term = ESTACK_POP(s); switch (term) { - case HASH_MAP_TAIL: { - hash = (Uint32) ESTACK_POP(s); - UINT32_HASH(hash_xor_pairs, HCONST_19); - hash_xor_pairs = (Uint32) ESTACK_POP(s); - goto pop_next; - } - case HASH_MAP_PAIR: - hash_xor_pairs ^= hash; - hash = (Uint32) ESTACK_POP(s); /* initial seed for all pairs */ - goto pop_next; - case HASH_CDR: CONST_HASH(HCONST_18); /* Hash CDR i cons cell */ goto pop_next; -- cgit v1.2.3 From 7ce5c7a0d1da593d7718131580b4ce2e73de48c6 Mon Sep 17 00:00:00 2001 From: Salikhov Dinislam Date: Sun, 30 Apr 2017 08:28:20 +0300 Subject: Make a function static --- erts/emulator/beam/erl_port.h | 10 ---------- erts/emulator/beam/io.c | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index c59b42cdae..1c676c7c4f 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -988,16 +988,6 @@ typedef enum { ERTS_PORT_OP_DONE } ErtsPortOpResult; -ErtsPortOpResult -erts_schedule_proc2port_signal(Process *, - Port *, - Eterm, - Eterm *, - ErtsProc2PortSigData *, - int, - ErtsPortTaskHandle *, - ErtsProc2PortSigCallback); - int erts_deliver_port_exit(Port *, Eterm, Eterm, int, int); /* diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index c3eb610fdc..f672444836 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -1511,7 +1511,7 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg, Port* prt) } -ErtsPortOpResult +static ErtsPortOpResult erts_schedule_proc2port_signal(Process *c_p, Port *prt, Eterm caller, -- cgit v1.2.3 From 83e20c62057ebc1d8064bf57b01be560cd244e1d Mon Sep 17 00:00:00 2001 From: Raimo Niskanen Date: Thu, 4 May 2017 15:42:21 +0200 Subject: Update copyright year --- erts/emulator/beam/atom.names | 2 +- erts/emulator/beam/beam_bif_load.c | 2 +- erts/emulator/beam/beam_bp.c | 2 +- erts/emulator/beam/beam_bp.h | 2 +- erts/emulator/beam/beam_debug.c | 2 +- erts/emulator/beam/beam_emu.c | 2 +- erts/emulator/beam/beam_load.c | 2 +- erts/emulator/beam/bif.c | 2 +- erts/emulator/beam/bif.tab | 2 +- erts/emulator/beam/binary.c | 2 +- erts/emulator/beam/break.c | 2 +- erts/emulator/beam/code_ix.h | 2 +- erts/emulator/beam/copy.c | 2 +- erts/emulator/beam/dist.c | 2 +- erts/emulator/beam/dist.h | 2 +- erts/emulator/beam/dtrace-wrapper.h | 2 +- erts/emulator/beam/erl_alloc.c | 2 +- erts/emulator/beam/erl_alloc.types | 2 +- erts/emulator/beam/erl_alloc_util.c | 2 +- erts/emulator/beam/erl_async.h | 2 +- erts/emulator/beam/erl_bif_binary.c | 2 +- erts/emulator/beam/erl_bif_ddll.c | 2 +- erts/emulator/beam/erl_bif_guard.c | 2 +- erts/emulator/beam/erl_bif_info.c | 2 +- erts/emulator/beam/erl_bif_os.c | 2 +- erts/emulator/beam/erl_bif_port.c | 2 +- erts/emulator/beam/erl_bif_re.c | 2 +- erts/emulator/beam/erl_bif_trace.c | 2 +- erts/emulator/beam/erl_bif_unique.c | 2 +- erts/emulator/beam/erl_bif_unique.h | 2 +- erts/emulator/beam/erl_binary.h | 2 +- erts/emulator/beam/erl_bits.c | 2 +- erts/emulator/beam/erl_bits.h | 2 +- erts/emulator/beam/erl_db.c | 2 +- erts/emulator/beam/erl_db_hash.c | 2 +- erts/emulator/beam/erl_db_hash.h | 2 +- erts/emulator/beam/erl_db_tree.c | 2 +- erts/emulator/beam/erl_db_util.c | 2 +- erts/emulator/beam/erl_db_util.h | 2 +- erts/emulator/beam/erl_debug.c | 2 +- erts/emulator/beam/erl_driver.h | 2 +- erts/emulator/beam/erl_drv_nif.h | 2 +- erts/emulator/beam/erl_fun.c | 2 +- erts/emulator/beam/erl_fun.h | 2 +- erts/emulator/beam/erl_gc.c | 2 +- erts/emulator/beam/erl_gc.h | 2 +- erts/emulator/beam/erl_hl_timer.c | 2 +- erts/emulator/beam/erl_hl_timer.h | 2 +- erts/emulator/beam/erl_init.c | 2 +- erts/emulator/beam/erl_lock_check.c | 2 +- erts/emulator/beam/erl_lock_count.h | 2 +- erts/emulator/beam/erl_map.c | 2 +- erts/emulator/beam/erl_map.h | 2 +- erts/emulator/beam/erl_message.c | 2 +- erts/emulator/beam/erl_monitors.c | 2 +- erts/emulator/beam/erl_monitors.h | 2 +- erts/emulator/beam/erl_msacc.c | 2 +- erts/emulator/beam/erl_mtrace.c | 2 +- erts/emulator/beam/erl_nif.c | 2 +- erts/emulator/beam/erl_nif.h | 2 +- erts/emulator/beam/erl_nif_api_funcs.h | 2 +- erts/emulator/beam/erl_node_container_utils.h | 2 +- erts/emulator/beam/erl_node_tables.c | 2 +- erts/emulator/beam/erl_node_tables.h | 2 +- erts/emulator/beam/erl_printf_term.c | 2 +- erts/emulator/beam/erl_process.c | 2 +- erts/emulator/beam/erl_process.h | 2 +- erts/emulator/beam/erl_process_dict.c | 2 +- erts/emulator/beam/erl_process_dump.c | 2 +- erts/emulator/beam/erl_process_lock.c | 2 +- erts/emulator/beam/erl_process_lock.h | 2 +- erts/emulator/beam/erl_ptab.c | 2 +- erts/emulator/beam/erl_rbtree.h | 2 +- erts/emulator/beam/erl_term.c | 2 +- erts/emulator/beam/erl_term.h | 2 +- erts/emulator/beam/erl_time.h | 2 +- erts/emulator/beam/erl_time_sup.c | 2 +- erts/emulator/beam/erl_trace.c | 2 +- erts/emulator/beam/erl_unicode.c | 2 +- erts/emulator/beam/erl_utils.h | 2 +- erts/emulator/beam/erl_vm.h | 2 +- erts/emulator/beam/export.c | 2 +- erts/emulator/beam/external.c | 2 +- erts/emulator/beam/global.h | 2 +- erts/emulator/beam/index.c | 2 +- erts/emulator/beam/index.h | 2 +- erts/emulator/beam/io.c | 2 +- erts/emulator/beam/sys.h | 2 +- erts/emulator/beam/time.c | 2 +- erts/emulator/beam/utils.c | 2 +- 90 files changed, 90 insertions(+), 90 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index df2866b40e..477a7676d6 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2016. All Rights Reserved. +# Copyright Ericsson AB 1996-2017. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 769fe89219..5aceae8ffe 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2016. All Rights Reserved. + * Copyright Ericsson AB 1999-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 1efe7536d6..92ede09102 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2016. All Rights Reserved. + * Copyright Ericsson AB 2000-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 3dba3cc1b5..56fa82b912 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2016. All Rights Reserved. + * Copyright Ericsson AB 2000-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 3f4a1f9e80..a2060c80de 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2016. All Rights Reserved. + * Copyright Ericsson AB 1998-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 3d36094e07..311bb94242 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 7b79e6303b..a0351746b9 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 84c9f9d7c4..7134d2da56 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 3f6d82d65c..a8bbf5f8c1 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2016. All Rights Reserved. +# Copyright Ericsson AB 1996-2017. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index 0df6bbb289..ca3e48e205 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index e23fdb83d9..76a0c5c716 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h index e802ad5dd7..a28b0cd36e 100644 --- a/erts/emulator/beam/code_ix.h +++ b/erts/emulator/beam/code_ix.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012-2016. All Rights Reserved. + * Copyright Ericsson AB 2012-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 62c3f9520d..fefde256d7 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index a1581908e5..982f1066df 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index 6ed36a478e..93a651b24d 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/dtrace-wrapper.h b/erts/emulator/beam/dtrace-wrapper.h index f6fc28b801..15ea182976 100644 --- a/erts/emulator/beam/dtrace-wrapper.h +++ b/erts/emulator/beam/dtrace-wrapper.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2016. + * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2017. * All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 71957b2259..b2c1133ded 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2016. All Rights Reserved. + * Copyright Ericsson AB 2002-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index f296a98125..8a23a1526e 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2003-2016. All Rights Reserved. +# Copyright Ericsson AB 2003-2017. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index a6bac06478..4889fac923 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2016. All Rights Reserved. + * Copyright Ericsson AB 2002-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_async.h b/erts/emulator/beam/erl_async.h index c884a5040d..4b470e7679 100644 --- a/erts/emulator/beam/erl_async.h +++ b/erts/emulator/beam/erl_async.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011-2016. All Rights Reserved. + * Copyright Ericsson AB 2011-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index f79b5b6843..756c7dce05 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010-2016. All Rights Reserved. + * Copyright Ericsson AB 2010-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index 2ce872d2bc..e9bfb39035 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2016. All Rights Reserved. + * Copyright Ericsson AB 2006-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c index cdac4c1d11..8a5c6ada6c 100644 --- a/erts/emulator/beam/erl_bif_guard.c +++ b/erts/emulator/beam/erl_bif_guard.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2016. All Rights Reserved. + * Copyright Ericsson AB 2006-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 1a680b127c..5fc70dfc02 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2016. All Rights Reserved. + * Copyright Ericsson AB 1999-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bif_os.c b/erts/emulator/beam/erl_bif_os.c index edc3c82b23..5cd172c26f 100644 --- a/erts/emulator/beam/erl_bif_os.c +++ b/erts/emulator/beam/erl_bif_os.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2016. All Rights Reserved. + * Copyright Ericsson AB 1999-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 09d2c5376b..ff03151619 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2016. All Rights Reserved. + * Copyright Ericsson AB 2001-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index a72697eb80..ad124fd979 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2016. All Rights Reserved. + * Copyright Ericsson AB 2008-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 023bfca797..45159c4392 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2016. All Rights Reserved. + * Copyright Ericsson AB 1999-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c index 7f438daec0..fc6fb5f868 100644 --- a/erts/emulator/beam/erl_bif_unique.c +++ b/erts/emulator/beam/erl_bif_unique.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2014-2016. All Rights Reserved. + * Copyright Ericsson AB 2014-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h index 0f3f794878..9aa631fde9 100644 --- a/erts/emulator/beam/erl_bif_unique.h +++ b/erts/emulator/beam/erl_bif_unique.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2014-2016. All Rights Reserved. + * Copyright Ericsson AB 2014-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index de7dbf4e20..b036b28dbf 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2016. All Rights Reserved. + * Copyright Ericsson AB 2000-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index df3f6ad557..eea55b3239 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2016. All Rights Reserved. + * Copyright Ericsson AB 1999-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h index 1b4546722f..632855255e 100644 --- a/erts/emulator/beam/erl_bits.h +++ b/erts/emulator/beam/erl_bits.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2016. All Rights Reserved. + * Copyright Ericsson AB 1999-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 98c689f13f..17e0f2aeec 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 9009c00833..08a0f0e83b 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2016. All Rights Reserved. + * Copyright Ericsson AB 1998-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index c340c72311..f491c85d95 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2016. All Rights Reserved. + * Copyright Ericsson AB 1998-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index f6918b8ec4..d7deadacf0 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2016. All Rights Reserved. + * Copyright Ericsson AB 1998-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 24b22eafb8..91ac53ad0b 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2016. All Rights Reserved. + * Copyright Ericsson AB 1998-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index ed7b9c8618..19055c6110 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2016. All Rights Reserved. + * Copyright Ericsson AB 1998-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c index 517dad9cbb..bf8244564a 100644 --- a/erts/emulator/beam/erl_debug.c +++ b/erts/emulator/beam/erl_debug.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2016. All Rights Reserved. + * Copyright Ericsson AB 1998-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index b386b68cff..0e8ebf0c98 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2016. All Rights Reserved. + * Copyright Ericsson AB 1999-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h index 2d21dac87a..f88138063e 100644 --- a/erts/emulator/beam/erl_drv_nif.h +++ b/erts/emulator/beam/erl_drv_nif.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010-2016. All Rights Reserved. + * Copyright Ericsson AB 2010-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index cedc0517e1..d18016c42e 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2016. All Rights Reserved. + * Copyright Ericsson AB 2000-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index e3073eb874..289d0d0b28 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2016. All Rights Reserved. + * Copyright Ericsson AB 2000-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index a991c2c164..a7a8da4ed8 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2016. All Rights Reserved. + * Copyright Ericsson AB 2002-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 414aff1e06..6a529b8443 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2007-2016. All Rights Reserved. + * Copyright Ericsson AB 2007-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index 13d6136672..99995be464 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2015-2016. All Rights Reserved. + * Copyright Ericsson AB 2015-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h index f70fcdd1c0..ff31f04cb9 100644 --- a/erts/emulator/beam/erl_hl_timer.h +++ b/erts/emulator/beam/erl_hl_timer.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2015-2016. All Rights Reserved. + * Copyright Ericsson AB 2015-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index eaaf5c911a..fb38327d02 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2016. All Rights Reserved. + * Copyright Ericsson AB 1997-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index da73469516..f270d8baef 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2016. All Rights Reserved. + * Copyright Ericsson AB 2005-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h index 71cd73ee27..3041474382 100644 --- a/erts/emulator/beam/erl_lock_count.h +++ b/erts/emulator/beam/erl_lock_count.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2016. All Rights Reserved. + * Copyright Ericsson AB 2008-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index a80f5d6a16..f0c54e05f7 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2014-2016. All Rights Reserved. + * Copyright Ericsson AB 2014-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h index f7d0413685..c3ccf80b85 100644 --- a/erts/emulator/beam/erl_map.h +++ b/erts/emulator/beam/erl_map.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2014-2016. All Rights Reserved. + * Copyright Ericsson AB 2014-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 17982a2d14..c1af70592a 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2016. All Rights Reserved. + * Copyright Ericsson AB 1997-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c index 6dee1d5ef3..3994800ba7 100644 --- a/erts/emulator/beam/erl_monitors.c +++ b/erts/emulator/beam/erl_monitors.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2016. All Rights Reserved. + * Copyright Ericsson AB 2004-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_monitors.h b/erts/emulator/beam/erl_monitors.h index f659829e6c..1cacecd7e9 100644 --- a/erts/emulator/beam/erl_monitors.h +++ b/erts/emulator/beam/erl_monitors.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2016. All Rights Reserved. + * Copyright Ericsson AB 2004-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c index 1c3160efaf..2d70f0d874 100644 --- a/erts/emulator/beam/erl_msacc.c +++ b/erts/emulator/beam/erl_msacc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2014-2016. All Rights Reserved. + * Copyright Ericsson AB 2014-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c index bb6f8660f1..19bb7d5b31 100644 --- a/erts/emulator/beam/erl_mtrace.c +++ b/erts/emulator/beam/erl_mtrace.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2016. All Rights Reserved. + * Copyright Ericsson AB 2003-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index e041fd7b83..ea835d1b64 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2016. All Rights Reserved. + * Copyright Ericsson AB 2009-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 5a81f5fbbb..b0d5c39798 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2016. All Rights Reserved. + * Copyright Ericsson AB 2009-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index b34058f303..bbdadafade 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2016. All Rights Reserved. + * Copyright Ericsson AB 2009-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h index e27f1d121a..6ec428e282 100644 --- a/erts/emulator/beam/erl_node_container_utils.h +++ b/erts/emulator/beam/erl_node_container_utils.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2016. All Rights Reserved. + * Copyright Ericsson AB 2001-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 89b627aaf5..3c5945d48d 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2016. All Rights Reserved. + * Copyright Ericsson AB 2001-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index 35051173d0..489da1ba17 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2016. All Rights Reserved. + * Copyright Ericsson AB 2001-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index 6b64bbd2f1..e6f8460164 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2016. All Rights Reserved. + * Copyright Ericsson AB 2005-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 9947e33f47..a56ef4aedb 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 5b35dc3c78..6113c8aa05 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c index 01e240c65d..3c80f0e0f6 100644 --- a/erts/emulator/beam/erl_process_dict.c +++ b/erts/emulator/beam/erl_process_dict.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2016. All Rights Reserved. + * Copyright Ericsson AB 1999-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index fbf14df92b..b826e6c5d3 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2016. All Rights Reserved. + * Copyright Ericsson AB 2003-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index a93f1755c8..c0e7380ed0 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2007-2016. All Rights Reserved. + * Copyright Ericsson AB 2007-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index 46a72fcb0c..6e704b185d 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2007-2016. All Rights Reserved. + * Copyright Ericsson AB 2007-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c index abf2fe44f3..c3d59cb3a8 100644 --- a/erts/emulator/beam/erl_ptab.c +++ b/erts/emulator/beam/erl_ptab.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012-2016. All Rights Reserved. + * Copyright Ericsson AB 2012-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_rbtree.h b/erts/emulator/beam/erl_rbtree.h index 6a42853957..e59d6900b0 100644 --- a/erts/emulator/beam/erl_rbtree.h +++ b/erts/emulator/beam/erl_rbtree.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2015. All Rights Reserved. + * Copyright Ericsson AB 2015-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c index 6b25728af7..d904e35e40 100644 --- a/erts/emulator/beam/erl_term.c +++ b/erts/emulator/beam/erl_term.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2016. All Rights Reserved. + * Copyright Ericsson AB 2000-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index 097d580d99..842802f8d9 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2016. All Rights Reserved. + * Copyright Ericsson AB 2000-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 46d6da6448..ccc5526664 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2016. All Rights Reserved. + * Copyright Ericsson AB 2006-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index c69fec3c80..3084a8db75 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2016. All Rights Reserved. + * Copyright Ericsson AB 1999-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index a5fc3a2477..4b06c55770 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2016. All Rights Reserved. + * Copyright Ericsson AB 1999-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index 01629db9ad..2d1d1443a7 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2016. All Rights Reserved. + * Copyright Ericsson AB 2008-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index 75d7e47239..07cf4f6903 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012-2016. All Rights Reserved. + * Copyright Ericsson AB 2012-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index 5366ee3644..0b8d78c469 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index ecfef28c57..57f5ba5436 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index f5a5da981c..1190d90b8e 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index bb4d442240..e3be6a5a22 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c index cd834e2c12..a1f6f54543 100644 --- a/erts/emulator/beam/index.c +++ b/erts/emulator/beam/index.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h index 10f5d1eb39..6c07571df6 100644 --- a/erts/emulator/beam/index.h +++ b/erts/emulator/beam/index.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 2f3117223f..bf89740fda 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 24de35696c..d752ea4330 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index a7e5a64b22..a3069e419a 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 96a7bfe8ac..457cada745 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. -- cgit v1.2.3 From e01b87f45486330561d5673fd109a6b6e40828b2 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 3 May 2017 20:27:01 +0200 Subject: erts: Move and rename erts_is_native_break() --- erts/emulator/beam/beam_bp.c | 21 +++++---------------- erts/emulator/beam/beam_load.c | 12 ++++++++++++ erts/emulator/beam/beam_load.h | 1 + 3 files changed, 18 insertions(+), 16 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 1efe7536d6..b01527fa52 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -211,7 +211,7 @@ erts_bp_match_functions(BpFunctions* f, ErtsCodeMFA *mfa, int specified) ci = code_hdr->functions[fi]; ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - if (erts_is_native_break(ci)) { + if (erts_is_function_native(ci)) { continue; } if (is_nil(ci->mfa.module)) { /* Ignore BIF stub */ @@ -277,7 +277,7 @@ erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified) continue; } ASSERT(*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)); - } else if (erts_is_native_break(erts_code_to_codeinfo(ep->addressv[code_ix]))) { + } else if (erts_is_function_native(erts_code_to_codeinfo(ep->addressv[code_ix]))) { continue; } @@ -611,7 +611,7 @@ erts_clear_module_break(Module *modp) { n = (Uint)(UWord) code_hdr->num_functions; for (i = 0; i < n; ++i) { ErtsCodeInfo *ci = code_hdr->functions[i]; - if (erts_is_native_break(ci)) + if (erts_is_function_native(ci)) continue; clear_function_break(ci, ERTS_BPF_ALL); } @@ -620,7 +620,7 @@ erts_clear_module_break(Module *modp) { for (i = 0; i < n; ++i) { ErtsCodeInfo *ci = code_hdr->functions[i]; - if (erts_is_native_break(ci)) + if (erts_is_function_native(ci)) continue; uninstall_breakpoint(ci); consolidate_bp_data(modp, ci, 1); @@ -1212,17 +1212,6 @@ erts_is_mtrace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, return 0; } -int -erts_is_native_break(ErtsCodeInfo *ci) { -#ifdef HIPE - ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - return erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call) - || erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call_closure); -#else - return 0; -#endif -} - int erts_is_count_break(ErtsCodeInfo *ci, Uint *count_ret) { @@ -1731,7 +1720,7 @@ check_break(ErtsCodeInfo *ci, Uint break_flags) GenericBp* g = ci->u.gen_bp; ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - if (erts_is_native_break(ci)) { + if (erts_is_function_native(ci)) { return 0; } if (g) { diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 7b79e6303b..2763d8a38e 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -5696,6 +5696,18 @@ erts_is_module_native(BeamCodeHeader* code_hdr) return 0; } +int +erts_is_function_native(ErtsCodeInfo *ci) +{ +#ifdef HIPE + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + return erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call) + || erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call_closure); +#else + return 0; +#endif +} + /* * Builds a list of all functions including native addresses. * [{Name,Arity,NativeAddress},...] diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 659b9c303f..b8d8634e28 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -119,6 +119,7 @@ typedef struct beam_code_header { void erts_release_literal_area(struct ErtsLiteralArea_* literal_area); int erts_is_module_native(BeamCodeHeader* code); +int erts_is_function_native(ErtsCodeInfo*); void erts_beam_bif_load_init(void); struct erl_fun_entry; void erts_purge_state_add_fun(struct erl_fun_entry *fe); -- cgit v1.2.3 From f95094d67e56a51bbdaadaf18625bb01c584bf2d Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 3 May 2017 20:33:13 +0200 Subject: erts: Fix code:is_module_native for local trace Local trace on first function in module made code:is_module_native/1 return true. Use new erts_is_function_native() to make a proper check. --- erts/emulator/beam/beam_load.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 2763d8a38e..4f5c68d29a 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -5685,12 +5685,11 @@ erts_is_module_native(BeamCodeHeader* code_hdr) if (code_hdr != NULL) { num_functions = code_hdr->num_functions; for (i=0; ifunctions[i]; - Eterm name = (Eterm) func_info[3]; - if (is_atom(name)) { - return func_info[1] != 0; + ErtsCodeInfo* ci = code_hdr->functions[i]; + if (is_atom(ci->mfa.function)) { + return erts_is_function_native(ci); } - else ASSERT(is_nil(name)); /* ignore BIF stubs */ + else ASSERT(is_nil(ci->mfa.function)); /* ignore BIF stubs */ } } return 0; -- cgit v1.2.3 From ff94c59ca65ab3f4d9a1660e64ae31ed2232a5c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Fri, 5 May 2017 05:23:54 +0200 Subject: Clear p->fvalue when handling a try/catch p->fvalue will be set by BIFs that generate exceptions (such as throw/1), and it will not be cleared before another exception is generated. Potentially, p->fvalue may contain a huge term (e.g. after throw(HugeTerm)) which will be kept in the heap. We can shorten the lifetime of f->value by clearing it in the instructions that handle catches: catch_end and try_end. That is safe because BEAM code will never access p->fvalue. If BEAM code needs to rethrow an exception it will use a reference to the value passed in an X register. The reason that p->fvalue must not be cleared already in handle_error() is that native code trap handlers will use it. (See the comment before handle_error() and the comment at the end of handle_error() in beam_emu.c for some more information about exception handling.) --- erts/emulator/beam/beam_emu.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 311bb94242..a90e6a0ba8 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1814,6 +1814,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) c_p->catches--; make_blank(yb(Arg(0))); if (is_non_value(r(0))) { + c_p->fvalue = NIL; if (x(1) == am_throw) { r(0) = x(2); } else { @@ -1843,6 +1844,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) c_p->catches--; make_blank(yb(Arg(0))); if (is_non_value(r(0))) { + c_p->fvalue = NIL; r(0) = x(1); x(1) = x(2); x(2) = x(3); -- cgit v1.2.3 From 900affc2420905ec70f3cf906324093fc0c2e8a8 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Fri, 5 May 2017 14:55:36 +0200 Subject: erts: the esdp is not always available in matchspec This happens for instance when a receive_trace is run in the sys_msg_dispatcher thread when the trace_delivered trace message is traced. --- erts/emulator/beam/erl_db_util.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 91ac53ad0b..16142e69fb 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -2006,7 +2006,8 @@ restart: do_catch = 0; fail_label = -1; build_proc = psp; - esdp->current_process = psp; + if (esdp) + esdp->current_process = psp; #ifdef DEBUG ASSERT(variables == mpsp->u.variables); @@ -2675,7 +2676,8 @@ restart: do_catch = 1; if (in_flags & ERTS_PAM_COPY_RESULT) { build_proc = c_p; - esdp->current_process = c_p; + if (esdp) + esdp->current_process = c_p; } break; case matchHalt: -- cgit v1.2.3 From 3f0e95a08394e92a58f99f99a94f9349e35842dd Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 5 May 2017 19:47:07 +0200 Subject: erts: Unbreak --enable-lock-checking --enable-lock-counter both together. Broken in OTP-20.0-rc1 by 7d161f5b475575bd79bd90977b3a79334a8ec658. --- erts/emulator/beam/erl_lock_count.c | 8 ++++++-- erts/emulator/beam/erl_lock_count.h | 2 ++ erts/emulator/beam/erl_port_task.h | 8 +++++--- erts/emulator/beam/erl_smp.h | 2 +- erts/emulator/beam/erl_threads.h | 6 +++--- erts/emulator/beam/io.c | 8 +++++--- 6 files changed, 22 insertions(+), 12 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index 6354fc8663..49b1f08b44 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -358,8 +358,11 @@ void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) { void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) { int i; - if (name == NULL) { ERTS_LCNT_CLEAR_FLAG(lock); return; } - lcnt_lock(); + + if (flag & ERTS_LCNT_LT_DISABLE) { + ERTS_LCNT_CLEAR_FLAG(lock); + return; + } lock->next = NULL; lock->prev = NULL; @@ -379,6 +382,7 @@ void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eter lcnt_clear_stats(&lock->stats[i]); } + lcnt_lock(); erts_lcnt_list_insert(erts_lcnt_data->current_locks, lock); lcnt_unlock(); } diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h index 3041474382..be601a26fc 100644 --- a/erts/emulator/beam/erl_lock_count.h +++ b/erts/emulator/beam/erl_lock_count.h @@ -95,6 +95,8 @@ #define ERTS_LCNT_LO_READ (((Uint16) 1) << 6) #define ERTS_LCNT_LO_WRITE (((Uint16) 1) << 7) +#define ERTS_LCNT_LT_DISABLE (((Uint16) 1) << 8) + #define ERTS_LCNT_LO_READ_WRITE ( ERTS_LCNT_LO_READ \ | ERTS_LCNT_LO_WRITE ) diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index e3550e878e..858ddfe029 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -189,11 +189,13 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp, Eterm instr_id) erts_smp_atomic32_init_nob(&ptsp->flags, 0); #ifdef ERTS_SMP #ifdef ERTS_ENABLE_LOCK_COUNT - if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)) - lock_str = NULL; -#endif + erts_mtx_init_x_opt(&ptsp->mtx, lock_str, instr_id, + ((erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK) + ? 0 : ERTS_LCNT_LT_DISABLE)); +#else erts_mtx_init_x(&ptsp->mtx, lock_str, instr_id); #endif +#endif } ERTS_GLB_INLINE void diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index 14be511f86..55ba943bdd 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -1073,7 +1073,7 @@ ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, char *name, Eterm extra) { #ifdef ERTS_SMP - erts_mtx_init_locked_x(mtx, name, extra); + erts_mtx_init_locked_x_opt(mtx, name, extra, 0); #endif } diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index 9e75f6fee5..28ff5d3a42 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -481,7 +481,7 @@ ERTS_GLB_INLINE int erts_thr_getname(erts_tid_t tid, char *buf, size_t len); ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y); ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra); ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt); -ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra); +ERTS_GLB_INLINE void erts_mtx_init_locked_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt); ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx); @@ -2192,7 +2192,7 @@ erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt) ERTS_GLB_INLINE void -erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra) +erts_mtx_init_locked_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt) { #ifdef USE_THREADS int res = ethr_mutex_init(&mtx->mtx); @@ -2202,7 +2202,7 @@ erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra) erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); + erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra); #endif ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_CHECK diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index bf89740fda..93a5ed4397 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -260,10 +260,12 @@ static ERTS_INLINE void port_init_instr(Port *prt if (!prt->drv_ptr->lock) { char *lock_str = "port_lock"; #ifdef ERTS_ENABLE_LOCK_COUNT - if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)) - lock_str = NULL; + Uint16 opt = ((erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK) + ? 0 : ERTS_LCNT_LT_DISABLE); +#else + Uint16 opt = 0; #endif - erts_mtx_init_locked_x(prt->lock, lock_str, id); + erts_mtx_init_locked_x_opt(prt->lock, lock_str, id, opt); } #endif erts_port_task_init_sched(&prt->sched, id); -- cgit v1.2.3 From eb9ffb378b9eb3b9f4a0f898e60ade01d3d48283 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 5 May 2017 20:29:58 +0200 Subject: erts: Add minor lock optimization for lcnt --- erts/emulator/beam/erl_lock_count.c | 21 ++++++++++----------- erts/emulator/beam/erl_lock_count.h | 1 - 2 files changed, 10 insertions(+), 12 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index 49b1f08b44..aee9796171 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -299,22 +299,16 @@ erts_lcnt_lock_list_t *erts_lcnt_list_init(void) { return list; } -/* only do this on the list with the deleted locks! */ -void erts_lcnt_list_clear(erts_lcnt_lock_list_t *list) { - erts_lcnt_lock_t *lock = NULL, - *next = NULL; +static void lcnt_list_free(erts_lcnt_lock_t *head) { + erts_lcnt_lock_t *lock, *next; - lock = list->head; + lock = head; while(lock != NULL) { next = lock->next; free(lock); lock = next; } - - list->head = NULL; - list->tail = NULL; - list->n = 0; } void erts_lcnt_list_insert(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) { @@ -679,12 +673,17 @@ void erts_lcnt_clear_counters(void) { lock->n_stats = 1; } - /* empty deleted locks in lock list */ - erts_lcnt_list_clear(erts_lcnt_data->deleted_locks); + lock = erts_lcnt_data->deleted_locks->head; + erts_lcnt_data->deleted_locks->head = NULL; + erts_lcnt_data->deleted_locks->tail = NULL; + erts_lcnt_data->deleted_locks->n = 0; lcnt_time(&timer_start); lcnt_unlock(); + + /* free deleted locks */ + lcnt_list_free(lock); } erts_lcnt_data_t *erts_lcnt_get_data(void) { diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h index be601a26fc..6caffbfe86 100644 --- a/erts/emulator/beam/erl_lock_count.h +++ b/erts/emulator/beam/erl_lock_count.h @@ -206,7 +206,6 @@ void erts_lcnt_thread_exit_handler(void); /* list operations (local) */ erts_lcnt_lock_list_t *erts_lcnt_list_init(void); -void erts_lcnt_list_clear( erts_lcnt_lock_list_t *list); void erts_lcnt_list_insert(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock); void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock); -- cgit v1.2.3 From 95dec7497abbdb06defcb23e99fa31e06597496e Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 3 May 2017 18:22:45 +0200 Subject: erts: Fix ets:select_replace with {const, NewTuple} Enable ets:select_replace to do a generic single object compare-and-swap operation of any ets-tuple using a matchspec like this: [{Old, [], [{const, New}]}] The only exception when this does not work is if the key contains maps or atoms looking like variables (like '$1'). --- erts/emulator/beam/erl_db_util.c | 41 +++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 24b22eafb8..d1130fac87 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1125,12 +1125,11 @@ error: * Returns true if 'b' is guaranteed to always construct * the same term as 'a' has matched. */ -static int db_match_eq_body(Eterm a, Eterm b) +static int db_match_eq_body(Eterm a, Eterm b, int const_mode) { DECLARE_ESTACK(s); Uint arity; Eterm *ap, *bp; - int const_mode = 0; const Eterm CONST_MODE_OFF = THE_NON_VALUE; while (1) { @@ -1224,6 +1223,7 @@ int db_match_keeps_key(int keypos, Eterm match, Eterm guard, Eterm body) Eterm single_body_subterm; Eterm single_body_subterm_key; Eterm* single_body_subterm_key_tpl; + int const_mode; if (!is_list(body)) { return 0; @@ -1244,42 +1244,53 @@ int db_match_keeps_key(int keypos, Eterm match, Eterm guard, Eterm body) return 0; } - single_body_term_tpl = tuple_val(single_body_term); - if (arityval(*single_body_term_tpl) != 1) { - // not the 1-element tuple we're expecting - return 0; - } - match_key = db_getkey(keypos, match); if (!is_value(match_key)) { // can't get key out of match return 0; } - single_body_subterm = single_body_term_tpl[1]; + single_body_term_tpl = tuple_val(single_body_term); + if (single_body_term_tpl[0] == make_arityval(2) && + single_body_term_tpl[1] == am_const) { + /* {const, {"ets-tuple constant"}} */ + single_body_subterm = single_body_term_tpl[2]; + const_mode = 1; + } + else if (*single_body_term_tpl == make_arityval(1)) { + /* {{"ets-tuple construction"}} */ + single_body_subterm = single_body_term_tpl[1]; + const_mode = 0; + } + else { + /* not a tuple construction */ + return 0; + } + single_body_subterm_key = db_getkey(keypos, single_body_subterm); if (!is_value(single_body_subterm_key)) { // can't get key out of single body subterm return 0; } - if (db_match_eq_body(match_key, single_body_subterm_key)) { + if (db_match_eq_body(match_key, single_body_subterm_key, const_mode)) { /* tuple with same key is returned */ return 1; } - if (!is_tuple(single_body_subterm_key)) { - /* can't possibly be an element instruction */ + if (const_mode) { + /* constant key did not match */ return 0; } - single_body_subterm_key_tpl = tuple_val(single_body_subterm_key); - if (arityval(*single_body_subterm_key_tpl) != 3) { + if (!is_tuple(single_body_subterm_key)) { /* can't possibly be an element instruction */ return 0; } - if (single_body_subterm_key_tpl[1] == am_element && + single_body_subterm_key_tpl = tuple_val(single_body_subterm_key); + if (single_body_subterm_key_tpl[0] == make_arityval(3) && + single_body_subterm_key_tpl[1] == am_element && single_body_subterm_key_tpl[3] == am_DollarUnderscore && single_body_subterm_key_tpl[2] == make_small(keypos)) { -- cgit v1.2.3 From 2746827fe38db572bb26b280287da22b958d89c3 Mon Sep 17 00:00:00 2001 From: Daniel Goertzen Date: Fri, 12 May 2017 14:05:05 -0500 Subject: use ErlNifMonitor instead of ErlDrvMonitor for enif_monitor/demonitor --- erts/emulator/beam/erl_nif_api_funcs.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index bbdadafade..c305732d63 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -177,8 +177,8 @@ ERL_NIF_API_FUNC_DECL(int,enif_thread_type,(void)); ERL_NIF_API_FUNC_DECL(int,enif_snprintf,(char * buffer, size_t size, const char *format, ...)); ERL_NIF_API_FUNC_DECL(int,enif_select,(ErlNifEnv* env, ErlNifEvent e, enum ErlNifSelectFlags flags, void* obj, const ErlNifPid* pid, ERL_NIF_TERM ref)); ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type_x,(ErlNifEnv*, const char* name_str, const ErlNifResourceTypeInit*, ErlNifResourceFlags flags, ErlNifResourceFlags* tried)); -ERL_NIF_API_FUNC_DECL(int, enif_monitor_process,(ErlNifEnv*,void* obj,const ErlNifPid*,ErlDrvMonitor *monitor)); -ERL_NIF_API_FUNC_DECL(int, enif_demonitor_process,(ErlNifEnv*,void* obj,const ErlDrvMonitor *monitor)); +ERL_NIF_API_FUNC_DECL(int, enif_monitor_process,(ErlNifEnv*,void* obj,const ErlNifPid*,ErlNifMonitor *monitor)); +ERL_NIF_API_FUNC_DECL(int, enif_demonitor_process,(ErlNifEnv*,void* obj,const ErlNifMonitor *monitor)); ERL_NIF_API_FUNC_DECL(int, enif_compare_monitors,(const ErlNifMonitor*,const ErlNifMonitor*)); ERL_NIF_API_FUNC_DECL(ErlNifUInt64,enif_hash,(ErlNifHash type, ERL_NIF_TERM term, ErlNifUInt64 salt)); -- cgit v1.2.3 From e6437e926340c3024449b83826f8013d187caaed Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Wed, 3 May 2017 17:18:44 +0200 Subject: erts: Remove old unused functions The functions have been found using: https://github.com/caolanm/callcatcher --- erts/emulator/beam/big.c | 25 ---------------------- erts/emulator/beam/big.h | 2 -- erts/emulator/beam/erl_alloc.c | 22 -------------------- erts/emulator/beam/erl_alloc.h | 7 ------- erts/emulator/beam/erl_cpu_topology.c | 11 ---------- erts/emulator/beam/erl_cpu_topology.h | 8 ------- erts/emulator/beam/erl_db_util.c | 5 ----- erts/emulator/beam/erl_gc.c | 2 +- erts/emulator/beam/erl_init.c | 24 --------------------- erts/emulator/beam/erl_port_task.c | 7 ------- erts/emulator/beam/erl_port_task.h | 1 - erts/emulator/beam/erl_process.c | 39 ----------------------------------- erts/emulator/beam/erl_process.h | 7 ------- erts/emulator/beam/global.h | 2 -- 14 files changed, 1 insertion(+), 161 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index 4baee7900b..1f6feade1c 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -2265,21 +2265,6 @@ Eterm big_minus(Eterm x, Eterm y, Eterm *r) BIG_V(yp),BIG_SIZE(yp),(short) !BIG_SIGN(yp), r); } -/* -** Subtract a digit from big number -*/ -Eterm big_minus_small(Eterm x, Eterm y, Eterm *r) -{ - Eterm* xp = big_val(x); - - if (BIG_SIGN(xp)) - return big_norm(r, D_add(BIG_V(xp),BIG_SIZE(xp), (ErtsDigit) y, BIG_V(r)), - (short) BIG_SIGN(xp)); - else - return big_norm(r, D_sub(BIG_V(xp),BIG_SIZE(xp), (ErtsDigit) y, BIG_V(r)), - (short) BIG_SIGN(xp)); -} - /* ** Multiply smallnums */ @@ -2412,16 +2397,6 @@ Eterm big_rem(Eterm x, Eterm y, Eterm *r) } } -Eterm big_neg(Eterm x, Eterm *r) -{ - Eterm* xp = big_val(x); - dsize_t xsz = BIG_SIZE(xp); - short xsgn = BIG_SIGN(xp); - - MOVE_DIGITS(BIG_V(r), BIG_V(xp), xsz); - return big_norm(r, xsz, (short) !xsgn); -} - Eterm big_band(Eterm x, Eterm y, Eterm *r) { Eterm* xp = big_val(x); diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index 4a96d971c3..258038a157 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -118,9 +118,7 @@ Eterm big_minus(Eterm, Eterm, Eterm*); Eterm big_times(Eterm, Eterm, Eterm*); Eterm big_div(Eterm, Eterm, Eterm*); Eterm big_rem(Eterm, Eterm, Eterm*); -Eterm big_neg(Eterm, Eterm*); -Eterm big_minus_small(Eterm, Uint, Eterm*); Eterm big_plus_small(Eterm, Uint, Eterm*); Eterm big_times_small(Eterm, Uint, Eterm*); diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 71957b2259..5aea917ddc 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -3512,28 +3512,6 @@ void erts_allctr_wrapper_pre_unlock(void) } - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ - * Deprecated functions * - * * - * These functions are still defined since "non-OTP linked in drivers" may * - * contain (illegal) calls to them. * -\* */ - -/* --- DO *NOT* USE THESE FUNCTIONS --- */ - -void *sys_alloc(Uint sz) -{ return erts_alloc_fnf(ERTS_ALC_T_UNDEF, sz); } -void *sys_realloc(void *ptr, Uint sz) -{ return erts_realloc_fnf(ERTS_ALC_T_UNDEF, ptr, sz); } -void sys_free(void *ptr) -{ erts_free(ERTS_ALC_T_UNDEF, ptr); } -void *safe_alloc(Uint sz) -{ return erts_alloc(ERTS_ALC_T_UNDEF, sz); } -void *safe_realloc(void *ptr, Uint sz) -{ return erts_realloc(ERTS_ALC_T_UNDEF, ptr, sz); } - - /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * NOTE: erts_alc_test() is only supposed to be used for testing. * * * diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index 56a3b73bf9..758d529f87 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -173,13 +173,6 @@ __decl_noreturn void erts_realloc_n_enomem(ErtsAlcType_t,void*,Uint) __decl_noreturn void erts_alc_fatal_error(int,int,ErtsAlcType_t,...) __noreturn; -/* --- DO *NOT* USE THESE DEPRECATED FUNCTIONS --- Instead use: */ -void *safe_alloc(Uint) __deprecated; /* erts_alloc() */ -void *safe_realloc(void *, Uint) __deprecated; /* erts_realloc() */ -void sys_free(void *) __deprecated; /* erts_free() */ -void *sys_alloc(Uint ) __deprecated; /* erts_alloc_fnf() */ -void *sys_realloc(void *, Uint) __deprecated; /* erts_realloc_fnf() */ - #undef ERTS_HAVE_IS_IN_LITERAL_RANGE #if defined(ARCH_32) || defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) # define ERTS_HAVE_IS_IN_LITERAL_RANGE diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c index 28aaeeb479..4347f9f2b7 100644 --- a/erts/emulator/beam/erl_cpu_topology.c +++ b/erts/emulator/beam/erl_cpu_topology.c @@ -827,17 +827,6 @@ erts_sched_bind_atfork_child(int unbind) return 0; } -char * -erts_sched_bind_atvfork_child(int unbind) -{ - if (unbind) { - ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx) - || erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); - return erts_get_unbind_from_cpu_str(cpuinfo); - } - return "false"; -} - void erts_sched_bind_atfork_parent(int unbind) { diff --git a/erts/emulator/beam/erl_cpu_topology.h b/erts/emulator/beam/erl_cpu_topology.h index 45324ac4a0..cf139d95a9 100644 --- a/erts/emulator/beam/erl_cpu_topology.h +++ b/erts/emulator/beam/erl_cpu_topology.h @@ -85,22 +85,14 @@ void erts_sched_bind_atthrcreate_parent(int unbind); int erts_sched_bind_atfork_prepare(void); int erts_sched_bind_atfork_child(int unbind); -char *erts_sched_bind_atvfork_child(int unbind); void erts_sched_bind_atfork_parent(int unbind); Eterm erts_fake_scheduler_bindings(Process *p, Eterm how); Eterm erts_debug_cpu_groups_map(Process *c_p, int groups); - typedef void (*erts_cpu_groups_callback_t)(int, ErtsSchedulerData *, int, void *); -void erts_add_cpu_groups(int groups, - erts_cpu_groups_callback_t callback, - void *arg); -void erts_remove_cpu_groups(erts_cpu_groups_callback_t callback, - void *arg); - #endif diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 24b22eafb8..d69035cf64 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1290,11 +1290,6 @@ int db_match_keeps_key(int keypos, Eterm match, Eterm guard, Eterm body) return 0; } -/* This is used when tracing */ -Eterm erts_match_set_lint(Process *p, Eterm matchexpr) { - return db_match_set_lint(p, matchexpr, DCOMP_TRACE); -} - Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags) { Eterm l; diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index a991c2c164..6ec6be5a08 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -439,7 +439,7 @@ Eterm erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity) { return erts_gc_after_bif_call_lhf(p, ERTS_INVALID_HFRAG_PTR, - result, regs, arity); + result, regs, arity); } static ERTS_INLINE void reset_active_writer(Process *p) diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index eaaf5c911a..2527732eb3 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -309,30 +309,6 @@ void erl_error(char *fmt, va_list args) static int early_init(int *argc, char **argv); -void -erts_short_init(void) -{ - - int ncpu; - int time_correction; - ErtsTimeWarpMode time_warp_mode; - - set_default_time_adj(&time_correction, - &time_warp_mode); - ncpu = early_init(NULL, NULL); - erl_init(ncpu, - ERTS_DEFAULT_MAX_PROCESSES, - 0, - ERTS_DEFAULT_MAX_PORTS, - 0, - 0, - time_correction, - time_warp_mode, - ERTS_NODE_TAB_DELAY_GC_DEFAULT, - ERTS_DB_SPNCNT_NORMAL); - erts_initialized = 1; -} - static void erl_init(int ncpu, int proc_tab_sz, diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 4836b9e2d3..55526e1d5e 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -2161,13 +2161,6 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) #endif } -int -erts_port_is_scheduled(Port *pp) -{ - erts_aint32_t flags = erts_smp_atomic32_read_acqb(&pp->sched.flags); - return (flags & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC)) != 0; -} - #ifdef ERTS_SMP void diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index e3550e878e..a48b492ba7 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -265,7 +265,6 @@ int erts_port_task_schedule(Eterm, ErtsPortTaskType, ...); void erts_port_task_free_port(Port *); -int erts_port_is_scheduled(Port *); ErtsProc2PortSigData *erts_port_task_alloc_p2p_sig_data(void); ErtsProc2PortSigData *erts_port_task_alloc_p2p_sig_data_extra(size_t extra, void **extra_ptr); void erts_port_task_free_p2p_sig_data(ErtsProc2PortSigData *sigdp); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 9947e33f47..a594578793 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1537,12 +1537,6 @@ proclist_destroy(ErtsProcList *plp) proclist_free(plp); } -ErtsProcList * -erts_proclist_copy(ErtsProcList *plp) -{ - return proclist_copy(plp); -} - ErtsProcList * erts_proclist_create(Process *p) { @@ -3275,13 +3269,6 @@ thr_prgr_fin_wait(void *vssi) static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp); -void -erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time) -{ - /* TODO only poke when needed (based on timeout_time) */ - erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(-1)); -} - static void * aux_thread(void *unused) { @@ -9315,17 +9302,6 @@ erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, return pid2proc_not_running(c_p, c_p_locks, pid, pid_locks, 0); } -/* - * Like erts_pid2proc_not_running(), but hands over the process - * in a suspended state unless (c_p is looked up). - */ -Process * -erts_pid2proc_suspend(Process *c_p, ErtsProcLocks c_p_locks, - Eterm pid, ErtsProcLocks pid_locks) -{ - return pid2proc_not_running(c_p, c_p_locks, pid, pid_locks, 1); -} - /* * erts_pid2proc_nropt() is normally the same as * erts_pid2proc_not_running(). However it is only @@ -9444,21 +9420,6 @@ handle_pend_bif_async_suspend(Process *suspendee, } } -#else - -/* - * Non-smp version of erts_pid2proc_suspend(). - */ -Process * -erts_pid2proc_suspend(Process *c_p, ErtsProcLocks c_p_locks, - Eterm pid, ErtsProcLocks pid_locks) -{ - Process *rp = erts_pid2proc(c_p, c_p_locks, pid, pid_locks); - if (rp) - erts_suspend(rp, pid_locks, NULL); - return rp; -} - #endif /* ERTS_SMP */ /* diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 5b35dc3c78..d6d7750a33 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1606,7 +1606,6 @@ Uint64 erts_ensure_later_proc_interval(Uint64); Uint64 erts_step_proc_interval(void); ErtsProcList *erts_proclist_create(Process *); -ErtsProcList *erts_proclist_copy(ErtsProcList *); void erts_proclist_destroy(ErtsProcList *); ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *); @@ -2556,10 +2555,6 @@ ERTS_TIME2REDS_IMPL__(ErtsMonotonicTime start, ErtsMonotonicTime end) } #endif -Process *erts_pid2proc_suspend(Process *, - ErtsProcLocks, - Eterm, - ErtsProcLocks); #ifdef ERTS_SMP Process *erts_pid2proc_not_running(Process *, @@ -2601,8 +2596,6 @@ extern int erts_disable_proc_not_running_opt; void erts_smp_notify_inc_runq(ErtsRunQueue *runq); -void erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time); - #ifdef ERTS_SMP void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t); ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index bb4d442240..d6a66ce7cb 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1132,7 +1132,6 @@ extern erts_tid_t erts_main_thread; #endif extern int erts_compat_rel; extern int erts_use_sender_punish; -void erts_short_init(void); void erl_start(int, char**); void erts_usage(void); Eterm erts_preloaded(Process* p); @@ -1439,7 +1438,6 @@ do { \ #define MatchSetGetSource(MPSP) erts_match_set_get_source(MPSP) extern Binary *erts_match_set_compile(Process *p, Eterm matchexpr, Eterm MFA); -Eterm erts_match_set_lint(Process *p, Eterm matchexpr); extern void erts_match_set_release_result(Process* p); ERTS_GLB_INLINE void erts_match_set_release_result_trace(Process* p, Eterm); -- cgit v1.2.3 From a3407eaa2104d634e4400c0c805202da462ee66f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Thu, 18 May 2017 07:17:22 +0200 Subject: Eliminate the -gen_dest macro flag Instructions that take a 'd' argument needs a -gen_dest flag in their macros. For example: %macro:put_list PutList -pack -gen_dest put_list s s d -gen_dest was needed when x(0) was stored in a register, since it is not possible to take the address of a register. Now that x(0) is stored in memory and we can take the address, we can eliminate gen_dest. --- erts/emulator/beam/beam_emu.c | 74 ++++++++++++++++++++----------------------- erts/emulator/beam/ops.tab | 12 +++---- 2 files changed, 41 insertions(+), 45 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index a90e6a0ba8..79d751d13e 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -158,7 +158,9 @@ do { \ /* * Register target (X or Y register). */ -#define REG_TARGET(Target) (*(((Target) & 1) ? &yb(Target-1) : &xb(Target))) + +#define REG_TARGET_PTR(Target) (((Target) & 1) ? &yb(Target-1) : &xb(Target)) +#define REG_TARGET(Target) (*REG_TARGET_PTR(Target)) /* * Store a result into a register given a destination descriptor. @@ -172,8 +174,6 @@ do { \ REG_TARGET(stb_reg) = (Result); \ } while (0) -#define StoreSimpleDest(Src, Dest) Dest = (Src) - /* * Store a result into a register and execute the next instruction. * Dst points to the word with a destination descriptor, which MUST @@ -420,7 +420,7 @@ void** beam_ops; #define TestHeapPutList(Need, Reg) \ do { \ TestHeap((Need), 1); \ - PutList(Reg, r(0), r(0), StoreSimpleDest); \ + PutList(Reg, r(0), r(0)); \ CHECK_TERM(r(0)); \ } while (0) @@ -547,11 +547,11 @@ void** beam_ops; GetR((N)+1, Dst2); \ } while (0) -#define PutList(H, T, Dst, Store) \ - do { \ - HTOP[0] = (H); HTOP[1] = (T); \ - Store(make_list(HTOP), Dst); \ - HTOP += 2; \ +#define PutList(H, T, Dst) \ + do { \ + HTOP[0] = (H); HTOP[1] = (T); \ + Dst = make_list(HTOP); \ + HTOP += 2; \ } while (0) #define Swap(R1, R2) \ @@ -568,11 +568,7 @@ void** beam_ops; R2 = Tmp = V; \ } while (0) -#define Move(Src, Dst, Store) \ - do { \ - Eterm term = (Src); \ - Store(term, Dst); \ - } while (0) +#define Move(Src, Dst) Dst = (Src) #define Move2Par(S1, D1, S2, D2) \ do { \ @@ -911,7 +907,7 @@ do { \ Target = _uint_size * Unit; \ } while (0) -#define BsGetFloat2(Ms, Live, Sz, Flags, Dst, Store, Fail) \ +#define BsGetFloat2(Ms, Live, Sz, Flags, Dst, Fail) \ do { \ ErlBinMatchBuffer *_mb; \ Eterm _result; Sint _size; \ @@ -922,12 +918,12 @@ do { \ LIGHT_SWAPOUT; \ _result = erts_bs_get_float_2(c_p, _size, (Flags), _mb); \ LIGHT_SWAPIN; \ - HEAP_SPACE_VERIFIED(0); \ + HEAP_SPACE_VERIFIED(0); \ if (is_non_value(_result)) { Fail; } \ - else { Store(_result, Dst); } \ + else { Dst = _result; } \ } while (0) -#define BsGetBinaryImm_2(Ms, Live, Sz, Flags, Dst, Store, Fail) \ +#define BsGetBinaryImm_2(Ms, Live, Sz, Flags, Dst, Fail) \ do { \ ErlBinMatchBuffer *_mb; \ Eterm _result; \ @@ -936,12 +932,12 @@ do { \ LIGHT_SWAPOUT; \ _result = erts_bs_get_binary_2(c_p, (Sz), (Flags), _mb); \ LIGHT_SWAPIN; \ - HEAP_SPACE_VERIFIED(0); \ + HEAP_SPACE_VERIFIED(0); \ if (is_non_value(_result)) { Fail; } \ - else { Store(_result, Dst); } \ + else { Dst = _result; } \ } while (0) -#define BsGetBinary_2(Ms, Live, Sz, Flags, Dst, Store, Fail) \ +#define BsGetBinary_2(Ms, Live, Sz, Flags, Dst, Fail) \ do { \ ErlBinMatchBuffer *_mb; \ Eterm _result; Uint _size; \ @@ -951,27 +947,27 @@ do { \ LIGHT_SWAPOUT; \ _result = erts_bs_get_binary_2(c_p, _size, (Flags), _mb); \ LIGHT_SWAPIN; \ - HEAP_SPACE_VERIFIED(0); \ + HEAP_SPACE_VERIFIED(0); \ if (is_non_value(_result)) { Fail; } \ - else { Store(_result, Dst); } \ + else { Dst = _result; } \ } while (0) -#define BsGetBinaryAll_2(Ms, Live, Unit, Dst, Store, Fail) \ - do { \ - ErlBinMatchBuffer *_mb; \ - Eterm _result; \ - TestHeap(ERL_SUB_BIN_SIZE, Live); \ - _mb = ms_matchbuffer(Ms); \ - if (((_mb->size - _mb->offset) % Unit) == 0) { \ - LIGHT_SWAPOUT; \ - _result = erts_bs_get_binary_all_2(c_p, _mb); \ - LIGHT_SWAPIN; \ - HEAP_SPACE_VERIFIED(0); \ - ASSERT(is_value(_result)); \ - Store(_result, Dst); \ - } else { \ - HEAP_SPACE_VERIFIED(0); \ - Fail; } \ +#define BsGetBinaryAll_2(Ms, Live, Unit, Dst, Fail) \ + do { \ + ErlBinMatchBuffer *_mb; \ + Eterm _result; \ + TestHeap(ERL_SUB_BIN_SIZE, Live); \ + _mb = ms_matchbuffer(Ms); \ + if (((_mb->size - _mb->offset) % Unit) == 0) { \ + LIGHT_SWAPOUT; \ + _result = erts_bs_get_binary_all_2(c_p, _mb); \ + LIGHT_SWAPIN; \ + HEAP_SPACE_VERIFIED(0); \ + ASSERT(is_value(_result)); \ + Dst = _result; \ + } else { \ + HEAP_SPACE_VERIFIED(0); \ + Fail; } \ } while (0) #define BsSkipBits2(Ms, Bits, Unit, Fail) \ diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 9b5bd7a749..896ab82a50 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -409,7 +409,7 @@ move3 x x x x x x move S=n D=y => init D move S=c D=y => move S x | move x D -%macro:move Move -pack -gen_dest +%macro:move Move -pack move x x move x y move y x @@ -538,7 +538,7 @@ i_put_tuple y I # put_list Const=c n Dst => move Const x | put_list x n Dst -%macro:put_list PutList -pack -gen_dest +%macro:put_list PutList -pack put_list x n x put_list y n x @@ -1209,9 +1209,9 @@ i_bs_get_integer_32 x f I d bs_get_binary2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \ gen_get_binary2(Fail, Ms, Live, Sz, Unit, Flags, Dst) -%macro: i_bs_get_binary_imm2 BsGetBinaryImm_2 -fail_action -gen_dest -%macro: i_bs_get_binary2 BsGetBinary_2 -fail_action -gen_dest -%macro: i_bs_get_binary_all2 BsGetBinaryAll_2 -fail_action -gen_dest +%macro: i_bs_get_binary_imm2 BsGetBinaryImm_2 -fail_action +%macro: i_bs_get_binary2 BsGetBinary_2 -fail_action +%macro: i_bs_get_binary_all2 BsGetBinaryAll_2 -fail_action i_bs_get_binary_imm2 f x I I I d i_bs_get_binary2 f x I s I d @@ -1224,7 +1224,7 @@ bs_get_float2 Fail=f Ms=x Live=u Sz=s Unit=u Flags=u Dst=d => \ bs_get_float2 Fail=f Ms=x Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail -%macro: i_bs_get_float2 BsGetFloat2 -fail_action -gen_dest +%macro: i_bs_get_float2 BsGetFloat2 -fail_action i_bs_get_float2 f x I s I d # Miscellanous -- cgit v1.2.3 From 29aea1a92e25ee47e55b4a3c48edfc62e8aa4a42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Thu, 18 May 2017 12:24:07 +0200 Subject: Allow multiple types per argument for specific instructions Inroduce syntactic sugar so that we can write: get_list xy xy xy instead of: get_list x x x get_list x x y get_list x y x get_list x y y get_list y x x get_list y x y get_list y y x get_list y y y --- erts/emulator/beam/ops.tab | 116 +++++++++++++-------------------------------- 1 file changed, 32 insertions(+), 84 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 896ab82a50..8abe871c14 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -182,15 +182,7 @@ i_jump_on_val x f I I i_jump_on_val y f I I %macro: get_list GetList -pack -get_list x x x -get_list x x y -get_list x y x -get_list x y y - -get_list y x x -get_list y x y -get_list y y x -get_list y y y +get_list xy xy xy # The following get_list instructions using x(0) are frequently used. get_list r x x @@ -218,12 +210,10 @@ set_tuple_element s d P # Get tuple element %macro: i_get_tuple_element GetTupleElement -pack -i_get_tuple_element x P x -i_get_tuple_element y P x +i_get_tuple_element xy P x %cold -i_get_tuple_element x P y -i_get_tuple_element y P y +i_get_tuple_element xy P y %hot %macro: i_get_tuple_element2 GetTupleElement2 -pack @@ -270,11 +260,7 @@ system_limit j move C=cxy x==0 | jump Lbl => move_jump Lbl C %macro: move_jump MoveJump -nonext -move_jump f n -move_jump f c -move_jump f x -move_jump f y - +move_jump f ncxy # Movement to and from the stack is common # Try to pack as much as we can into one instruction @@ -326,12 +312,10 @@ swap_temp R1 R2 Tmp | line Loc | call_ext_last Live Addr D | \ is_killed(Tmp, Live) => swap R1 R2 | line Loc | call_ext_last Live Addr D %macro: swap_temp SwapTemp -pack -swap_temp x x x -swap_temp x y x +swap_temp x xy x %macro: swap Swap -pack -swap x x -swap x y +swap x xy move Src=x D1=x | move Src=x D2=x => move_dup Src D1 D2 move Src=x SD=x | move SD=x D=x => move_dup Src SD D @@ -381,10 +365,7 @@ move_shift x y x move_shift x x y %macro: move_dup MoveDup -pack -move_dup x x x -move_dup x x y -move_dup y x x -move_dup y x y +move_dup xy x xy %macro: move2_par Move2Par -pack @@ -483,8 +464,7 @@ i_is_ne_exact_literal f y c is_eq_exact Lbl Y=y X=x => is_eq_exact Lbl X Y %macro: is_eq_exact EqualExact -fail_action -pack -is_eq_exact f x x -is_eq_exact f x y +is_eq_exact f x xy is_eq_exact f s s %macro: is_lt IsLessThan -fail_action @@ -630,9 +610,7 @@ is_tagged_tuple Fail=f c Arity Atom => jump Fail %macro:is_tagged_tuple IsTaggedTuple -fail_action -is_tagged_tuple f r A a -is_tagged_tuple f x A a -is_tagged_tuple f y A a +is_tagged_tuple f rxy A a # Test tuple & arity (head) @@ -642,21 +620,16 @@ is_tuple Fail=f S=xy | test_arity Fail=f S=xy Arity => is_tuple_of_arity Fail S %macro:is_tuple_of_arity IsTupleOfArity -fail_action -is_tuple_of_arity f r A -is_tuple_of_arity f x A -is_tuple_of_arity f y A +is_tuple_of_arity f rxy A %macro: is_tuple IsTuple -fail_action -is_tuple f r -is_tuple f x -is_tuple f y +is_tuple f rxy test_arity Fail Literal=q Arity => move Literal x | test_arity Fail x Arity test_arity Fail=f c Arity => jump Fail %macro: test_arity IsArity -fail_action -test_arity f x A -test_arity f y A +test_arity f xy A get_tuple_element Reg=x P1 D1=x | get_tuple_element Reg=x P2 D2=x | \ get_tuple_element Reg=x P3 D3=x | \ @@ -681,8 +654,7 @@ is_integer Fail=f S=x | allocate Need Regs => is_integer_allocate Fail S Need Re is_integer_allocate f x I I %macro: is_integer IsInteger -fail_action -is_integer f x -is_integer f y +is_integer f xy is_list Fail=f n => is_list Fail Literal=q => move Literal x | is_list Fail x @@ -696,8 +668,7 @@ is_list f y is_nonempty_list Fail=f S=x | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs %macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action -pack -is_nonempty_list_allocate f r I t -is_nonempty_list_allocate f x I t +is_nonempty_list_allocate f rx I t is_nonempty_list F=f x==0 | test_heap I1 I2 => is_non_empty_list_test_heap F I1 I2 @@ -708,12 +679,10 @@ is_nonempty_list Fail=f S=x | get_list S D1=x D2=x => \ is_nonempty_list_get_list Fail S D1 D2 %macro: is_nonempty_list_get_list IsNonemptyListGetList -fail_action -pack -is_nonempty_list_get_list f r x x -is_nonempty_list_get_list f x x x +is_nonempty_list_get_list f rx x x %macro: is_nonempty_list IsNonemptyList -fail_action -is_nonempty_list f x -is_nonempty_list f y +is_nonempty_list f xy %macro: is_atom IsAtom -fail_action is_atom f x @@ -735,8 +704,7 @@ is_nil Fail=f n => is_nil Fail=f qia => jump Fail %macro: is_nil IsNil -fail_action -is_nil f x -is_nil f y +is_nil f xy is_binary Fail Literal=q => move Literal x | is_binary Fail x is_binary Fail=f c => jump Fail @@ -784,8 +752,7 @@ is_boolean Fail=f ac => jump Fail %cold %macro: is_boolean IsBoolean -fail_action -is_boolean f x -is_boolean f y +is_boolean f xy %hot is_function2 Fail=f acq Arity => jump Fail @@ -1079,8 +1046,7 @@ i_get_hash c I d i_get s d %macro: self Self -self x -self y +self xy %macro: node Node node x @@ -1091,8 +1057,7 @@ node y i_fast_element j x I d i_fast_element j y I d -i_element j x s d -i_element j y s d +i_element j xy s d bif1 f b s d bif1_body b s d @@ -1111,8 +1076,7 @@ i_move_call c f %macro:move_call MoveCall -arg_f -size -nonext move_call/2 -move_call x f -move_call y f +move_call xy f move S=c x==0 | call_last Ar P=f D => i_move_call_last P D S move S x==0 | call_last Ar P=f D => move_call_last S P D @@ -1122,8 +1086,7 @@ i_move_call_last f P c %macro:move_call_last MoveCallLast -arg_f -nonext -pack move_call_last/3 -move_call_last x f Q -move_call_last y f Q +move_call_last xy f Q move S=c x==0 | call_only Ar P=f => i_move_call_only P S move S=x x==0 | call_only Ar P=f => move_call_only S P @@ -1167,8 +1130,7 @@ i_make_fun I t %hot %macro: is_function IsFunction -fail_action -is_function f x -is_function f y +is_function f xy is_function Fail=f c => jump Fail func_info M F A => i_func_info u M F A @@ -1180,8 +1142,7 @@ func_info M F A => i_func_info u M F A %cold bs_start_match2 Fail=f ica X Y D => jump Fail bs_start_match2 Fail Bin X Y D => i_bs_start_match2 Bin Fail X Y D -i_bs_start_match2 x f I I d -i_bs_start_match2 y f I I d +i_bs_start_match2 xy f I I d bs_save2 Reg Index => gen_bs_save(Reg, Index) i_bs_save2 x I @@ -1236,8 +1197,7 @@ bs_skip_bits2 Fail=f Ms=x Sz=sq Unit=u Flags=u => \ i_bs_skip_bits_imm2 f x I %macro: i_bs_skip_bits2 BsSkipBits2 -fail_action -i_bs_skip_bits2 f x x I -i_bs_skip_bits2 f x y I +i_bs_skip_bits2 f x xy I %macro: i_bs_skip_bits_all2 BsSkipBitsAll2 -fail_action i_bs_skip_bits_all2 f x I @@ -1302,8 +1262,7 @@ bs_init2 Fail Sz Words=u==0 Regs Flags Dst => \ bs_init2 Fail Sz Words Regs Flags Dst => \ i_bs_init_fail_heap Sz Words Fail Regs Dst -i_bs_init_fail x j I d -i_bs_init_fail y j I d +i_bs_init_fail xy j I d i_bs_init_fail_heap s I j I d @@ -1324,8 +1283,7 @@ bs_init_bits Fail Sz Words=u==0 Regs Flags Dst => \ bs_init_bits Fail Sz Words Regs Flags Dst => \ i_bs_init_bits_fail_heap Sz Words Fail Regs Dst -i_bs_init_bits_fail x j I d -i_bs_init_bits_fail y j I d +i_bs_init_bits_fail xy j I d i_bs_init_bits_fail_heap s I j I d @@ -1493,8 +1451,7 @@ is_map Fail Lit=q | literal_is_map(Lit) => is_map Fail cq => jump Fail %macro: is_map IsMap -fail_action -is_map f x -is_map f y +is_map f xy ## Transform has_map_fields #{ K1 := _, K2 := _ } to has_map_elements @@ -1514,16 +1471,10 @@ i_get_map_element Fail Src=xy Key=y Dst => \ move Key x | i_get_map_element Fail Src x Dst %macro: i_get_map_element_hash GetMapElementHash -fail_action -i_get_map_element_hash f x c I x -i_get_map_element_hash f y c I x -i_get_map_element_hash f x c I y -i_get_map_element_hash f y c I y +i_get_map_element_hash f xy c I xy %macro: i_get_map_element GetMapElement -fail_action -i_get_map_element f x x x -i_get_map_element f y x x -i_get_map_element f x x y -i_get_map_element f y x y +i_get_map_element f xy x xy # # Convert the plus operations to a generic plus instruction. @@ -1589,12 +1540,9 @@ gc_bif2 Fail Live u$bif:erlang:bxor/2 S1 S2 Dst => \ gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst -i_increment r I I d -i_increment x I I d -i_increment y I I d +i_increment rxy I I d -i_plus j I x x d -i_plus j I x y d +i_plus j I x xy d i_plus j I s s d i_minus j I x x d -- cgit v1.2.3 From 32ea8ba368c455afba07afd85bed6fb57879f56d Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 18 May 2017 20:42:04 +0200 Subject: Make statistics/1 aware of dirty run-queues and tasks --- erts/emulator/beam/atom.names | 4 + erts/emulator/beam/erl_bif_info.c | 34 +++++-- erts/emulator/beam/erl_process.c | 188 ++++++++++++++++++++++++++++---------- erts/emulator/beam/erl_process.h | 23 +++-- 4 files changed, 181 insertions(+), 68 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 477a7676d6..a44d23b181 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -76,6 +76,7 @@ atom ac atom accessor atom active atom active_tasks +atom active_tasks_all atom alive atom all atom all_but_first @@ -567,6 +568,7 @@ atom return_to atom return_trace atom run_queue atom run_queue_lengths +atom run_queue_lengths_all atom runnable atom runnable_ports atom runnable_procs @@ -656,8 +658,10 @@ atom Times='*' atom timestamp atom total atom total_active_tasks +atom total_active_tasks_all atom total_heap_size atom total_run_queue_lengths +atom total_run_queue_lengths_all atom tpkt atom trace trace_ts traced atom trace_control_word diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 5fc70dfc02..e2773475b0 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -3450,9 +3450,15 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) if (is_non_value(res)) BIF_RET(am_undefined); BIF_TRAP1(gather_sched_wall_time_res_trap, BIF_P, res); - } else if (BIF_ARG_1 == am_total_active_tasks - || BIF_ARG_1 == am_total_run_queue_lengths) { - Uint no = erts_run_queues_len(NULL, 0, BIF_ARG_1 == am_total_active_tasks); + } else if ((BIF_ARG_1 == am_total_active_tasks) + | (BIF_ARG_1 == am_total_run_queue_lengths) + | (BIF_ARG_1 == am_total_active_tasks_all) + | (BIF_ARG_1 == am_total_run_queue_lengths_all)) { + Uint no = erts_run_queues_len(NULL, 0, + ((BIF_ARG_1 == am_total_active_tasks) + | (BIF_ARG_1 == am_total_active_tasks_all)), + ((BIF_ARG_1 == am_total_active_tasks_all) + | (BIF_ARG_1 == am_total_run_queue_lengths_all))); if (IS_USMALL(0, no)) res = make_small(no); else { @@ -3460,13 +3466,21 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) res = uint_to_big(no, hp); } BIF_RET(res); - } else if (BIF_ARG_1 == am_active_tasks - || BIF_ARG_1 == am_run_queue_lengths) { + } else if ((BIF_ARG_1 == am_active_tasks) + | (BIF_ARG_1 == am_run_queue_lengths) + | (BIF_ARG_1 == am_active_tasks_all) + | (BIF_ARG_1 == am_run_queue_lengths_all)) { Eterm res, *hp, **hpp; Uint sz, *szp; - int no_qs = erts_no_run_queues; + int incl_dirty_io = ((BIF_ARG_1 == am_active_tasks_all) + | (BIF_ARG_1 == am_run_queue_lengths_all)); + int no_qs = (erts_no_run_queues + ERTS_NUM_DIRTY_CPU_RUNQS + + (incl_dirty_io ? ERTS_NUM_DIRTY_IO_RUNQS : 0)); Uint *qszs = erts_alloc(ERTS_ALC_T_TMP,sizeof(Uint)*no_qs*2); - (void) erts_run_queues_len(qszs, 0, BIF_ARG_1 == am_active_tasks); + (void) erts_run_queues_len(qszs, 0, + ((BIF_ARG_1 == am_active_tasks) + | (BIF_ARG_1 == am_active_tasks_all)), + incl_dirty_io); sz = 0; szp = &sz; hpp = NULL; @@ -3539,7 +3553,7 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) res = TUPLE2(hp, b1, b2); BIF_RET(res); } else if (BIF_ARG_1 == am_run_queue) { - res = erts_run_queues_len(NULL, 1, 0); + res = erts_run_queues_len(NULL, 1, 0, 0); BIF_RET(make_small(res)); } else if (BIF_ARG_1 == am_wall_clock) { UWord w1, w2; @@ -3557,9 +3571,9 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) { Eterm res, *hp, **hpp; Uint sz, *szp; - int no_qs = erts_no_run_queues; + int no_qs = erts_no_run_queues + ERTS_NUM_DIRTY_RUNQS; Uint *qszs = erts_alloc(ERTS_ALC_T_TMP,sizeof(Uint)*no_qs*2); - (void) erts_run_queues_len(qszs, 0, 0); + (void) erts_run_queues_len(qszs, 0, 0, 1); sz = 0; szp = &sz; hpp = NULL; diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index a56ef4aedb..6f96833b21 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -405,13 +405,59 @@ static erts_atomic_t runq_supervisor_sleeping; ErtsSchedulerData *erts_scheduler_data; #endif -ErtsAlignedRunQueue *erts_aligned_run_queues; -Uint erts_no_run_queues; +ErtsAlignedRunQueue * ERTS_WRITE_UNLIKELY(erts_aligned_run_queues); +Uint ERTS_WRITE_UNLIKELY(erts_no_run_queues); -ErtsAlignedSchedulerData *erts_aligned_scheduler_data; #ifdef ERTS_DIRTY_SCHEDULERS -ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data; -ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data; + +struct { + union { + erts_smp_atomic32_t active; + char align__[ERTS_CACHE_LINE_SIZE]; + } cpu; + union { + erts_smp_atomic32_t active; + char align__[ERTS_CACHE_LINE_SIZE]; + } io; +} dirty_count erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +#endif + +static ERTS_INLINE void +dirty_active(ErtsSchedulerData *esdp, erts_aint32_t add) +{ +#ifdef ERTS_DIRTY_SCHEDULERS + erts_aint32_t val; + erts_smp_atomic32_t *ap; + switch (esdp->type) { + case ERTS_SCHED_DIRTY_CPU: + ap = &dirty_count.cpu.active; + break; + case ERTS_SCHED_DIRTY_IO: + ap = &dirty_count.io.active; + break; + default: + ap = NULL; + ERTS_INTERNAL_ERROR("Not a dirty scheduler"); + break; + } + + /* + * All updates done under run-queue lock, so + * no inc or dec needed... + */ + ERTS_SMP_ASSERT(erts_smp_lc_runq_is_locked(esdp->run_queue)); + + val = erts_smp_atomic32_read_nob(ap); + val += add; + erts_smp_atomic32_set_nob(ap, val); +#endif +} + +ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_scheduler_data); +#ifdef ERTS_DIRTY_SCHEDULERS +ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_cpu_scheduler_data); +ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_io_scheduler_data); typedef union { Process dsp; char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(Process))]; @@ -539,22 +585,28 @@ do { \ } \ } while (0) -#define ERTS_ATOMIC_FOREACH_RUNQ_X(RQVAR, DO, DOX) \ +#define ERTS_ATOMIC_FOREACH_RUNQ_X(RQVAR, NRQS, DO, DOX) \ do { \ ErtsRunQueue *RQVAR; \ + int nrqs = (NRQS); \ int ix__; \ - for (ix__ = 0; ix__ < erts_no_run_queues; ix__++) { \ + for (ix__ = 0; ix__ < nrqs; ix__++) { \ RQVAR = ERTS_RUNQ_IX(ix__); \ erts_smp_runq_lock(RQVAR); \ { DO; } \ } \ { DOX; } \ - for (ix__ = 0; ix__ < erts_no_run_queues; ix__++) \ + for (ix__ = 0; ix__ < nrqs; ix__++) \ erts_smp_runq_unlock(ERTS_RUNQ_IX(ix__)); \ } while (0) -#define ERTS_ATOMIC_FOREACH_RUNQ(RQVAR, DO) \ - ERTS_ATOMIC_FOREACH_RUNQ_X(RQVAR, DO, ) +#define ERTS_ATOMIC_FOREACH_RUNQ(RQVAR, DO) \ + ERTS_ATOMIC_FOREACH_RUNQ_X(RQVAR, erts_no_run_queues + ERTS_NUM_DIRTY_RUNQS, DO, ) + +#define ERTS_ATOMIC_FOREACH_NORMAL_RUNQ(RQVAR, DO) \ + ERTS_ATOMIC_FOREACH_RUNQ_X(RQVAR, erts_no_run_queues, DO, ) + + /* * Local functions. */ @@ -2977,7 +3029,7 @@ erts_active_schedulers(void) { Uint as = erts_no_schedulers; - ERTS_ATOMIC_FOREACH_RUNQ(rq, as -= abs(rq->waiting)); + ERTS_ATOMIC_FOREACH_NORMAL_RUNQ(rq, as -= abs(rq->waiting)); return as; } @@ -3396,6 +3448,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) rq->sleepers.list->prev = ssi; rq->sleepers.list = ssi; erts_smp_spin_unlock(&rq->sleepers.lock); + dirty_active(esdp, -1); } #endif @@ -3737,6 +3790,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) sched_active_sys(esdp->no, rq); } + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) + dirty_active(esdp, 1); + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); } @@ -6136,7 +6192,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online #endif ) { - int ix, n, no_ssi; + int ix, n, no_ssi, tot_rqs; char *daww_ptr; size_t daww_sz; size_t size_runqs; @@ -6169,26 +6225,19 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online /* Create and initialize run queues */ n = no_schedulers; - size_runqs = sizeof(ErtsAlignedRunQueue) * (n + ERTS_NUM_DIRTY_RUNQS); + tot_rqs = (n + ERTS_NUM_DIRTY_RUNQS); + size_runqs = sizeof(ErtsAlignedRunQueue) * tot_rqs; erts_aligned_run_queues = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS, size_runqs); #ifdef ERTS_SMP -#ifdef ERTS_DIRTY_SCHEDULERS - erts_aligned_run_queues += ERTS_NUM_DIRTY_RUNQS; -#endif erts_smp_atomic32_init_nob(&no_empty_run_queues, 0); #endif erts_no_run_queues = n; - for (ix = -(ERTS_NUM_DIRTY_RUNQS); ix < n; ix++) { + for (ix = 0; ix < tot_rqs; ix++) { int pix, rix; -#ifdef ERTS_DIRTY_SCHEDULERS - ErtsRunQueue *rq = ERTS_RUNQ_IX_IS_DIRTY(ix) ? - ERTS_DIRTY_RUNQ_IX(ix) : ERTS_RUNQ_IX(ix); -#else ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); -#endif rq->ix = ix; @@ -6461,6 +6510,11 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online ERTS_SCHED_DIRTY_IO, no_dirty_io_schedulers); + erts_smp_atomic32_init_nob(&dirty_count.cpu.active, + (erts_aint32_t) no_dirty_cpu_schedulers); + erts_smp_atomic32_init_nob(&dirty_count.io.active, + (erts_aint32_t) no_dirty_io_schedulers); + #endif if (set_schdlr_sspnd_change_flags) @@ -7835,6 +7889,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) #endif if (sched_type != ERTS_SCHED_NORMAL) { + dirty_active(esdp, -1); erts_smp_runq_unlock(esdp->run_queue); dirty_sched_wall_time_change(esdp, 0); } @@ -8143,7 +8198,9 @@ suspend_scheduler(ErtsSchedulerData *esdp) erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); - if (sched_type == ERTS_SCHED_NORMAL) { + if (sched_type != ERTS_SCHED_NORMAL) + dirty_active(esdp, 1); + else { schedule_bound_processes(esdp->run_queue, &sbp); erts_sched_check_cpu_bind_post_suspend(esdp); @@ -9803,38 +9860,69 @@ erts_internal_is_process_executing_dirty_1(BIF_ALIST_1) BIF_RET(am_false); } +static ERTS_INLINE void +run_queues_len_aux(ErtsRunQueue *rq, Uint *tot_len, Uint *qlen, int *ip, int incl_active_sched, int locked) +{ + Sint rq_len; + + if (locked) + rq_len = (Sint) erts_smp_atomic32_read_dirty(&rq->len); + else + rq_len = (Sint) erts_smp_atomic32_read_nob(&rq->len); + ASSERT(rq_len >= 0); + + if (incl_active_sched) { +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) { + erts_aint32_t dcnt; + if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(rq)) { + dcnt = erts_smp_atomic32_read_nob(&dirty_count.cpu.active); + ASSERT(0 <= dcnt && dcnt <= erts_no_dirty_cpu_schedulers); + } + else { + ASSERT(ERTS_RUNQ_IS_DIRTY_IO_RUNQ(rq)); + dcnt = erts_smp_atomic32_read_nob(&dirty_count.io.active); + ASSERT(0 <= dcnt && dcnt <= erts_no_dirty_io_schedulers); + } + rq_len += (Sint) dcnt; + } + else +#endif + { + if (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC) + rq_len++; + } + } + if (qlen) + qlen[(*ip)++] = rq_len; + *tot_len += (Uint) rq_len; +} Uint -erts_run_queues_len(Uint *qlen, int atomic_queues_read, int incl_active_sched) +erts_run_queues_len(Uint *qlen, int atomic_queues_read, int incl_active_sched, + int incl_dirty_io) { - int i = 0; + int i = 0, j = 0; Uint len = 0; - if (atomic_queues_read) - ERTS_ATOMIC_FOREACH_RUNQ(rq, - { - Sint rq_len = (Sint) erts_smp_atomic32_read_dirty(&rq->len); - ASSERT(rq_len >= 0); - if (incl_active_sched - && (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC)) { - rq_len++; - } - if (qlen) - qlen[i++] = rq_len; - len += (Uint) rq_len; - } - ); + int no_rqs = erts_no_run_queues; + +#ifdef ERTS_DIRTY_SCHEDULERS + if (incl_dirty_io) + no_rqs += ERTS_NUM_DIRTY_RUNQS; + else + no_rqs += ERTS_NUM_DIRTY_CPU_RUNQS; +#endif + + if (atomic_queues_read) { + ERTS_ATOMIC_FOREACH_RUNQ_X(rq, no_rqs, + run_queues_len_aux(rq, &len, qlen, &j, + incl_active_sched, 1), + /* Nothing... */); + } else { - for (i = 0; i < erts_no_run_queues; i++) { + for (i = 0; i < no_rqs; i++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(i); - Sint rq_len = (Sint) erts_smp_atomic32_read_nob(&rq->len); - ASSERT(rq_len >= 0); - if (incl_active_sched - && (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC)) { - rq_len++; - } - if (qlen) - qlen[i] = rq_len; - len += (Uint) rq_len; + run_queues_len_aux(rq, &len, qlen, &j, incl_active_sched, 0); } } @@ -12136,6 +12224,8 @@ erts_get_total_reductions(Uint *redsp, Uint *diffp) Uint reds = 0; ERTS_ATOMIC_FOREACH_RUNQ_X(rq, + erts_no_run_queues + ERTS_NUM_DIRTY_RUNQS, + reds += rq->procs.reductions, if (redsp) *redsp = reds; diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 6113c8aa05..aa88ff26e8 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1541,24 +1541,29 @@ extern int erts_system_profile_ts_type; } while (0) #if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP) -#define ERTS_NUM_DIRTY_RUNQS 2 +#define ERTS_NUM_DIRTY_CPU_RUNQS 1 +#define ERTS_NUM_DIRTY_IO_RUNQS 1 #else -#define ERTS_NUM_DIRTY_RUNQS 0 +#define ERTS_NUM_DIRTY_CPU_RUNQS 0 +#define ERTS_NUM_DIRTY_IO_RUNQS 0 #endif +#define ERTS_NUM_DIRTY_RUNQS (ERTS_NUM_DIRTY_CPU_RUNQS+ERTS_NUM_DIRTY_IO_RUNQS) + #define ERTS_RUNQ_IX(IX) \ - (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues), \ + (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues+ERTS_NUM_DIRTY_RUNQS), \ &erts_aligned_run_queues[(IX)].runq) #ifdef ERTS_DIRTY_SCHEDULERS #define ERTS_RUNQ_IX_IS_DIRTY(IX) \ - (-(ERTS_NUM_DIRTY_RUNQS) <= (IX) && (IX) < 0) + (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues+ERTS_NUM_DIRTY_RUNQS), \ + (erts_no_run_queues <= (IX))) #define ERTS_DIRTY_RUNQ_IX(IX) \ (ASSERT(ERTS_RUNQ_IX_IS_DIRTY(IX)), \ &erts_aligned_run_queues[(IX)].runq) -#define ERTS_DIRTY_CPU_RUNQ (&erts_aligned_run_queues[-1].runq) -#define ERTS_DIRTY_IO_RUNQ (&erts_aligned_run_queues[-2].runq) -#define ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(RQ) ((RQ)->ix == -1) -#define ERTS_RUNQ_IS_DIRTY_IO_RUNQ(RQ) ((RQ)->ix == -2) +#define ERTS_DIRTY_CPU_RUNQ (&erts_aligned_run_queues[erts_no_run_queues].runq) +#define ERTS_DIRTY_IO_RUNQ (&erts_aligned_run_queues[erts_no_run_queues+1].runq) +#define ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(RQ) ((RQ) == ERTS_DIRTY_CPU_RUNQ) +#define ERTS_RUNQ_IS_DIRTY_IO_RUNQ(RQ) ((RQ) == ERTS_DIRTY_IO_RUNQ) #else #define ERTS_RUNQ_IX_IS_DIRTY(IX) 0 #endif @@ -1837,7 +1842,7 @@ Uint erts_active_schedulers(void); void erts_init_process(int, int, int); Eterm erts_process_state2status(erts_aint32_t); Eterm erts_process_status(Process *, Eterm); -Uint erts_run_queues_len(Uint *, int, int); +Uint erts_run_queues_len(Uint *, int, int, int); void erts_add_to_runq(Process *); Eterm erts_bound_schedulers_term(Process *c_p); Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which); -- cgit v1.2.3 From 13fe88d704d2cab0f4c369447117d552ae15a7c4 Mon Sep 17 00:00:00 2001 From: Salikhov Dinislam Date: Mon, 8 May 2017 15:39:30 +0300 Subject: erts: Remove unused functions from erl_cpu_topology --- erts/emulator/beam/erl_cpu_topology.c | 57 ----------------------------------- 1 file changed, 57 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c index 4347f9f2b7..d0fd763798 100644 --- a/erts/emulator/beam/erl_cpu_topology.c +++ b/erts/emulator/beam/erl_cpu_topology.c @@ -2243,45 +2243,6 @@ add_cpu_groups(int groups, return cgm; } -static void -remove_cpu_groups(erts_cpu_groups_callback_t callback, void *arg) -{ - erts_cpu_groups_map_t *prev_cgm, *cgm; - erts_cpu_groups_callback_list_t *prev_cgcl, *cgcl; - - ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); - - no_cpu_groups_callbacks--; - - prev_cgm = NULL; - for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) { - prev_cgcl = NULL; - for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) { - if (cgcl->callback == callback && cgcl->arg == arg) { - if (prev_cgcl) - prev_cgcl->next = cgcl->next; - else - cgm->callback_list = cgcl->next; - erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgcl); - if (!cgm->callback_list) { - if (prev_cgm) - prev_cgm->next = cgm->next; - else - cpu_groups_maps = cgm->next; - if (cgm->array) - erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgm->array); - erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgm); - } - return; - } - prev_cgcl = cgcl; - } - prev_cgm = cgm; - } - - erts_exit(ERTS_ABORT_EXIT, "Cpu groups not found\n"); -} - static int cpu_groups_lookup(erts_cpu_groups_map_t *map, ErtsSchedulerData *esdp) @@ -2321,21 +2282,3 @@ update_cpu_groups_maps(void) for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) make_cpu_groups_map(cgm, 0); } - -void -erts_add_cpu_groups(int groups, - erts_cpu_groups_callback_t callback, - void *arg) -{ - erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); - add_cpu_groups(groups, callback, arg); - erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); -} - -void erts_remove_cpu_groups(erts_cpu_groups_callback_t callback, - void *arg) -{ - erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); - remove_cpu_groups(callback, arg); - erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); -} -- cgit v1.2.3 From 51e4cc4097c1937873fcaf5968ddfa37952ba863 Mon Sep 17 00:00:00 2001 From: Salikhov Dinislam Date: Mon, 8 May 2017 15:13:59 +0300 Subject: erts: Make allocator functions static --- erts/emulator/beam/erl_alloc_util.c | 12 ++++++------ erts/emulator/beam/erl_alloc_util.h | 7 ------- 2 files changed, 6 insertions(+), 13 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 4889fac923..6fddba4b34 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -821,7 +821,7 @@ static void clear_literal_range(void* start, Uint size) #if HAVE_ERTS_MSEG -void* +static void* erts_alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) { void *res; @@ -832,7 +832,7 @@ erts_alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) return res; } -void* +static void* erts_alcu_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p) { @@ -845,7 +845,7 @@ erts_alcu_mseg_realloc(Allctr_t *allctr, void *seg, return res; } -void +static void erts_alcu_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) { erts_mseg_dealloc_opt(allctr->alloc_no, seg, (UWord) size, flags, &allctr->mseg_opt); @@ -996,7 +996,7 @@ erts_alcu_exec_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) #endif /* HAVE_ERTS_MSEG */ -void* +static void* erts_alcu_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign) { void *res; @@ -1013,7 +1013,7 @@ erts_alcu_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign) return res; } -void* +static void* erts_alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign) { void *res; @@ -1035,7 +1035,7 @@ erts_alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, return res; } -void +static void erts_alcu_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign) { #if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index f570703f25..7a96bd0319 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -195,10 +195,6 @@ extern UWord erts_literal_vspace_map[]; # define ERTS_VSPACE_WORD_BITS (sizeof(UWord)*8) #endif -void* erts_alcu_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); -void* erts_alcu_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); -void erts_alcu_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); - #if HAVE_ERTS_MSEG # if defined(ARCH_32) void* erts_alcu_literal_32_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); @@ -218,9 +214,6 @@ void erts_alcu_exec_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); # endif #endif /* HAVE_ERTS_MSEG */ -void* erts_alcu_sys_alloc(Allctr_t*, Uint *size_p, int superalign); -void* erts_alcu_sys_realloc(Allctr_t*, void *ptr, Uint *size_p, Uint old_size, int superalign); -void erts_alcu_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign); #ifdef ARCH_32 void* erts_alcu_literal_32_sys_alloc(Allctr_t*, Uint *size_p, int superalign); void* erts_alcu_literal_32_sys_realloc(Allctr_t*, void *ptr, Uint *size_p, Uint old_size, int superalign); -- cgit v1.2.3 From 2084cb04283e11409dfe2b2b26cdf53355c5c117 Mon Sep 17 00:00:00 2001 From: Salikhov Dinislam Date: Mon, 8 May 2017 15:32:54 +0300 Subject: erts: Make erts_align_utf8_bytes() static --- erts/emulator/beam/erl_bits.c | 2 +- erts/emulator/beam/erl_bits.h | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index eea55b3239..71c64997c1 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -1636,7 +1636,7 @@ erts_bs_get_unaligned_uint32(ErlBinMatchBuffer* mb) return LSB[0] | (LSB[1]<<8) | (LSB[2]<<16) | (LSB[3]<<24); } -void +static void erts_align_utf8_bytes(ErlBinMatchBuffer* mb, byte* buf) { Uint bits = mb->size - mb->offset; diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h index 632855255e..5da2b28a89 100644 --- a/erts/emulator/beam/erl_bits.h +++ b/erts/emulator/beam/erl_bits.h @@ -202,7 +202,6 @@ void erts_new_bs_put_string(ERL_BITS_PROTO_2(byte* iptr, Uint num_bytes)); Uint erts_bits_bufs_size(void); Uint32 erts_bs_get_unaligned_uint32(ErlBinMatchBuffer* mb); -void erts_align_utf8_bytes(ErlBinMatchBuffer* mb, byte* buf); Eterm erts_bs_get_utf8(ErlBinMatchBuffer* mb); Eterm erts_bs_get_utf16(ErlBinMatchBuffer* mb, Uint flags); Eterm erts_bs_append(Process* p, Eterm* reg, Uint live, Eterm build_size_term, -- cgit v1.2.3 From bc91e7e9c000145d5feec6877c3abe5e6747351b Mon Sep 17 00:00:00 2001 From: Salikhov Dinislam Date: Thu, 11 May 2017 00:38:25 +0300 Subject: erts: Make bif's do_send() static --- erts/emulator/beam/bif.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 7134d2da56..40dd4129d2 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -2047,8 +2047,6 @@ ebif_bang_2(BIF_ALIST_2) #define SEND_YIELD_CONTINUE (-8) -Sint do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext*); - static Sint remote_send(Process *p, DistEntry *dep, Eterm to, Eterm full_to, Eterm msg, ErtsSendContext* ctx) @@ -2102,8 +2100,8 @@ static Sint remote_send(Process *p, DistEntry *dep, return res; } -Sint -do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx) +static Sint +do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx) { Eterm portid; Port *pt; -- cgit v1.2.3 From e606eae1325a9f73140ab309d5dbbb7cdb589a04 Mon Sep 17 00:00:00 2001 From: Ted Burghart Date: Mon, 22 May 2017 15:36:09 -0400 Subject: Add enif_whereis_...() functions #### Why do we need this new feature? There are cases when a NIF needs to send a message, using `enif_send()`, to a long-lived process with a registered name. A common use-case is logging, where asynchronous fire-and-forget messages are the norm. There can also be cases where a yielding or dirty NIF or background thread may request a callback from a service with additional information it needs to complete its operation, yielding or waiting (with suitable timeouts, etc) until its state has been updated through the NIF module's API. NIFs can only send messages to pids, and the lack of name resolution leaves a complicated dance between separate monitoring processes and the NIF as the only way to keep a NIF informed of the whereabouts of such long-lived processes. Providing a reliable, built-in facility for NIFs to resolve process (or port) names simplifies these use cases considerably. #### Risks or uncertain artifacts? Testing has not exposed any significant risk. The implementation behaves as expected on regular and dirty scheduler threads as well as non-scheduler threads. By constraining the `enif_whereis_...()` functions to their minimal scopes and using patterns consistent with related functions, the implementation, testing, and maintenance burden is low. The API and behavior of existing functions is unchanged. #### How did you solve it? While extending `enif_send()` to operate on a pid or an atom (as `erlang:send/2` does) was attractive, it would have entailed changing the type of its `to_pid` parameter and thereby breaking backward compatibility. The same consideration applies to `enif_port_command()`. That leaves a choice between 1, 2, or 3 new functions: 1. `enif_whereis()` 2. `enif_whereis_pid()` and `enif_whereis_port()` 3. All of the above. While option (1), directly mimicking the behavior of `erlang:whereis/1`, is appealing, it poses potential problems if `pid()` or `port()` are subsequently implemented as non-integral types that must be bound to an owning `ErlNifEnv` instance. Therefore, option (2) has been chosen to use `ErlNifPid`/`ErlNifPort` structures in the API to maintain proper term ownership semantics. --- erts/emulator/beam/erl_nif.c | 58 ++++++++++++++++++++++++++++++++++ erts/emulator/beam/erl_nif_api_funcs.h | 4 +++ 2 files changed, 62 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index ea835d1b64..4815e5e7bb 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -879,6 +879,64 @@ enif_port_command(ErlNifEnv *env, const ErlNifPort* to_port, return res; } +/* + * env must be the caller's environment in a scheduler or NULL in a + * non-scheduler thread. + * name must be an atom - anything else will just waste time. + */ +static Eterm call_whereis(ErlNifEnv *env, Eterm name) +{ + Process *c_p; + Eterm res; + int scheduler; + int unlock; + + execution_state(env, &c_p, &scheduler); + ASSERT((c_p && scheduler) || (!c_p && !scheduler)); + + unlock = 0; + if (scheduler < 0) { + /* dirty scheduler */ + if (ERTS_PROC_IS_EXITING(c_p)) + return 0; + + if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + unlock = 1; + } + } + res = erts_whereis_name_to_id(c_p, name); + + if (unlock) + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + + return res; +} + +int enif_whereis_pid(ErlNifEnv *env, ERL_NIF_TERM name, ErlNifPid *pid) +{ + Eterm res; + + if (is_not_atom(name)) + return 0; + + res = call_whereis(env, name); + /* enif_get_local_ functions check the type */ + return enif_get_local_pid(env, res, pid); +} + +int enif_whereis_port(ErlNifEnv *env, ERL_NIF_TERM name, ErlNifPort *port) +{ + Eterm res; + + if (is_not_atom(name)) + return 0; + + res = call_whereis(env, name); + /* enif_get_local_ functions check the type */ + return enif_get_local_port(env, res, port); +} + ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term) { Uint sz; diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index c305732d63..94c04cd126 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -181,6 +181,8 @@ ERL_NIF_API_FUNC_DECL(int, enif_monitor_process,(ErlNifEnv*,void* obj,const ErlN ERL_NIF_API_FUNC_DECL(int, enif_demonitor_process,(ErlNifEnv*,void* obj,const ErlNifMonitor *monitor)); ERL_NIF_API_FUNC_DECL(int, enif_compare_monitors,(const ErlNifMonitor*,const ErlNifMonitor*)); ERL_NIF_API_FUNC_DECL(ErlNifUInt64,enif_hash,(ErlNifHash type, ERL_NIF_TERM term, ErlNifUInt64 salt)); +ERL_NIF_API_FUNC_DECL(int, enif_whereis_pid, (ErlNifEnv *env, ERL_NIF_TERM name, ErlNifPid *pid)); +ERL_NIF_API_FUNC_DECL(int, enif_whereis_port, (ErlNifEnv *env, ERL_NIF_TERM name, ErlNifPort *port)); /* ** ADD NEW ENTRIES HERE (before this comment) !!! @@ -344,6 +346,8 @@ ERL_NIF_API_FUNC_DECL(ErlNifUInt64,enif_hash,(ErlNifHash type, ERL_NIF_TERM term # define enif_demonitor_process ERL_NIF_API_FUNC_MACRO(enif_demonitor_process) # define enif_compare_monitors ERL_NIF_API_FUNC_MACRO(enif_compare_monitors) # define enif_hash ERL_NIF_API_FUNC_MACRO(enif_hash) +# define enif_whereis_pid ERL_NIF_API_FUNC_MACRO(enif_whereis_pid) +# define enif_whereis_port ERL_NIF_API_FUNC_MACRO(enif_whereis_port) /* ** ADD NEW ENTRIES HERE (before this comment) -- cgit v1.2.3 From a261e4f4eda13f6edb37349b18e096694a86e29a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20H=C3=B6gberg?= Date: Wed, 24 May 2017 15:47:37 +0200 Subject: Identify db_hash_slot locks by their table name instead of slot index --- erts/emulator/beam/erl_db_hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 390369fdb9..036cc9d9e9 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -684,7 +684,7 @@ int db_create_hash(Process *p, DbTable *tbl) sizeof(DbTableHashFineLocks)); for (i=0; ilocks->lck_vec[i].lck, &rwmtx_opt, - "db_hash_slot", make_small(i)); + "db_hash_slot", tb->common.the_name); } /* This important property is needed to guarantee that the buckets * involved in a grow/shrink operation it protected by the same lock: -- cgit v1.2.3 From 23f132d9ab776ab94c29f1faed76c9fde8f4a73f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Mon, 29 May 2017 14:41:14 +0200 Subject: Make sure that asynchronous replies are not lost If an synchronous GC was requested by calling: erlang:garbage_collect(Pid, [{async,Ref}]) the reply message could in certain circumstances be lost. The problem is in cleanup_sys_tasks() in erl_process.c. If there were at least one dirty task, only the first dirty task would be cleaned up. All other systems tasks would not be cleaned up (that is, no replies would be sent to other processes waiting for the tasks to finish). --- erts/emulator/beam/erl_process.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 7952e3031d..0575729bbd 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -11331,7 +11331,9 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) erts_aint32_t state = in_state; int max_reds = in_reds; int reds = 0; - int qmask = 0; + int qmask = 1; /* Set to 1 to force looping as long as there + * are dirty tasks. + */ ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN); -- cgit v1.2.3 From c819af0538083d861e8bd8ba66768f5e5a633311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Mon, 29 May 2017 14:51:01 +0200 Subject: erl_process.c: Add more assertions in process termination Before terminating a process, add assertions to make sure that all queues for system tasks have been emptied. --- erts/emulator/beam/erl_process.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 0575729bbd..d4385e3987 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -13937,6 +13937,16 @@ erts_continue_exit_process(Process *p) goto yield; } +#ifdef DEBUG + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + ASSERT(p->sys_task_qs == NULL); + ASSERT(ERTS_PROC_GET_DELAYED_GC_TASK_QS(p) == NULL); +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(p->dirty_sys_tasks == NULL); +#endif + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); +#endif + if (p->flags & F_USING_DDLL) { erts_ddll_proc_dead(p, ERTS_PROC_LOCK_MAIN); p->flags &= ~F_USING_DDLL; -- cgit v1.2.3 From 8fe44f0f81603697174fcf6ad20e7acbb22fea5f Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 29 May 2017 17:19:18 +0200 Subject: Enable off-heap message queue for code purger, et. al. --- erts/emulator/beam/erl_init.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index ac99f043e6..6172595552 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -431,7 +431,7 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** } static Eterm -erl_system_process_otp(Eterm parent_pid, char* modname) +erl_system_process_otp(Eterm parent_pid, char* modname, int off_heap_msgq) { Eterm start_mod; Process* parent; @@ -447,6 +447,8 @@ erl_system_process_otp(Eterm parent_pid, char* modname) parent = erts_pid2proc(NULL, 0, parent_pid, ERTS_PROC_LOCK_MAIN); so.flags = erts_default_spo_flags|SPO_SYSTEM_PROC; + if (off_heap_msgq) + so.flags |= SPO_OFF_HEAP_MSGQ; res = erl_create_process(parent, start_mod, am_start, NIL, &so); erts_smp_proc_unlock(parent, ERTS_PROC_LOCK_MAIN); return res; @@ -2326,14 +2328,14 @@ erl_start(int argc, char **argv) */ Eterm pid; - pid = erl_system_process_otp(otp_ring0_pid, "erts_code_purger"); + pid = erl_system_process_otp(otp_ring0_pid, "erts_code_purger", !0); erts_code_purger = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, internal_pid_index(pid)); ASSERT(erts_code_purger && erts_code_purger->common.id == pid); erts_proc_inc_refc(erts_code_purger); - pid = erl_system_process_otp(otp_ring0_pid, "erts_literal_area_collector"); + pid = erl_system_process_otp(otp_ring0_pid, "erts_literal_area_collector", !0); erts_literal_area_collector = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, internal_pid_index(pid)); @@ -2342,7 +2344,7 @@ erl_start(int argc, char **argv) erts_proc_inc_refc(erts_literal_area_collector); #ifdef ERTS_DIRTY_SCHEDULERS - pid = erl_system_process_otp(otp_ring0_pid, "erts_dirty_process_code_checker"); + pid = erl_system_process_otp(otp_ring0_pid, "erts_dirty_process_code_checker", !0); erts_dirty_process_code_checker = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, internal_pid_index(pid)); -- cgit v1.2.3 From acd8571b35756c3fdc389807c34d80c9d44435e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Wed, 31 May 2017 12:55:40 +0200 Subject: Make sure that asynchronous replies are not lost (First attempt to fix in 23f132d9ab776a.) If an synchronous GC was requested by calling: erlang:garbage_collect(Pid, [{async,Ref}]) the reply message could in certain circumstances be lost. The problem is that cleanup_sys_tasks() is never called if there are dirty tasks, but no other active system tasks. Also shorten the long waiting times in the test case binary_SUITE:trapping/1 to make it much more likely that the GC have not already finished when the process is killed. --- erts/emulator/beam/erl_process.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index d4385e3987..285fa05d63 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -13932,7 +13932,11 @@ erts_continue_exit_process(Process *p) erts_set_gc_state(p, 1); state = erts_smp_atomic32_read_acqb(&p->state); - if (state & ERTS_PSFLG_ACTIVE_SYS) { + if (state & ERTS_PSFLG_ACTIVE_SYS +#ifdef ERTS_DIRTY_SCHEDULERS + || p->dirty_sys_tasks +#endif + ) { if (cleanup_sys_tasks(p, state, CONTEXT_REDS) >= CONTEXT_REDS/2) goto yield; } -- cgit v1.2.3 From f4836a7883842f0e2ae20285ccedf9856c66b24a Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 1 Jun 2017 19:24:27 +0200 Subject: Fix check_process_code() on hibernated process --- erts/emulator/beam/beam_bif_load.c | 13 ++++++++++++- erts/emulator/beam/erl_gc.c | 5 ++++- erts/emulator/beam/erl_process.h | 1 + 3 files changed, 17 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 0e192b1ebd..6072eaa8eb 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -808,6 +808,10 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp) if (done_gc) { return am_true; } else { + if (rp->flags & F_HIBERNATED) { + /* GC wont help; everything on heap is live... */ + return am_true; + } if (!allow_gc) return am_aborted; /* @@ -870,6 +874,8 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp) if (done_gc) { return am_true; } else { + int hibernated = !!(rp->flags & F_HIBERNATED); + int gc_cost; Eterm* literals; Uint lit_size; struct erl_off_heap_header* oh; @@ -886,13 +892,18 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp) rp->ftrace = NIL; done_gc = 1; FLAGS(rp) |= F_NEED_FULLSWEEP; - *redsp += erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); + gc_cost = erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); + *redsp += gc_cost; literals = (Eterm *) modp->old.code[MI_LITERALS_START]; lit_size = (Eterm *) modp->old.code[MI_LITERALS_END] - literals; oh = (struct erl_off_heap_header *) modp->old.code[MI_LITERALS_OFF_HEAP]; *redsp += lit_size / 10; /* Need, better value... */ erts_garbage_collect_literals(rp, literals, lit_size, oh); + if (hibernated) { + erts_garbage_collect_hibernate(rp); + *redsp += gc_cost; + } } } return am_false; diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 2f21111a2e..c22577a254 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -490,7 +490,7 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) esdp->gc_info.garbage_cols++; esdp->gc_info.reclaimed += reclaimed_now; - FLAGS(p) &= ~F_FORCE_GC; + FLAGS(p) &= ~(F_FORCE_GC|F_HIBERNATED); #ifdef CHECK_FOR_HOLES /* @@ -658,6 +658,8 @@ erts_garbage_collect_hibernate(Process* p) ErtsGcQuickSanityCheck(p); + p->flags |= F_HIBERNATED; + erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); } @@ -830,6 +832,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, /* * Restore status. */ + p->flags &= ~F_HIBERNATED; erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); } diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 10c6fa4a67..41e5f55476 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1292,6 +1292,7 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; #define F_P2PNR_RESCHED (1 << 9) /* Process has been rescheduled via erts_pid2proc_not_running() */ #define F_FORCE_GC (1 << 10) /* Force gc at process in-scheduling */ #define F_DISABLE_GC (1 << 11) /* Disable GC */ +#define F_HIBERNATED (1 << 12) /* Hibernated */ /* process trace_flags */ #define F_SENSITIVE (1 << 0) -- cgit v1.2.3 From c8f1d8218a2aa01a2b8d4f65af6e07e5ca2b631e Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 1 Jun 2017 19:29:36 +0200 Subject: Do not GC hibernated process from other processes --- erts/emulator/beam/erl_process.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index d583118e7b..bce4e7fff3 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -10059,7 +10059,8 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) reds++; } else { - if (!garbage_collected) { + if (!garbage_collected + && !(c_p->flags & F_HIBERNATED)) { FLAGS(c_p) |= F_NEED_FULLSWEEP; reds += erts_garbage_collect(c_p, 0, -- cgit v1.2.3 From f1c69ee583dcd1f525562cf6adc382b8464b1578 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 13 Jun 2017 15:51:42 +0200 Subject: Revert "erts: Do not generate atoms on old latin1 external format" This reverts commit 65b04e233e09e3cc2e0fda3c28e155b95c5a4baf. --- erts/emulator/beam/dist.h | 1 - erts/emulator/beam/external.c | 39 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index 93a651b24d..3e17645997 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -53,7 +53,6 @@ | DFLAG_EXPORT_PTR_TAG \ | DFLAG_BIT_BINARIES \ | DFLAG_MAP_TAG \ - | DFLAG_UTF8_ATOMS \ | DFLAG_BIG_CREATION) /* opcodes used in distribution messages */ diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 1190d90b8e..06213daa67 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2090,6 +2090,7 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) { int iix; int len; + int utf8_atoms = (int) (dflags & DFLAG_UTF8_ATOMS); ASSERT(is_atom(atom)); @@ -2118,8 +2119,8 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) if (iix < 0) { Atom *a = atom_tab(atom_val(atom)); len = a->len; - { - if (len > 255) { + if (utf8_atoms || a->latin1_chars < 0) { + if (len > 255) { *ep++ = ATOM_UTF8_EXT; put_int16(len, ep); ep += 2; @@ -2131,6 +2132,32 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) } sys_memcpy((char *) ep, (char *) a->name, len); } + else { + if (a->latin1_chars <= 255 && (dflags & DFLAG_SMALL_ATOM_TAGS)) { + *ep++ = SMALL_ATOM_EXT; + if (len == a->latin1_chars) { + sys_memcpy(ep+1, a->name, len); + } + else { + len = erts_utf8_to_latin1(ep+1, a->name, len); + ASSERT(len == a->latin1_chars); + } + put_int8(len, ep); + ep++; + } + else { + *ep++ = ATOM_EXT; + if (len == a->latin1_chars) { + sys_memcpy(ep+2, a->name, len); + } + else { + len = erts_utf8_to_latin1(ep+2, a->name, len); + ASSERT(len == a->latin1_chars); + } + put_int16(len, ep); + ep += 2; + } + } ep += len; return ep; } @@ -4053,13 +4080,19 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, else { Atom *a = atom_tab(atom_val(obj)); int alen; - { + if ((dflags & DFLAG_UTF8_ATOMS) || a->latin1_chars < 0) { alen = a->len; result += 1 + 1 + alen; if (alen > 255) { result++; /* ATOM_UTF8_EXT (not small) */ } } + else { + alen = a->latin1_chars; + result += 1 + 1 + alen; + if (alen > 255 || !(dflags & DFLAG_SMALL_ATOM_TAGS)) + result++; /* ATOM_EXT (not small) */ + } insert_acache_map(acmp, obj, dflags); } break; -- cgit v1.2.3 From bdc0f3504fb13f777a7cc826caa5fd10dc6fc291 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 14 Jun 2017 13:23:34 +0200 Subject: Introduce minor vsn 2 in term_to_binary/2 --- erts/emulator/beam/external.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 06213daa67..95275847f8 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1129,8 +1129,10 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) case 0: flags = TERM_TO_BINARY_DFLAGS & ~DFLAG_NEW_FLOATS; break; - case 1: + case 1: /* Current default... */ flags = TERM_TO_BINARY_DFLAGS; + case 2: + flags = TERM_TO_BINARY_DFLAGS | DFLAG_UTF8_ATOMS; break; default: goto error; -- cgit v1.2.3 From 43718d3b81d7f3d08e25047e22d579801bbe5044 Mon Sep 17 00:00:00 2001 From: Hans Nilsson Date: Wed, 14 Jun 2017 15:36:21 +0200 Subject: Update copyright year --- erts/emulator/beam/beam_load.h | 2 +- erts/emulator/beam/big.c | 2 +- erts/emulator/beam/big.h | 2 +- erts/emulator/beam/erl_alloc.h | 2 +- erts/emulator/beam/erl_alloc_util.h | 2 +- erts/emulator/beam/erl_cpu_topology.c | 2 +- erts/emulator/beam/erl_cpu_topology.h | 2 +- erts/emulator/beam/erl_lock_count.c | 2 +- erts/emulator/beam/erl_port.h | 2 +- erts/emulator/beam/erl_port_task.c | 2 +- erts/emulator/beam/erl_port_task.h | 2 +- erts/emulator/beam/erl_smp.h | 2 +- erts/emulator/beam/erl_threads.h | 2 +- erts/emulator/beam/ops.tab | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index b8d8634e28..c088bdb751 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2016. All Rights Reserved. + * Copyright Ericsson AB 1999-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index 1f6feade1c..7128b8ed23 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index 258038a157..48efce20e7 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index 758d529f87..7b5cbe2178 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2016. All Rights Reserved. + * Copyright Ericsson AB 2002-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index 7a96bd0319..e889980fa4 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2016. All Rights Reserved. + * Copyright Ericsson AB 2002-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c index d0fd763798..50f33b2014 100644 --- a/erts/emulator/beam/erl_cpu_topology.c +++ b/erts/emulator/beam/erl_cpu_topology.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010-2016. All Rights Reserved. + * Copyright Ericsson AB 2010-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_cpu_topology.h b/erts/emulator/beam/erl_cpu_topology.h index cf139d95a9..c922214702 100644 --- a/erts/emulator/beam/erl_cpu_topology.h +++ b/erts/emulator/beam/erl_cpu_topology.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010-2016. All Rights Reserved. + * Copyright Ericsson AB 2010-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index aee9796171..678bc43f04 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2016. All Rights Reserved. + * Copyright Ericsson AB 2008-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index 5c947ad1c0..6a3213ec52 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012-2016. All Rights Reserved. + * Copyright Ericsson AB 2012-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 55526e1d5e..a044de3fee 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2016. All Rights Reserved. + * Copyright Ericsson AB 2006-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index ab536c6f27..9cca62ffaf 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2016. All Rights Reserved. + * Copyright Ericsson AB 2006-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index 55ba943bdd..181736b009 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2016. All Rights Reserved. + * Copyright Ericsson AB 2005-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index 28ff5d3a42..9612b70469 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2016. All Rights Reserved. + * Copyright Ericsson AB 2001-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 8abe871c14..cdf9cb58b9 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2016. All Rights Reserved. +# Copyright Ericsson AB 1997-2017. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -- cgit v1.2.3 From 5337eff8e86e352eb5ca63cc07bb9880a4f5c9ec Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 14 Jun 2017 16:21:56 +0200 Subject: Fix minor vsn 1 in term_to_binary/2 broken in this branch --- erts/emulator/beam/external.c | 1 + 1 file changed, 1 insertion(+) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 95275847f8..1560844521 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1131,6 +1131,7 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) break; case 1: /* Current default... */ flags = TERM_TO_BINARY_DFLAGS; + break; case 2: flags = TERM_TO_BINARY_DFLAGS | DFLAG_UTF8_ATOMS; break; -- cgit v1.2.3