diff options
Diffstat (limited to 'erts')
94 files changed, 3638 insertions, 1652 deletions
diff --git a/erts/Makefile.in b/erts/Makefile index cddabbecee..3fe567b7d5 100644 --- a/erts/Makefile.in +++ b/erts/Makefile @@ -20,9 +20,9 @@ .NOTPARALLEL: -include $(ERL_TOP)/make/output.mk include $(ERL_TOP)/make/target.mk include vsn.mk +include $(ERL_TOP)/make/$(TARGET)/otp.mk # ---------------------------------------------------------------------- @@ -33,10 +33,8 @@ ifeq ($(NO_START_SCRIPTS),) ERTSDIRS += start_scripts endif -EXTRA_FLAVORS=smp - .PHONY: all -all: smp opt +all: $(FLAVORS) .PHONY: docs docs: @@ -56,9 +54,9 @@ debug opt clean: # - don't use them in scripts or assume they will always stay like this! # -.PHONY: $(EXTRA_FLAVORS) -$(EXTRA_FLAVORS): - $(V_at)( cd emulator && $(MAKE) opt FLAVOR=$@ ) +.PHONY: $(FLAVORS) +$(FLAVORS): + $(V_at)( $(MAKE) opt FLAVOR=$@ ) # Make erl script and erlc in $(ERL_TOP)/bin which runs the compiled version # Note that erlc is not a script and requires extra handling on cygwin. @@ -75,10 +73,12 @@ local_setup: $(ERL_TOP)/bin/erl.exe $(ERL_TOP)/bin/erlc.exe \ $(ERL_TOP)/bin/escript $(ERL_TOP)/bin/escript.exe \ $(ERL_TOP)/bin/dialyzer $(ERL_TOP)/bin/dialyzer.exe \ + $(ERL_TOP)/bin/typer $(ERL_TOP)/bin/typer.exe \ $(ERL_TOP)/bin/ct_run $(ERL_TOP)/bin/ct_run.exe \ $(ERL_TOP)/bin/start*.boot $(ERL_TOP)/bin/start*.script @if [ "X$(TARGET)" = "Xwin32" ]; then \ cp $(ERL_TOP)/bin/$(TARGET)/dialyzer.exe $(ERL_TOP)/bin/dialyzer.exe; \ + cp $(ERL_TOP)/bin/$(TARGET)/typer.exe $(ERL_TOP)/bin/typer.exe; \ cp $(ERL_TOP)/bin/$(TARGET)/ct_run.exe $(ERL_TOP)/bin/ct_run.exe; \ cp $(ERL_TOP)/bin/$(TARGET)/erlc.exe $(ERL_TOP)/bin/erlc.exe; \ cp $(ERL_TOP)/bin/$(TARGET)/erl.exe $(ERL_TOP)/bin/erl.exe; \ @@ -98,6 +98,7 @@ local_setup: -e "s;%VSN%;$(VSN);" \ $(ERL_TOP)/erts/etc/unix/cerl.src > $(ERL_TOP)/bin/cerl; \ cp $(ERL_TOP)/bin/$(TARGET)/dialyzer $(ERL_TOP)/bin/dialyzer; \ + cp $(ERL_TOP)/bin/$(TARGET)/typer $(ERL_TOP)/bin/typer; \ cp $(ERL_TOP)/bin/$(TARGET)/ct_run $(ERL_TOP)/bin/ct_run; \ cp $(ERL_TOP)/bin/$(TARGET)/erlc $(ERL_TOP)/bin/erlc; \ cp $(ERL_TOP)/bin/$(TARGET)/escript $(ERL_TOP)/bin/escript; \ @@ -126,7 +127,7 @@ makefiles: .PHONY: release release: - $(V_at)for f in plain $(EXTRA_FLAVORS) ; do \ + $(V_at)for f in $(FLAVORS); do \ ( cd emulator && $(MAKE) release FLAVOR=$$f ) \ done $(V_at)for d in $(ERTSDIRS) $(XINSTDIRS); do \ diff --git a/erts/configure.in b/erts/configure.in index b488ba9171..9ab5b4cfd2 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -154,6 +154,14 @@ AS_HELP_STRING([--disable-smp-support], [disable smp support]), *) enable_smp_support=yes ;; esac ], enable_smp_support=unknown) +AC_ARG_ENABLE(plain-emulator, +AS_HELP_STRING([--enable-plain-emulator], [enable plain emulator]) +AS_HELP_STRING([--disable-plain-emulator], [disable plain emulator]), +[ case "$enableval" in + no) enable_plain_emulator=no ;; + *) enable_plain_emulator=yes ;; + esac ], enable_plain_emulator=unknown) + AC_ARG_ENABLE(smp-require-native-atomics, AS_HELP_STRING([--disable-smp-require-native-atomics], [disable the SMP requirement of a native atomic implementation]), @@ -993,7 +1001,7 @@ else found_threads=yes fi - +FLAVORS= ERTS_BUILD_SMP_EMU=$enable_smp_support AC_MSG_CHECKING(whether an emulator with smp support should be built) case $ERTS_BUILD_SMP_EMU in @@ -1078,6 +1086,9 @@ EOF if test $ERTS_BUILD_SMP_EMU = yes; then + DEFAULT_FLAVOR=smp + FLAVORS="$FLAVORS smp" + if test $found_threads = no; then AC_MSG_ERROR([cannot build smp enabled emulator since no thread library was found]) fi @@ -1153,6 +1164,66 @@ fi AC_SUBST(ERTS_BUILD_SMP_EMU) +ERTS_BUILD_PLAIN_EMU=$enable_plain_emulator +AC_MSG_CHECKING(whether an emulator without smp support should be built) +case $ERTS_BUILD_PLAIN_EMU in + yes) + AC_MSG_RESULT(yes; enabled by user) + ;; + no) + AC_MSG_RESULT(no; disabled by user) + ;; + unknown) + case "$enable_threads-$ERTS_BUILD_SMP_EMU" in + no-*) + ERTS_BUILD_PLAIN_EMU=yes + AC_MSG_RESULT(yes) + ;; + *-no) + ERTS_BUILD_PLAIN_EMU=yes + AC_MSG_RESULT(yes; enabled as smp emulator was disabled) + ;; + *) + ERTS_BUILD_PLAIN_EMU=no + AC_MSG_RESULT(no) + ;; + esac + ;; +esac + +case $ERTS_BUILD_PLAIN_EMU in + yes) + AC_DEFINE(ERTS_HAVE_PLAIN_EMU, 1, [Define if the non-smp emulator is built]) + FLAVORS="$FLAVORS plain" + test -f "$ERL_TOP/erts/CONF_INFO" || echo "" > "$ERL_TOP/erts/CONF_INFO" + cat >> $ERL_TOP/erts/CONF_INFO <<EOF + + The PLAIN aka NON-SMP emulator has been enabled. + This is a DEPRECATED feature scheduled for removal + in a future major release. + +EOF + ;; + no) + ;; +esac + +AC_SUBST(ERTS_BUILD_PLAIN_EMU) +AC_SUBST(FLAVORS) + +case "$ERTS_BUILD_PLAIN_EMU-$ERTS_BUILD_SMP_EMU" in + no-no) + AC_MSG_ERROR([both smp and non-smp emulators have been disabled, one of them has to be enabled]) + ;; + *-no) + DEFAULT_FLAVOR=plain + ;; + *) + ;; +esac + +AC_SUBST(DEFAULT_FLAVOR) + AC_CHECK_FUNCS([posix_fadvise closefrom]) AC_CHECK_HEADERS([linux/falloc.h]) dnl * Old glibcs have broken fallocate64(). Make sure not to use it. @@ -4953,7 +5024,6 @@ AC_CONFIG_FILES([ include/internal/$host/ethread.mk:include/internal/ethread.mk.in include/internal/$host/erts_internal.mk:include/internal/erts_internal.mk.in lib_src/$host/Makefile:lib_src/Makefile.in - Makefile:Makefile.in ../make/$host/otp.mk:../make/otp.mk.in ../make/$host/otp_ded.mk:../make/otp_ded.mk.in ]) diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml index 29fef7348b..ef8126e86f 100644 --- a/erts/doc/src/erl.xml +++ b/erts/doc/src/erl.xml @@ -513,11 +513,11 @@ system with SMP support is available. <c>-smp auto</c> starts the Erlang runtime system with SMP support enabled if it is available and more than one logical processor is detected. - <c>-smp disable</c> starts a runtime system without SMP support.</p> + <c>-smp disable</c> starts a runtime system without SMP support. + The runtime system without SMP support is deprecated and will + be removed in a future major release.</p> <note> - <p>The runtime system with SMP support is not available on all - supported platforms. See also flag - <seealso marker="#+S"><c>+S</c></seealso>.</p> + <p>See also flag<seealso marker="#+S"><c>+S</c></seealso>.</p> </note> </item> <tag><c><![CDATA[-version]]></c> (emulator flag)</tag> @@ -1318,8 +1318,22 @@ <c><![CDATA[+sss size]]></c></tag> <item> <p>Suggested stack size, in kilowords, for scheduler threads. - Valid range is 4-8192 kilowords. The default stack size is - OS-dependent.</p> + Valid range is 20-8192 kilowords. The default suggested + stack size is 128 kilowords.</p> + </item> + <tag><marker id="dcpu_sched_thread_stack_size"/> + <c><![CDATA[+sssdcpu size]]></c></tag> + <item> + <p>Suggested stack size, in kilowords, for dirty CPU scheduler + threads. Valid range is 20-8192 kilowords. The default + suggested stack size is 40 kilowords.</p> + </item> + <tag><marker id="dio_sched_thread_stack_size"/> + <c><![CDATA[+sssdio size]]></c></tag> + <item> + <p>Suggested stack size, in kilowords, for dirty IO scheduler + threads. Valid range is 20-8192 kilowords. The default + suggested stack size is 40 kilowords.</p> </item> <tag><marker id="+stbt"/><c>+stbt BindType</c></tag> <item> diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index cb2cdec606..a26a695f2e 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -2499,6 +2499,42 @@ os_prompt%</pre> </func> <func> + <name name="list_to_port" arity="1"/> + <fsummary>Convert from text representation to a port.</fsummary> + <desc> + <p>Returns a port identifier whose text representation is a + <c><anno>String</anno></c>, for example:</p> + <pre> +> <input>list_to_port("#Port<0.4>").</input> +#Port<0.4></pre> + <p>Failure: <c>badarg</c> if <c><anno>String</anno></c> contains a bad + representation of a port identifier.</p> + <warning> + <p>This BIF is intended for debugging and is not to be used + in application programs.</p> + </warning> + </desc> + </func> + + <func> + <name name="list_to_ref" arity="1"/> + <fsummary>Convert from text representation to a ref.</fsummary> + <desc> + <p>Returns a reference whose text representation is a + <c><anno>String</anno></c>, for example:</p> + <pre> +> <input>list_to_ref("#Ref<0.4192537678.4073193475.71181>").</input> +#Ref<0.4192537678.4073193475.71181></pre> + <p>Failure: <c>badarg</c> if <c><anno>String</anno></c> contains a bad + representation of a reference.</p> + <warning> + <p>This BIF is intended for debugging and is not to be used + in application programs.</p> + </warning> + </desc> + </func> + + <func> <name name="list_to_tuple" arity="1"/> <fsummary>Convert a list to a tuple.</fsummary> <desc> diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index f93b386392..3b86a4dcdf 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -32,6 +32,24 @@ <p>This document describes the changes made to the ERTS application.</p> +<section><title>Erts 8.3.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The <c>+Bi</c> command line argument of <c>erl</c> + erroneously caused <c>SIGTERM</c> to be ignored by the VM + as well as of all its child processes. This bug was + introduced in erts version 8.3.</p> + <p> + Own Id: OTP-14358 Aux Id: OTP-14085 </p> + </item> + </list> + </section> + +</section> + <section><title>Erts 8.3.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 7ea0111e59..227aabcf1e 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -175,7 +175,23 @@ endif # NOTE: When adding a new type update ERL_BUILD_TYPE_MARKER in sys/unix/sys.c # -ifeq ($(FLAVOR),smp) +FLAVOR=$(DEFAULT_FLAVOR) + +ifeq ($(FLAVOR),plain) + +DS_SUPPORT=no +DS_TEST=no + +FLAVOR_MARKER= +FLAVOR_FLAGS= +ENABLE_ALLOC_TYPE_VARS += nofrag +M4FLAGS += + +else # FLAVOR + +# If flavor isn't one of the above, it *is* smp flavor... +override FLAVOR=smp + FLAVOR_MARKER=.smp FLAVOR_FLAGS=-DERTS_SMP ENABLE_ALLOC_TYPE_VARS += smp nofrag @@ -196,30 +212,15 @@ DS_SUPPORT=no DS_TEST=no endif # DIRTY_SCHEDULER_SUPPORT -else # FLAVOR - -DS_SUPPORT=no -DS_TEST=no - -# If flavor isn't one of the above, it *is* plain flavor... -override FLAVOR=plain -FLAVOR_MARKER= -FLAVOR_FLAGS= -ENABLE_ALLOC_TYPE_VARS += nofrag -M4FLAGS += -endif +endif # FLAVOR TF_MARKER=$(TYPEMARKER)$(FLAVOR_MARKER) -ifeq ($(FLAVOR)-@ERTS_BUILD_SMP_EMU@,smp-no) -VOID_EMULATOR = '*** SMP emulator disabled by configure' -else ifeq ($(TYPE)-@HAVE_VALGRIND@,valgrind-no) VOID_EMULATOR = '*** valgrind emulator disabled by configure' else VOID_EMULATOR = endif -endif OPSYS=@OPSYS@ sol2CFLAGS= diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 4ba8c2a669..769fe89219 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -233,7 +233,7 @@ prepare_loading_2(BIF_ALIST_2) } hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE); res = erts_mk_magic_ref(&hp, &MSO(BIF_P), magic); - erts_refc_dec(&magic->refc, 1); + erts_refc_dec(&magic->intern.refc, 1); BIF_RET(res); } @@ -435,7 +435,7 @@ finish_loading_1(BIF_ALIST_1) Eterm mod; Eterm retval; - erts_refc_inc(&p[i].code->refc, 1); + erts_refc_inc(&p[i].code->intern.refc, 1); retval = erts_finish_loading(p[i].code, BIF_P, 0, &mod); ASSERT(retval == NIL || retval == am_on_load); if (retval == am_on_load) { diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index b32c74ce7a..1efe7536d6 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -327,7 +327,7 @@ erts_consolidate_bif_bp_data(void) static void consolidate_bp_data(Module* modp, ErtsCodeInfo *ci, int local) { - GenericBp* g = (GenericBp *) ci->native; + GenericBp* g = ci->u.gen_bp; GenericBpData* src; GenericBpData* dst; Uint flags; @@ -376,7 +376,7 @@ consolidate_bp_data(Module* modp, ErtsCodeInfo *ci, int local) ASSERT(*erts_codeinfo_to_code(ci) != (BeamInstr) BeamOp(op_i_generic_breakpoint)); } - ci->native = 0; + ci->u.gen_bp = NULL; Free(g); return; } @@ -427,7 +427,7 @@ erts_install_breakpoints(BpFunctions* f) for (i = 0; i < n; i++) { ErtsCodeInfo* ci = f->matching[i].ci; BeamInstr *pc = erts_codeinfo_to_code(ci); - GenericBp* g = (GenericBp *) ci->native; + GenericBp* g = ci->u.gen_bp; if (*pc != br && g) { Module* modp = f->matching[i].mod; @@ -468,7 +468,7 @@ uninstall_breakpoint(ErtsCodeInfo *ci) { BeamInstr *pc = erts_codeinfo_to_code(ci); if (*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { - GenericBp* g = (GenericBp *) ci->native; + GenericBp* g = ci->u.gen_bp; if (g->data[erts_active_bp_ix()].flags == 0) { /* * The following write is not protected by any lock. We @@ -549,7 +549,7 @@ erts_clear_trace_break(BpFunctions* f) void erts_clear_call_trace_bif(ErtsCodeInfo *ci, int local) { - GenericBp* g = (GenericBp *) ci->native; + GenericBp* g = ci->u.gen_bp; if (g) { Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE; @@ -624,7 +624,7 @@ erts_clear_module_break(Module *modp) { continue; uninstall_breakpoint(ci); consolidate_bp_data(modp, ci, 1); - ASSERT(ci->native == 0); + ASSERT(ci->u.gen_bp == NULL); } return n; } @@ -638,7 +638,7 @@ erts_clear_export_break(Module* modp, ErtsCodeInfo *ci) erts_commit_staged_bp(); *erts_codeinfo_to_code(ci) = (BeamInstr) 0; consolidate_bp_data(modp, ci, 0); - ASSERT(ci->native == 0); + ASSERT(ci->u.gen_bp == NULL); } BeamInstr @@ -651,7 +651,7 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg) ASSERT(info->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - g = (GenericBp *) info->native; + g = info->u.gen_bp; bp = &g->data[ix]; bp_flags = bp->flags; ASSERT((bp_flags & ~ERTS_BPF_ALL) == 0); @@ -754,7 +754,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); - g = (GenericBp *) ep->info.native; + g = ep->info.u.gen_bp; if (g) { bp = &g->data[erts_active_bp_ix()]; bp_flags = bp->flags; @@ -1511,7 +1511,7 @@ set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags, ErtsBpIndex ix = erts_staging_bp_ix(); ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); - g = (GenericBp *) ci->native; + g = ci->u.gen_bp; if (g == 0) { int i; if (count_op == ERTS_BREAK_RESTART || count_op == ERTS_BREAK_PAUSE) { @@ -1523,7 +1523,7 @@ set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags, for (i = 0; i < ERTS_NUM_BP_IX; i++) { g->data[i].flags = 0; } - ci->native = (BeamInstr) g; + ci->u.gen_bp = g; } bp = &g->data[ix]; @@ -1633,7 +1633,7 @@ clear_function_break(ErtsCodeInfo *ci, Uint break_flags) ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); - if ((g = (GenericBp *) ci->native) == 0) { + if ((g = ci->u.gen_bp) == NULL) { return 1; } @@ -1728,7 +1728,7 @@ get_time_break(ErtsCodeInfo *ci) static GenericBpData* check_break(ErtsCodeInfo *ci, Uint break_flags) { - GenericBp* g = (GenericBp *) ci->native; + GenericBp* g = ci->u.gen_bp; ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); if (erts_is_native_break(ci)) { diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 6010c17c17..3d36094e07 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -3887,7 +3887,6 @@ do { \ * Allocate the binary struct itself. */ bptr = erts_bin_nrml_alloc(num_bytes); - erts_refc_init(&bptr->refc, 1); erts_current_bin = (byte *) bptr->orig_bytes; /* @@ -3982,7 +3981,6 @@ do { \ * Allocate the binary struct itself. */ bptr = erts_bin_nrml_alloc(BsOp1); - erts_refc_init(&bptr->refc, 1); erts_current_bin = (byte *) bptr->orig_bytes; /* @@ -4950,14 +4948,14 @@ do { \ */ ErtsCodeInfo *ci = erts_code_to_codeinfo(I); ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.u.ncallee = (void(*)(void)) ci->native; + c_p->hipe.u.ncallee = ci->u.ncallee; ++hipe_trap_count; HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (ci->mfa.arity << 8)); } OpCase(hipe_trap_call_closure): { ErtsCodeInfo *ci = erts_code_to_codeinfo(I); ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.u.ncallee = (void(*)(void)) ci->native; + c_p->hipe.u.ncallee = ci->u.ncallee; ++hipe_trap_count; HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (ci->mfa.arity << 8)); } @@ -5029,7 +5027,7 @@ do { \ * ... remainder of original BEAM code */ ErtsCodeInfo *ci = erts_code_to_codeinfo(I); - struct hipe_call_count *hcc = (struct hipe_call_count*)ci->native; + struct hipe_call_count *hcc = ci->u.hcc; ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); ASSERT(hcc != NULL); ASSERT(VALID_INSTR(hcc->opcode)); diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 6eea963016..7b79e6303b 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -913,7 +913,7 @@ erts_alloc_loader_state(void) magic = erts_create_magic_binary(sizeof(LoaderState), loader_state_dtor); - erts_refc_inc(&magic->refc, 1); + erts_refc_inc(&magic->intern.refc, 1); stp = ERTS_MAGIC_BIN_DATA(magic); stp->bin = NULL; stp->function = THE_NON_VALUE; /* Function not known yet */ @@ -996,9 +996,7 @@ static void free_loader_state(Binary* magic) { loader_state_dtor(magic); - if (erts_refc_dectest(&magic->refc, 0) == 0) { - erts_bin_free(magic); - } + erts_bin_release(magic); } static ErlHeapFragment* new_literal_fragment(Uint size) @@ -5672,9 +5670,7 @@ erts_release_literal_area(ErtsLiteralArea* literal_area) Binary* bptr; ASSERT(thing_subtag(oh->thing_word) == REFC_BINARY_SUBTAG); bptr = ((ProcBin*)oh)->val; - if (erts_refc_dectest(&bptr->refc, 0) == 0) { - erts_bin_free(bptr); - } + erts_bin_release(bptr); oh = oh->next; } erts_free(ERTS_ALC_T_LITERAL, literal_area); @@ -5708,12 +5704,13 @@ erts_is_module_native(BeamCodeHeader* code_hdr) static Eterm native_addresses(Process* p, BeamCodeHeader* code_hdr) { + Eterm result = NIL; +#ifdef HIPE int i; Eterm* hp; Uint num_functions; Uint need; Eterm* hp_end; - Eterm result = NIL; num_functions = code_hdr->num_functions; need = (6+BIG_UINT_HEAP_SIZE)*num_functions; @@ -5725,16 +5722,17 @@ native_addresses(Process* p, BeamCodeHeader* code_hdr) ASSERT(is_atom(ci->mfa.function) || is_nil(ci->mfa.function)); /* [] if BIF stub */ - if (ci->native != 0) { + if (ci->u.ncallee != NULL) { Eterm addr; ASSERT(is_atom(ci->mfa.function)); - addr = erts_bld_uint(&hp, NULL, ci->native); + addr = erts_bld_uint(&hp, NULL, (Uint)ci->u.ncallee); tuple = erts_bld_tuple(&hp, NULL, 3, ci->mfa.function, make_small(ci->mfa.arity), addr); result = erts_bld_cons(&hp, NULL, tuple, result); } } HRelease(p, hp_end, hp); +#endif return result; } @@ -6033,7 +6031,7 @@ make_stub(ErtsCodeInfo* info, Eterm mod, Eterm func, Uint arity, Uint native, Be DBG_TRACE_MFA(mod,func,arity,"make beam stub at %p", erts_codeinfo_to_code(info)); ASSERT(WORDS_PER_FUNCTION == 6); info->op = (BeamInstr) BeamOp(op_i_func_info_IaaI); - info->native = native; + info->u.ncallee = (void (*)(void)) native; info->mfa.module = mod; info->mfa.function = func; info->mfa.arity = arity; @@ -6104,7 +6102,7 @@ stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci) Lambda* lp; if (is_bif(ci->mfa.module, ci->mfa.function, ci->mfa.arity)) { - ci->native = 0; + ci->u.ncallee = NULL; ci->mfa.module = 0; ci->mfa.function = 0; ci->mfa.arity = 0; @@ -6267,16 +6265,7 @@ patch_funentries(Eterm Patchlist) fe = erts_get_fun_entry(Mod, uniq, index); fe->native_address = (Uint *)native_address; - /* Deliberate MEMORY LEAK of native fun entries!!! - * - * Uncomment line below when hipe code upgrade and purging works correctly. - * Today we may get cases when old (leaked) native code of a purged module - * gets called and tries to create instances of a deleted fun entry. - * - * Reproduced on a debug emulator with stdlib_test/qlc_SUITE:join_merge - * - * erts_smp_refc_dec(&fe->refc, 1); - */ + erts_smp_refc_dec(&fe->refc, 1); if (!patch(Addresses, (Uint) fe)) return 0; diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 214de3652f..84c9f9d7c4 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -3109,7 +3109,7 @@ BIF_RETTYPE integer_to_list_1(BIF_ALIST_1) * On error returns: {error,not_a_list}, or {error, no_integer} */ -BIF_RETTYPE string_to_integer_1(BIF_ALIST_1) +BIF_RETTYPE string_list_to_integer_1(BIF_ALIST_1) { Eterm res; Eterm tail; @@ -3295,7 +3295,7 @@ BIF_RETTYPE float_to_binary_2(BIF_ALIST_2) #define LOAD_E(xi,xim,xl,xlm) ((xi)=(xim), (xl)=(xlm)) #define STRING_TO_FLOAT_BUF_INC_SZ (128) -BIF_RETTYPE string_to_float_1(BIF_ALIST_1) +BIF_RETTYPE string_list_to_float_1(BIF_ALIST_1) { Eterm orig = BIF_ARG_1; Eterm list = orig; @@ -4254,6 +4254,75 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } +BIF_RETTYPE list_to_port_1(BIF_ALIST_1) +{ + /* + * A valid port identifier is on the format + * "#Port<N.P>" where N is node and P is + * the port id. Both N and P are of type Uint32. + */ + Uint32 n, p; + char* cp; + int i; + DistEntry *dep = NULL; + char buf[6 /* #Port< */ + + (2)*(10 + 1) /* N.P> */ + + 1 /* \0 */]; + + /* walk down the list and create a C string */ + if ((i = intlist_to_buf(BIF_ARG_1, buf, sizeof(buf)-1)) < 0) + goto bad; + + buf[i] = '\0'; /* null terminal */ + + cp = &buf[0]; + if (strncmp("#Port<", cp, 6) != 0) + goto bad; + + cp += 6; /* strlen("#Port<") */ + + if (sscanf(cp, "%u.%u>", &n, &p) < 2) + goto bad; + + if (p > ERTS_MAX_PORT_NUMBER) + goto bad; + + dep = erts_channel_no_to_dist_entry(n); + + if (!dep) + goto bad; + + if(dep == erts_this_dist_entry) { + erts_deref_dist_entry(dep); + BIF_RET(make_internal_port(p)); + } + else { + ExternalThing *etp; + ErlNode *enp; + + if (is_nil(dep->cid)) + goto bad; + + enp = erts_find_or_insert_node(dep->sysname, dep->creation); + ASSERT(enp != erts_this_node); + + etp = (ExternalThing *) HAlloc(BIF_P, EXTERNAL_THING_HEAD_SIZE + 1); + etp->header = make_external_port_header(1); + etp->next = MSO(BIF_P).first; + etp->node = enp; + etp->data.ui[0] = p; + + MSO(BIF_P).first = (struct erl_off_heap_header*) etp; + erts_deref_dist_entry(dep); + BIF_RET(make_external_port(etp)); + } + + bad: + if (dep) + erts_deref_dist_entry(dep); + BIF_ERROR(BIF_P, BADARG); +} + BIF_RETTYPE list_to_ref_1(BIF_ALIST_1) { /* diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 66e5dc2988..3f6d82d65c 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -88,6 +88,7 @@ bif erlang:list_to_binary/1 bif erlang:list_to_float/1 bif erlang:list_to_integer/1 bif erlang:list_to_pid/1 +bif erlang:list_to_port/1 bif erlang:list_to_ref/1 bif erlang:list_to_tuple/1 bif erlang:loaded/0 @@ -460,8 +461,8 @@ bif error_logger:warning_map/0 bif erlang:get_module_info/1 bif erlang:get_module_info/2 ubif erlang:is_boolean/1 -bif string:to_integer/1 -bif string:to_float/1 +bif string:list_to_integer/1 +bif string:list_to_float/1 bif erlang:make_fun/3 bif erlang:iolist_size/1 bif erlang:iolist_to_binary/1 diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index 4dd8316dad..0df6bbb289 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -84,7 +84,6 @@ new_binary(Process *p, byte *buf, Uint len) * Allocate the binary struct itself. */ bptr = erts_bin_nrml_alloc(len); - erts_refc_init(&bptr->refc, 1); if (buf != NULL) { sys_memcpy(bptr->orig_bytes, buf, len); } @@ -121,7 +120,6 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, Uint len) * Allocate the binary struct itself. */ bptr = erts_bin_nrml_alloc(len); - erts_refc_init(&bptr->refc, 1); if (buf != NULL) { sys_memcpy(bptr->orig_bytes, buf, len); } diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 0b40d70cb7..e23fdb83d9 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -590,7 +590,7 @@ do_break(void) #endif #ifdef DEBUG case 't': - erts_p_slpq(); + /* erts_p_slpq(); */ return; case 'b': bin_check(); @@ -660,7 +660,7 @@ bin_check(void) erts_printf("%p orig_size: %bpd, norefs = %bpd\n", bp->val, bp->val->orig_size, - erts_refc_read(&bp->val->refc, 1)); + erts_refc_read(&bp->val->intern.refc, 1)); } } if (printed) { diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h index 1b451bf921..e802ad5dd7 100644 --- a/erts/emulator/beam/code_ix.h +++ b/erts/emulator/beam/code_ix.h @@ -80,7 +80,13 @@ typedef struct ErtsCodeMFA_ { in ops.tab to reflect the new func_info size */ typedef struct ErtsCodeInfo_ { BeamInstr op; /* OpCode(i_func_info) */ - BeamInstr native; /* Used by hipe and trace to store extra data */ + union { + struct generic_bp* gen_bp; /* Trace breakpoint */ +#ifdef HIPE + void (*ncallee)(void); + struct hipe_call_count* hcc; +#endif + }u; ErtsCodeMFA mfa; } ErtsCodeInfo; diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 264ba89e8b..62c3f9520d 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -761,7 +761,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint } *argp = make_binary(hbot); pb = (ProcBin*) hbot; - erts_refc_inc(&pb->val->refc, 2); + erts_refc_inc(&pb->val->intern.refc, 2); pb->next = off_heap->first; pb->flags = 0; off_heap->first = (struct erl_off_heap_header*) pb; @@ -809,7 +809,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint to->thing_word = HEADER_PROC_BIN; to->size = real_size; to->val = from->val; - erts_refc_inc(&to->val->refc, 2); + erts_refc_inc(&to->val->intern.refc, 2); to->bytes = from->bytes + offset; to->next = off_heap->first; to->flags = 0; @@ -901,7 +901,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint case REF_SUBTAG: if (is_magic_ref_thing(objp)) { ErtsMRefThing *mreft = (ErtsMRefThing *) objp; - erts_refc_inc(&mreft->mb->refc, 2); + erts_refc_inc(&mreft->mb->intern.refc, 2); goto L_off_heap_node_container_common; } /* Fall through... */ @@ -1585,7 +1585,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, while (sz-- > 0) { *hp++ = *ptr++; } - erts_refc_inc(&pb->val->refc, 2); + erts_refc_inc(&pb->val->intern.refc, 2); pb->next = off_heap->first; pb->flags = 0; off_heap->first = (struct erl_off_heap_header*) pb; @@ -1644,7 +1644,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, to->thing_word = HEADER_PROC_BIN; to->size = real_size; to->val = from->val; - erts_refc_inc(&to->val->refc, 2); + erts_refc_inc(&to->val->intern.refc, 2); to->bytes = from->bytes + offset; to->next = off_heap->first; to->flags = 0; @@ -1678,7 +1678,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, case REF_SUBTAG: if (is_magic_ref_thing(ptr)) { ErtsMRefThing *mreft = (ErtsMRefThing *) ptr; - erts_refc_inc(&mreft->mb->refc, 2); + erts_refc_inc(&mreft->mb->intern.refc, 2); goto off_heap_node_container_common; } /* Fall through... */ @@ -1847,7 +1847,7 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) case REFC_BINARY_SUBTAG: { ProcBin* pb = (ProcBin *) (tp-1); - erts_refc_inc(&pb->val->refc, 2); + erts_refc_inc(&pb->val->intern.refc, 2); OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); } goto off_heap_common; @@ -1882,7 +1882,7 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) ErtsRefThing *rtp = (ErtsRefThing *) (tp - 1); if (is_magic_ref_thing(rtp)) { ErtsMRefThing *mreft = (ErtsMRefThing *) rtp; - erts_refc_inc(&mreft->mb->refc, 2); + erts_refc_inc(&mreft->mb->intern.refc, 2); goto off_heap_common; } /* Fall through... */ diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index d1c2da9074..a1581908e5 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -629,7 +629,6 @@ alloc_dist_obuf(Uint size) ErtsDistOutputBuf *obuf; Uint obuf_size = sizeof(ErtsDistOutputBuf)+sizeof(byte)*(size-1); Binary *bin = erts_bin_drv_alloc(obuf_size); - erts_refc_init(&bin->refc, 1); obuf = (ErtsDistOutputBuf *) &bin->orig_bytes[0]; #ifdef DEBUG obuf->dbg_pattern = ERTS_DIST_OUTPUT_BUF_DBG_PATTERN; @@ -643,8 +642,7 @@ free_dist_obuf(ErtsDistOutputBuf *obuf) { Binary *bin = ErtsDistOutputBuf2Binary(obuf); ASSERT(obuf->dbg_pattern == ERTS_DIST_OUTPUT_BUF_DBG_PATTERN); - if (erts_refc_dectest(&bin->refc, 0) == 0) - erts_bin_free(bin); + erts_bin_release(bin); } static ERTS_INLINE Sint diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index a374593c5d..71957b2259 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -678,10 +678,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) = erts_timer_type_size(ERTS_ALC_T_HL_PTIMER); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_BIF_TIMER)] = erts_timer_type_size(ERTS_ALC_T_BIF_TIMER); -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_ABIF_TIMER)] - = erts_timer_type_size(ERTS_ALC_T_ABIF_TIMER); -#endif fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_EXP_TRACE)] = sizeof(NifExportTrace); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MREF_NSCHED_ENT)] @@ -2440,12 +2436,6 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) &size.processes_used, fi, ERTS_ALC_T_BIF_TIMER); -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - add_fix_values(&size.processes, - &size.processes_used, - fi, - ERTS_ALC_T_ABIF_TIMER); -#endif add_fix_values(&size.processes, &size.processes_used, fi, diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 43f43f9034..f296a98125 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -168,7 +168,6 @@ type TIMER_SERVICE LONG_LIVED SYSTEM timer_service type LL_PTIMER FIXED_SIZE PROCESSES ll_ptimer type HL_PTIMER FIXED_SIZE PROCESSES hl_ptimer type BIF_TIMER FIXED_SIZE PROCESSES bif_timer -# type ABIF_TIMER FIXED_SIZE PROCESSES accessor_bif_timer type TIMER_REQUEST SHORT_LIVED PROCESSES timer_request type BTM_YIELD_STATE SHORT_LIVED PROCESSES btm_yield_state type REG_TABLE STANDARD SYSTEM reg_tab diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index 62a752d854..f79b5b6843 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -2669,7 +2669,6 @@ static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en) } cbs->result = erts_bin_nrml_alloc(target_size); /* Always offheap if trapping */ - erts_refc_init(&(cbs->result->refc), 1); t = (byte *) cbs->result->orig_bytes; /* No offset or anything */ pos = 0; i = 0; diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 3a8687dc59..3a70c6036b 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -177,7 +177,7 @@ bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh) if (szp) *szp += 4+2; if (hpp) { - Uint refc = (Uint) erts_refc_read(&pb->val->refc, 1); + Uint refc = (Uint) erts_refc_read(&pb->val->intern.refc, 1); tuple = TUPLE3(*hpp, val, orig_size, make_small(refc)); res = CONS(*hpp + 4, tuple, res); *hpp += 4+2; @@ -203,7 +203,7 @@ bld_magic_ref_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh) if (szp) *szp += 4+2; if (hpp) { - Uint refc = (Uint) erts_refc_read(&mrtp->mb->refc, 1); + Uint refc = (Uint) erts_refc_read(&mrtp->mb->intern.refc, 1); tuple = TUPLE3(*hpp, val, orig_size, make_small(refc)); res = CONS(*hpp + 4, tuple, res); *hpp += 4+2; @@ -3984,7 +3984,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(am_false); } bin = erts_magic_ref2bin(tp[2]); - refc = erts_refc_read(&bin->refc, 1); + refc = erts_refc_read(&bin->intern.refc, 1); bin_addr = (UWord) bin; sz = 4; erts_bld_uword(NULL, &sz, bin_addr); diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index f471390501..023bfca797 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -1737,7 +1737,7 @@ setup_bif_trace(void) for (i = 0; i < BIF_SIZE; ++i) { Export *ep = bif_export[i]; - GenericBp* g = (GenericBp *) ep->info.native; + GenericBp* g = ep->info.u.gen_bp; if (g) { if (ExportIsBuiltIn(ep)) { ASSERT(ep->beam[1]); @@ -1755,7 +1755,7 @@ reset_bif_trace(void) for (i = 0; i < BIF_SIZE; ++i) { Export *ep = bif_export[i]; - GenericBp* g = (GenericBp *) ep->info.native; + GenericBp* g = ep->info.u.gen_bp; if (g && g->data[active].flags == 0) { if (ExportIsBuiltIn(ep)) { ASSERT(ep->beam[1]); diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c index 3fd4c87094..7f438daec0 100644 --- a/erts/emulator/beam/erl_bif_unique.c +++ b/erts/emulator/beam/erl_bif_unique.c @@ -197,7 +197,7 @@ erts_magic_ref_lookup_bin__(Uint32 refn[ERTS_REF_NUMBERS]) else { erts_aint_t refc; mb = tep->mb; - refc = erts_refc_inc_unless(&mb->refc, 0, 0); + refc = erts_refc_inc_unless(&mb->intern.refc, 0, 0); if (refc == 0) mb = NULL; } diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h index 27c2a15a5e..0f3f794878 100644 --- a/erts/emulator/beam/erl_bif_unique.h +++ b/erts/emulator/beam/erl_bif_unique.h @@ -178,10 +178,10 @@ ERTS_GLB_INLINE Eterm erts_mk_magic_ref(Eterm **hpp, ErlOffHeap *ohp, Binary *bp) { Eterm *hp = *hpp; - ASSERT(bp->flags & BIN_FLAG_MAGIC); + ASSERT(bp->intern.flags & BIN_FLAG_MAGIC); write_magic_ref_thing(hp, ohp, (ErtsMagicBinary *) bp); *hpp += ERTS_MAGIC_REF_THING_SIZE; - erts_refc_inc(&bp->refc, 1); + erts_refc_inc(&bp->intern.refc, 1); OH_OVERHEAD(ohp, bp->orig_size / sizeof(Eterm)); return make_internal_ref(hp); } @@ -297,14 +297,14 @@ erts_iref_storage_save(ErtsIRefStorage *iref, Eterm ref) ASSERT(is_magic_ref_thing(hp)); iref->is_magic = 1; iref->u.mb = mrtp->mb; - erts_refc_inc(&mrtp->mb->refc, 1); + erts_refc_inc(&mrtp->mb->intern.refc, 1); } } ERTS_GLB_INLINE void erts_iref_storage_clean(ErtsIRefStorage *iref) { - if (iref->is_magic && erts_refc_dectest(&iref->u.mb->refc, 0) == 0) + if (iref->is_magic && erts_refc_dectest(&iref->u.mb->intern.refc, 0) == 0) erts_ref_bin_free(iref->u.mb); #ifdef DEBUG memset((void *) iref, 0xf, sizeof(ErtsIRefStorage)); @@ -337,7 +337,7 @@ erts_iref_storage_make_ref(ErtsIRefStorage *iref, * refc increment of the cleaned storage... */ if (!clean_storage) - erts_refc_inc(&iref->u.mb->refc, 1); + erts_refc_inc(&iref->u.mb->intern.refc, 1); } #ifdef DEBUG diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index 6ff71ec6d1..de7dbf4e20 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -42,14 +42,16 @@ #define ERTS_BINARY_STRUCT_ALIGNMENT #endif -/* Add fields in ERTS_INTERNAL_BINARY_FIELDS, otherwise the drivers crash */ -#define ERTS_INTERNAL_BINARY_FIELDS \ - UWord flags; \ - erts_refc_t refc; \ +/* Add fields in binary_internals, otherwise the drivers crash */ +struct binary_internals { + UWord flags; + erts_refc_t refc; ERTS_BINARY_STRUCT_ALIGNMENT +}; + typedef struct binary { - ERTS_INTERNAL_BINARY_FIELDS + struct binary_internals intern; SWord orig_size; char orig_bytes[1]; /* to be continued */ } Binary; @@ -63,7 +65,7 @@ typedef struct binary { typedef struct magic_binary ErtsMagicBinary; struct magic_binary { - ERTS_INTERNAL_BINARY_FIELDS + struct binary_internals intern; SWord orig_size; int (*destructor)(Binary *); Uint32 refn[ERTS_REF_NUMBERS]; @@ -87,7 +89,7 @@ typedef union { Binary binary; ErtsMagicBinary magic_binary; struct { - ERTS_INTERNAL_BINARY_FIELDS + struct binary_internals intern; ErlDrvBinary binary; } driver; } ErtsBinary; @@ -316,6 +318,7 @@ ERTS_GLB_INLINE Binary *erts_bin_nrml_alloc(Uint size); ERTS_GLB_INLINE Binary *erts_bin_realloc_fnf(Binary *bp, Uint size); ERTS_GLB_INLINE Binary *erts_bin_realloc(Binary *bp, Uint size); ERTS_GLB_INLINE void erts_bin_free(Binary *bp); +ERTS_GLB_INLINE void erts_bin_release(Binary *bp); ERTS_GLB_INLINE Binary *erts_create_magic_binary_x(Uint size, int (*destructor)(Binary *), ErtsAlcType_t alloc_type, @@ -374,7 +377,8 @@ erts_bin_drv_alloc_fnf(Uint size) ERTS_CHK_BIN_ALIGNMENT(res); if (res) { res->orig_size = size; - res->flags = BIN_FLAG_DRV; + res->intern.flags = BIN_FLAG_DRV; + erts_refc_init(&res->intern.refc, 1); } return res; } @@ -392,7 +396,8 @@ erts_bin_drv_alloc(Uint size) res = erts_alloc(ERTS_ALC_T_DRV_BINARY, bsize); ERTS_CHK_BIN_ALIGNMENT(res); res->orig_size = size; - res->flags = BIN_FLAG_DRV; + res->intern.flags = BIN_FLAG_DRV; + erts_refc_init(&res->intern.refc, 1); return res; } @@ -410,7 +415,8 @@ erts_bin_nrml_alloc(Uint size) res = erts_alloc(ERTS_ALC_T_BINARY, bsize); ERTS_CHK_BIN_ALIGNMENT(res); res->orig_size = size; - res->flags = 0; + res->intern.flags = 0; + erts_refc_init(&res->intern.refc, 1); return res; } @@ -419,9 +425,9 @@ erts_bin_realloc_fnf(Binary *bp, Uint size) { Binary *nbp; Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD; - ErtsAlcType_t type = (bp->flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY + ErtsAlcType_t type = (bp->intern.flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY : ERTS_ALC_T_BINARY; - ASSERT((bp->flags & BIN_FLAG_MAGIC) == 0); + ASSERT((bp->intern.flags & BIN_FLAG_MAGIC) == 0); if (bsize < size) /* overflow */ return NULL; nbp = erts_realloc_fnf(type, (void *) bp, bsize); @@ -436,9 +442,9 @@ erts_bin_realloc(Binary *bp, Uint size) { Binary *nbp; Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD; - ErtsAlcType_t type = (bp->flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY + ErtsAlcType_t type = (bp->intern.flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY : ERTS_ALC_T_BINARY; - ASSERT((bp->flags & BIN_FLAG_MAGIC) == 0); + ASSERT((bp->intern.flags & BIN_FLAG_MAGIC) == 0); if (bsize < size) /* overflow */ erts_realloc_enomem(type, bp, size); nbp = erts_realloc_fnf(type, (void *) bp, bsize); @@ -452,7 +458,7 @@ erts_bin_realloc(Binary *bp, Uint size) ERTS_GLB_INLINE void erts_bin_free(Binary *bp) { - if (bp->flags & BIN_FLAG_MAGIC) { + if (bp->intern.flags & BIN_FLAG_MAGIC) { if (!ERTS_MAGIC_BIN_DESTRUCTOR(bp)(bp)) { /* Destructor took control of the deallocation */ return; @@ -460,12 +466,20 @@ erts_bin_free(Binary *bp) erts_magic_ref_remove_bin(ERTS_MAGIC_BIN_REFN(bp)); erts_free(ERTS_MAGIC_BIN_ATYPE(bp), (void *) bp); } - else if (bp->flags & BIN_FLAG_DRV) + else if (bp->intern.flags & BIN_FLAG_DRV) erts_free(ERTS_ALC_T_DRV_BINARY, (void *) bp); else erts_free(ERTS_ALC_T_BINARY, (void *) bp); } +ERTS_GLB_INLINE void +erts_bin_release(Binary *bp) +{ + if (erts_refc_dectest(&bp->intern.refc, 0) == 0) { + erts_bin_free(bp); + } +} + ERTS_GLB_INLINE Binary * erts_create_magic_binary_x(Uint size, int (*destructor)(Binary *), ErtsAlcType_t alloc_type, @@ -478,10 +492,10 @@ erts_create_magic_binary_x(Uint size, int (*destructor)(Binary *), if (!bptr) erts_alloc_n_enomem(ERTS_ALC_T2N(alloc_type), bsize); ERTS_CHK_BIN_ALIGNMENT(bptr); - bptr->flags = BIN_FLAG_MAGIC; + bptr->intern.flags = BIN_FLAG_MAGIC; bptr->orig_size = unaligned ? ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(size) : ERTS_MAGIC_BIN_ORIG_SIZE(size); - erts_refc_init(&bptr->refc, 0); + erts_refc_init(&bptr->intern.refc, 0); ERTS_MAGIC_BIN_DESTRUCTOR(bptr) = destructor; ERTS_MAGIC_BIN_ATYPE(bptr) = alloc_type; erts_make_magic_ref_in_array(ERTS_MAGIC_BIN_REFN(bptr)); @@ -509,7 +523,7 @@ ERTS_GLB_INLINE erts_smp_atomic_t * erts_smp_binary_to_magic_indirection(Binary *bp) { ErtsMagicIndirectionWord *mip; - ASSERT(bp->flags & BIN_FLAG_MAGIC); + ASSERT(bp->intern.flags & BIN_FLAG_MAGIC); ASSERT(ERTS_MAGIC_BIN_ATYPE(bp) == ERTS_ALC_T_MINDIRECTION); mip = ERTS_MAGIC_BIN_UNALIGNED_DATA(bp); return &mip->smp_atomic_word; @@ -519,7 +533,7 @@ ERTS_GLB_INLINE erts_atomic_t * erts_binary_to_magic_indirection(Binary *bp) { ErtsMagicIndirectionWord *mip; - ASSERT(bp->flags & BIN_FLAG_MAGIC); + ASSERT(bp->intern.flags & BIN_FLAG_MAGIC); ASSERT(ERTS_MAGIC_BIN_ATYPE(bp) == ERTS_ALC_T_MINDIRECTION); mip = ERTS_MAGIC_BIN_UNALIGNED_DATA(bp); return &mip->atomic_word; diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index 885e955332..df3f6ad557 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -1404,7 +1404,6 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term, * Allocate the binary data struct itself. */ bptr = erts_bin_nrml_alloc(bin_size); - erts_refc_init(&bptr->refc, 1); erts_current_bin = (byte *) bptr->orig_bytes; /* @@ -1518,14 +1517,11 @@ erts_bs_private_append(Process* p, Eterm bin, Eterm build_size_term, Uint unit) * binary and copy the contents of the old binary into it. */ Binary* bptr = erts_bin_nrml_alloc(new_size); - erts_refc_init(&bptr->refc, 1); sys_memcpy(bptr->orig_bytes, binp->orig_bytes, binp->orig_size); pb->flags |= PB_IS_WRITABLE | PB_ACTIVE_WRITER; pb->val = bptr; pb->bytes = (byte *) bptr->orig_bytes; - if (erts_refc_dectest(&binp->refc, 0) == 0) { - erts_bin_free(binp); - } + erts_bin_release(binp); } } erts_current_bin = pb->bytes; @@ -1565,7 +1561,6 @@ erts_bs_init_writable(Process* p, Eterm sz) * Allocate the binary data struct itself. */ bptr = erts_bin_nrml_alloc(bin_size); - erts_refc_init(&bptr->refc, 1); /* * Now allocate the ProcBin on the heap. diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 378328856d..98c689f13f 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -217,7 +217,7 @@ make_btid(DbTable *tb) * and table is refered once by being alive... */ erts_smp_refc_init(&tb->common.refc, 2); - erts_refc_inc(&btid->refc, 1); + erts_refc_inc(&btid->intern.refc, 1); } static ERTS_INLINE DbTable* btid2tab(Binary* btid) @@ -402,8 +402,8 @@ free_dbtable(void *vtb) #endif ASSERT(is_immed(tb->common.heir_data)); - if (tb->common.btid && erts_refc_dectest(&tb->common.btid->refc, 0) == 0) - erts_bin_free(tb->common.btid); + if (tb->common.btid) + erts_bin_release(tb->common.btid); erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable)); } @@ -3616,9 +3616,7 @@ static SWord proc_cleanup_fixed_table(Process* p, DbFixation* fix) ASSERT(fix->counter == 0); } - if (erts_refc_dectest(&fix->tabs.btid->refc, 0) == 0) { - erts_bin_free(fix->tabs.btid); - } + erts_bin_release(fix->tabs.btid); erts_free(ERTS_ALC_T_DB_FIXATION, fix); ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation)); ++work; @@ -3783,7 +3781,7 @@ static void fix_table_locked(Process* p, DbTable* tb) tb, sizeof(DbFixation)); ERTS_ETS_MISC_MEM_ADD(sizeof(DbFixation)); fix->tabs.btid = tb->common.btid; - erts_refc_inc(&fix->tabs.btid->refc, 2); + erts_refc_inc(&fix->tabs.btid->intern.refc, 2); fix->procs.p = p; fix->counter = 1; fixing_procs_rbt_insert(&tb->common.fixing_procs, fix); @@ -3819,7 +3817,7 @@ static void unfix_table_locked(Process* p, DbTable* tb, #endif fixed_tabs_delete(p, fix); - erts_refc_dec(&fix->tabs.btid->refc, 1); + erts_refc_dec(&fix->tabs.btid->intern.refc, 1); erts_db_free(ERTS_ALC_T_DB_FIXATION, tb, (void *) fix, sizeof(DbFixation)); @@ -3889,9 +3887,7 @@ static void free_fixations_op(DbFixation* fix, void* vctx) { fixed_tabs_delete(fix->procs.p, fix); - if (erts_refc_dectest(&fix->tabs.btid->refc, 0) == 0) { - erts_bin_free(fix->tabs.btid); - } + erts_bin_release(fix->tabs.btid); erts_db_free(ERTS_ALC_T_DB_FIXATION, ctx->tb, (void *) fix, sizeof(DbFixation)); @@ -3906,9 +3902,8 @@ int erts_db_execute_free_fixation(Process* p, DbFixation* fix) ASSERT(fix->counter == 0); fixed_tabs_delete(p, fix); - if (erts_refc_dectest(&fix->tabs.btid->refc, 0) == 0) { - erts_bin_free(fix->tabs.btid); - } + erts_bin_release(fix->tabs.btid); + erts_free(ERTS_ALC_T_DB_FIXATION, fix); ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation)); return 1; diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 7ab27df00c..80c4824eeb 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -1258,7 +1258,7 @@ static int match_traverse(Process* p, DbTableHash* tb, } if (mpi.all_objects) { - mpi.mp->flags |= BIN_FLAG_ALL_OBJECTS; + mpi.mp->intern.flags |= BIN_FLAG_ALL_OBJECTS; } /* @@ -1383,7 +1383,7 @@ static int match_traverse_continue(Process* p, DbTableHash* tb, void* context_ptr, /* For callbacks */ Eterm* ret) { - int all_objects = (*mpp)->flags & BIN_FLAG_ALL_OBJECTS; + int all_objects = (*mpp)->intern.flags & BIN_FLAG_ALL_OBJECTS; HashDbTerm** current_ptr; /* Refers to either the bucket pointer or * the 'next' pointer in the previous term */ diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index ab8da6ccf6..f6918b8ec4 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -995,7 +995,7 @@ static int db_select_continue_tree(Process *p, sc.lastobj = NULL; sc.max = 1000; sc.keypos = tb->common.keypos; - sc.all_objects = mp->flags & BIN_FLAG_ALL_OBJECTS; + sc.all_objects = mp->intern.flags & BIN_FLAG_ALL_OBJECTS; sc.chunk_size = chunk_size; reverse = unsigned_val(tptr[7]); sc.got = signed_val(tptr[8]); @@ -1187,7 +1187,7 @@ static int db_select_tree(Process *p, DbTable *tbl, Eterm tid, hp = HAlloc(p, 9 + sz + ERTS_MAGIC_REF_THING_SIZE); key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) - (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; + (mpi.mp)->intern.flags |= BIN_FLAG_ALL_OBJECTS; mpb= erts_db_make_match_prog_ref(p,mpi.mp,&hp); continuation = TUPLE8 @@ -1385,7 +1385,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, Eterm tid, } key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) - (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; + (mpi.mp)->intern.flags |= BIN_FLAG_ALL_OBJECTS; mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp); continuation = TUPLE5 @@ -1510,7 +1510,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, Eterm tid, hp = HAlloc(p, 9 + sz + ERTS_MAGIC_REF_THING_SIZE); key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) - (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; + (mpi.mp)->intern.flags |= BIN_FLAG_ALL_OBJECTS; mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp); continuation = TUPLE8 @@ -1536,7 +1536,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, Eterm tid, key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) - (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; + (mpi.mp)->intern.flags |= BIN_FLAG_ALL_OBJECTS; mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp); continuation = TUPLE8 (hp, @@ -1932,7 +1932,7 @@ static int db_select_replace_tree(Process *p, DbTable *tbl, Eterm tid, } key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) - (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; + (mpi.mp)->intern.flags |= BIN_FLAG_ALL_OBJECTS; mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp); continuation = TUPLE5 diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 03cc11bdc4..24b22eafb8 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -3265,9 +3265,7 @@ void db_cleanup_offheap_comp(DbTerm* obj) } switch (thing_subtag(u.hdr->thing_word)) { case REFC_BINARY_SUBTAG: - if (erts_refc_dectest(&u.pb->val->refc, 0) == 0) { - erts_bin_free(u.pb->val); - } + erts_bin_release(u.pb->val); break; case FUN_SUBTAG: ASSERT(u.pb != &tmp); @@ -3277,8 +3275,7 @@ void db_cleanup_offheap_comp(DbTerm* obj) break; case REF_SUBTAG: ASSERT(is_magic_ref_thing(u.hdr)); - if (erts_refc_dectest(&u.mref->mb->refc, 0) == 0) - erts_bin_free((Binary *)u.mref->mb); + erts_bin_release((Binary *)u.mref->mb); break; default: ASSERT(is_external_header(u.hdr->thing_word)); diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 9be77fcefa..ed7b9c8618 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -504,7 +504,7 @@ ERTS_GLB_INLINE Binary * erts_db_get_match_prog_binary_unchecked(Eterm term) { Binary *bp = erts_magic_ref2bin(term); - ASSERT(bp->flags & BIN_FLAG_MAGIC); + ASSERT(bp->intern.flags & BIN_FLAG_MAGIC); ASSERT((ERTS_MAGIC_BIN_DESTRUCTOR(bp) == erts_db_match_prog_destructor)); return bp; } @@ -516,7 +516,7 @@ erts_db_get_match_prog_binary(Eterm term) if (!is_internal_magic_ref(term)) return NULL; bp = erts_magic_ref2bin(term); - ASSERT(bp->flags & BIN_FLAG_MAGIC); + ASSERT(bp->intern.flags & BIN_FLAG_MAGIC); if (ERTS_MAGIC_BIN_DESTRUCTOR(bp) != erts_db_match_prog_destructor) return NULL; return bp; @@ -528,7 +528,7 @@ erts_db_get_match_prog_binary(Eterm term) ** Convenience when compiling into Binary structures */ #define IsMatchProgBinary(BP) \ - (((BP)->flags & BIN_FLAG_MAGIC) \ + (((BP)->intern.flags & BIN_FLAG_MAGIC) \ && ERTS_MAGIC_BIN_DESTRUCTOR((BP)) == erts_db_match_prog_destructor) #define Binary2MatchProg(BP) \ diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index d51d4fff45..a991c2c164 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1252,7 +1252,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, * link it into the MSO list for the process. */ - erts_refc_inc(&bptr->refc, 1); + erts_refc_inc(&bptr->intern.refc, 1); *prev = ptr; prev = &ptr->next; } @@ -2889,9 +2889,7 @@ sweep_off_heap(Process *p, int fullsweep) case REFC_BINARY_SUBTAG: { Binary* bptr = ((ProcBin*)ptr)->val; - if (erts_refc_dectest(&bptr->refc, 0) == 0) { - erts_bin_free(bptr); - } + erts_bin_release(bptr); break; } case FUN_SUBTAG: @@ -2907,8 +2905,7 @@ sweep_off_heap(Process *p, int fullsweep) ErtsMagicBinary *bptr; ASSERT(is_magic_ref_thing(ptr)); bptr = ((ErtsMRefThing *) ptr)->mb; - if (erts_refc_dectest(&bptr->refc, 0) == 0) - erts_bin_free((Binary *) bptr); + erts_bin_release((Binary *) bptr); break; } default: @@ -3609,7 +3606,7 @@ erts_check_off_heap2(Process *p, Eterm *htop) erts_aint_t refc; switch (thing_subtag(u.hdr->thing_word)) { case REFC_BINARY_SUBTAG: - refc = erts_refc_read(&u.pb->val->refc, 1); + refc = erts_refc_read(&u.pb->val->intern.refc, 1); break; case FUN_SUBTAG: refc = erts_smp_refc_read(&u.fun->fe->refc, 1); @@ -3621,7 +3618,7 @@ erts_check_off_heap2(Process *p, Eterm *htop) break; case REF_SUBTAG: ASSERT(is_magic_ref_thing(u.hdr)); - refc = erts_refc_read(&u.mref->mb->refc, 1); + refc = erts_refc_read(&u.mref->mb->intern.refc, 1); break; default: ASSERT(!"erts_check_off_heap2: Invalid thing_word"); diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index 26be8c7edf..13d6136672 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -29,6 +29,8 @@ # include "config.h" #endif +/* #define ERTS_MAGIC_REF_BIF_TIMERS */ + #include "sys.h" #include "global.h" #include "bif.h" @@ -36,6 +38,9 @@ #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" #include "erl_hl_timer.h" +#ifdef ERTS_MAGIC_REF_BIF_TIMERS +#include "erl_binary.h" +#endif #define ERTS_TMR_CHECK_CANCEL_ON_CREATE 0 @@ -106,9 +111,6 @@ typedef enum { #define ERTS_TMR_ROFLG_PROC (((Uint32) 1) << 14) #define ERTS_TMR_ROFLG_PORT (((Uint32) 1) << 15) #define ERTS_TMR_ROFLG_CALLBACK (((Uint32) 1) << 16) -#ifdef ERTS_BTM_ACCESSOR_SUPPORT -#define ERTS_TMR_ROFLG_ABIF_TMR (((Uint32) 1) << 17) -#endif #define ERTS_TMR_ROFLG_SID_MASK \ (ERTS_TMR_ROFLG_HLT - (Uint32) 1) @@ -127,6 +129,13 @@ typedef struct ErtsHLTimer_ ErtsHLTimer; #define ERTS_HLT_PFIELD_NOT_IN_TABLE (~((UWord) 0)) +typedef struct ErtsBifTimer_ ErtsBifTimer; + +typedef struct { + ErtsBifTimer *next; + ErtsBifTimer *prev; +} ErtsBifTimerList; + typedef struct { UWord parent; /* parent pointer and flags... */ union { @@ -144,9 +153,9 @@ typedef struct { typedef struct { UWord parent; /* parent pointer and flags... */ - ErtsHLTimer *right; - ErtsHLTimer *left; -} ErtsHLTimerTree; + ErtsBifTimer *right; + ErtsBifTimer *left; +} ErtsBifTimerTree; typedef struct { Uint32 roflgs; @@ -155,67 +164,75 @@ typedef struct { void *arg; erts_atomic_t next; } u; + union { + Process *proc; + Port *port; + Eterm name; + void (*callback)(void *); + } receiver; } ErtsTmrHead; struct ErtsHLTimer_ { ErtsTmrHead head; /* NEED to be first! */ + ErtsMonotonicTime timeout; union { ErtsThrPrgrLaterOp cleanup; ErtsHLTimerTimeTree tree; } time; - ErtsMonotonicTime timeout; - union { - Process *proc; - Port *port; - Eterm name; - void (*callback)(void *); - } receiver; #ifdef ERTS_HLT_HARD_DEBUG int pending_timeout; #endif - - erts_smp_atomic32_t state; - - /* BIF timer only fields follow... */ - struct { - Uint32 refn[ERTS_REF_NUMBERS]; - ErtsHLTimerTree proc_tree; - ErtsHLTimerTree tree; - Eterm message; - ErlHeapFragment *bp; - } btm; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - struct { - Eterm accessor; - ErtsHLTimerTree tree; - } abtm; -#endif }; -#define ERTS_HL_PTIMER_SIZE offsetof(ErtsHLTimer, btm) -#ifdef ERTS_BTM_ACCESSOR_SUPPORT -#define ERTS_BIF_TIMER_SIZE offsetof(ErtsHLTimer, abtm) -#define ERTS_ABIF_TIMER_SIZE sizeof(ErtsHLTimer) -#else -#define ERTS_BIF_TIMER_SIZE sizeof(ErtsHLTimer) -#endif - typedef struct { ErtsTmrHead head; /* NEED to be first! */ union { - void *p; - void (*callback)(void *); + ErtsTWheelTimer tw_tmr; + ErtsThrPrgrLaterOp cleanup; } u; - ErtsTWheelTimer tw_tmr; } ErtsTWTimer; +struct ErtsBifTimer_ { + union { + ErtsTmrHead head; + ErtsHLTimer hlt; + ErtsTWTimer twt; + } type; + struct { + erts_smp_atomic32_t state; +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsMagicBinary *mbin; + ErtsHLTimerList proc_list; +#else + Uint32 refn[ERTS_REF_NUMBERS]; + ErtsBifTimerTree proc_tree; + ErtsBifTimerTree tree; +#endif + Eterm message; + ErlHeapFragment *bp; + } btm; +}; + typedef union { ErtsTmrHead head; ErtsHLTimer hlt; ErtsTWTimer twt; + ErtsBifTimer btm; } ErtsTimer; +typedef ErtsTimer *(*ErtsCreateTimerFunc)(ErtsSchedulerData *esdp, + ErtsMonotonicTime timeout_pos, + int short_time, ErtsTmrType type, + void *rcvrp, Eterm rcvr, + Eterm msg, +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsMagicBinary *mbin, +#else + Uint32 *refn, +#endif + void (*callback)(void *), void *arg); + #ifdef SMALL_MEMORY #define BIF_TIMER_PREALC_SZ 10 #define PTIMER_PREALC_SZ 10 @@ -225,7 +242,7 @@ typedef union { #endif ERTS_SCHED_PREF_PALLOC_IMPL(bif_timer_pre, - ErtsHLTimer, + ErtsBifTimer, BIF_TIMER_PREALC_SZ) ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(tw_timer, @@ -296,12 +313,16 @@ struct ErtsHLTimerService_ { ErtsHLTCncldTmrQ canceled_queue; #endif ErtsHLTimer *time_tree; - ErtsHLTimer *btm_tree; +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + ErtsBifTimer *btm_tree; +#endif ErtsHLTimer *next_timeout; ErtsYieldingTimeoutState yield; ErtsTWheelTimer service_timer; }; +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + static ERTS_INLINE int refn_is_lt(Uint32 *x, Uint32 *y) { @@ -317,6 +338,14 @@ refn_is_lt(Uint32 *x, Uint32 *y) return x[0] < y[0]; } +static ERTS_INLINE int +refn_is_eq(Uint32 *x, Uint32 *y) +{ + return (x[0] == y[0]) & (x[1] == y[1]) & (x[2] == y[2]); +} + +#endif + #define ERTS_RBT_PREFIX time #define ERTS_RBT_T ErtsHLTimer #define ERTS_RBT_KEY_T ErtsMonotonicTime @@ -506,8 +535,16 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) #endif /* ERTS_HLT_HARD_DEBUG */ +#ifdef ERTS_MAGIC_REF_BIF_TIMERS +#define ERTS_BTM_HLT2REFN(T) ((T)->btm.mbin->refn) +#else +#define ERTS_BTM_HLT2REFN(T) ((T)->btm.refn) +#endif + +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + #define ERTS_RBT_PREFIX btm -#define ERTS_RBT_T ErtsHLTimer +#define ERTS_RBT_T ErtsBifTimer #define ERTS_RBT_KEY_T Uint32 * #define ERTS_RBT_FLAGS_T UWord #define ERTS_RBT_INIT_EMPTY_TNODE(T) \ @@ -533,7 +570,7 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) (T)->btm.tree.parent |= (F); \ } while (0) #define ERTS_RBT_GET_PARENT(T) \ - ((ErtsHLTimer *) ((T)->btm.tree.parent & ~ERTS_HLT_PFLGS_MASK)) + ((ErtsBifTimer *) ((T)->btm.tree.parent & ~ERTS_HLT_PFLGS_MASK)) #define ERTS_RBT_SET_PARENT(T, P) \ do { \ ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0); \ @@ -544,20 +581,94 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) #define ERTS_RBT_SET_RIGHT(T, R) ((T)->btm.tree.right = (R)) #define ERTS_RBT_GET_LEFT(T) ((T)->btm.tree.left) #define ERTS_RBT_SET_LEFT(T, L) ((T)->btm.tree.left = (L)) -#define ERTS_RBT_GET_KEY(T) ((T)->btm.refn) +#define ERTS_RBT_GET_KEY(T) ERTS_BTM_HLT2REFN((T)) #define ERTS_RBT_IS_LT(KX, KY) refn_is_lt((KX), (KY)) -#define ERTS_RBT_IS_EQ(KX, KY) \ - (((KX)[0] == (KY)[0]) & ((KX)[1] == (KY)[1]) & ((KX)[2] == (KY)[2])) +#define ERTS_RBT_IS_EQ(KX, KY) refn_is_eq((KX), (KY)) #define ERTS_RBT_WANT_DELETE #define ERTS_RBT_WANT_INSERT +#ifndef ERTS_MAGIC_REF_BIF_TIMERS #define ERTS_RBT_WANT_LOOKUP +#endif #define ERTS_RBT_WANT_FOREACH #define ERTS_RBT_UNDEF #include "erl_rbtree.h" +#endif /* !ERTS_MAGIC_REF_BIF_TIMERS */ + +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + +static ERTS_INLINE void +proc_btm_list_insert(ErtsBifTimer **list, ErtsBifTimer *x) +{ + ErtsBifTimer *y = *list; + if (!y) { + x->btm.proc_list.next = x; + x->btm.proc_list.prev = x; + *list = x; + } + else { + ERTS_HLT_ASSERT(y->btm.proc_list.prev->btm.proc_list.next == y); + x->btm.proc_list.next = y; + x->btm.proc_list.prev = y->btm.proc_list.prev; + y->btm.proc_list.prev->btm.proc_list.next = x; + y->btm.proc_list.prev = x; + } +} + +static ERTS_INLINE void +proc_btm_list_delete(ErtsBifTimer **list, ErtsBifTimer *x) +{ + ErtsBifTimer *y = *list; + if (y == x && x->btm.proc_list.next == x) { + ERTS_HLT_ASSERT(x->btm.proc_list.prev == x); + *list = NULL; + } + else { + if (y == x) + *list = x->btm.proc_list.next; + ERTS_HLT_ASSERT(x->btm.proc_list.prev->btm.proc_list.next == x); + ERTS_HLT_ASSERT(x->btm.proc_list.next->btm.proc_list.prev == x); + x->btm.proc_list.prev->btm.proc_list.next = x->btm.proc_list.next; + x->btm.proc_list.next->btm.proc_list.prev = x->btm.proc_list.prev; + } + x->btm.proc_list.next = NULL; +} + +static ERTS_INLINE int +proc_btm_list_foreach_destroy_yielding(ErtsBifTimer **list, + void (*destroy)(ErtsBifTimer *, void *), + void *arg, + int limit) +{ + int i; + ErtsBifTimer *first, *last; + + first = *list; + if (!first) + return 0; + + last = first->btm.proc_list.prev; + for (i = 0; i < limit; i++) { + ErtsBifTimer *x = last; + last = last->btm.proc_list.prev; + (*destroy)(x, arg); + x->btm.proc_list.next = NULL; + if (x == first) { + *list = NULL; + return 0; + } + } + + last->btm.proc_list.next = first; + first->btm.proc_list.prev = last; + return 1; +} + +#else /* !ERTS_MAGIC_REF_BIF_TIMERS */ + #define ERTS_RBT_PREFIX proc_btm -#define ERTS_RBT_T ErtsHLTimer +#define ERTS_RBT_T ErtsBifTimer #define ERTS_RBT_KEY_T Uint32 * #define ERTS_RBT_FLAGS_T UWord #define ERTS_RBT_INIT_EMPTY_TNODE(T) \ @@ -583,7 +694,7 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) (T)->btm.proc_tree.parent |= (F); \ } while (0) #define ERTS_RBT_GET_PARENT(T) \ - ((ErtsHLTimer *) ((T)->btm.proc_tree.parent & ~ERTS_HLT_PFLGS_MASK)) + ((ErtsBifTimer *) ((T)->btm.proc_tree.parent & ~ERTS_HLT_PFLGS_MASK)) #define ERTS_RBT_SET_PARENT(T, P) \ do { \ ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0); \ @@ -594,71 +705,20 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) #define ERTS_RBT_SET_RIGHT(T, R) ((T)->btm.proc_tree.right = (R)) #define ERTS_RBT_GET_LEFT(T) ((T)->btm.proc_tree.left) #define ERTS_RBT_SET_LEFT(T, L) ((T)->btm.proc_tree.left = (L)) -#define ERTS_RBT_GET_KEY(T) ((T)->btm.refn) +#define ERTS_RBT_GET_KEY(T) ERTS_BTM_HLT2REFN((T)) #define ERTS_RBT_IS_LT(KX, KY) refn_is_lt((KX), (KY)) -#define ERTS_RBT_IS_EQ(KX, KY) \ - (((KX)[0] == (KY)[0]) & ((KX)[1] == (KY)[1]) & ((KX)[2] == (KY)[2])) -#define ERTS_RBT_WANT_DELETE -#define ERTS_RBT_WANT_INSERT -#define ERTS_RBT_WANT_LOOKUP -#define ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING -#define ERTS_RBT_UNDEF - -#include "erl_rbtree.h" - -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - -#define ERTS_RBT_PREFIX abtm -#define ERTS_RBT_T ErtsHLTimer -#define ERTS_RBT_KEY_T Uint32 * -#define ERTS_RBT_FLAGS_T UWord -#define ERTS_RBT_INIT_EMPTY_TNODE(T) \ - do { \ - (T)->abtm.tree.parent = (UWord) NULL; \ - (T)->abtm.tree.right = NULL; \ - (T)->abtm.tree.left = NULL; \ - } while (0) -#define ERTS_RBT_IS_RED(T) \ - ((int) ((T)->abtm.tree.parent & ERTS_HLT_PFLG_RED)) -#define ERTS_RBT_SET_RED(T) \ - ((T)->abtm.tree.parent |= ERTS_HLT_PFLG_RED) -#define ERTS_RBT_IS_BLACK(T) \ - (!ERTS_RBT_IS_RED((T))) -#define ERTS_RBT_SET_BLACK(T) \ - ((T)->abtm.tree.parent &= ~ERTS_HLT_PFLG_RED) -#define ERTS_RBT_GET_FLAGS(T) \ - ((T)->abtm.tree.parent & ERTS_HLT_PFLGS_MASK) -#define ERTS_RBT_SET_FLAGS(T, F) \ - do { \ - ERTS_HLT_ASSERT((((UWord) (F)) & ~ERTS_HLT_PFLGS_MASK) == 0); \ - (T)->abtm.tree.parent &= ~ERTS_HLT_PFLGS_MASK; \ - (T)->abtm.tree.parent |= (F); \ - } while (0) -#define ERTS_RBT_GET_PARENT(T) \ - ((ErtsHLTimer *) ((T)->abtm.tree.parent & ~ERTS_HLT_PFLGS_MASK)) -#define ERTS_RBT_SET_PARENT(T, P) \ - do { \ - ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0); \ - (T)->abtm.tree.parent &= ERTS_HLT_PFLGS_MASK; \ - (T)->abtm.tree.parent |= (UWord) (P); \ - } while (0) -#define ERTS_RBT_GET_RIGHT(T) ((T)->abtm.tree.right) -#define ERTS_RBT_SET_RIGHT(T, R) ((T)->abtm.tree.right = (R)) -#define ERTS_RBT_GET_LEFT(T) ((T)->abtm.tree.left) -#define ERTS_RBT_SET_LEFT(T, L) ((T)->abtm.tree.left = (L)) -#define ERTS_RBT_GET_KEY(T) ((T)->btm.refn) -#define ERTS_RBT_IS_LT(KX, KY) refn_is_lt((KX), (KY)) -#define ERTS_RBT_IS_EQ(KX, KY) \ - (((KX)[0] == (KY)[0]) & ((KX)[1] == (KY)[1]) & ((KX)[2] == (KY)[2])) +#define ERTS_RBT_IS_EQ(KX, KY) refn_is_eq((KX), (KY)) #define ERTS_RBT_WANT_DELETE #define ERTS_RBT_WANT_INSERT +#ifndef ERTS_MAGIC_REF_BIF_TIMERS #define ERTS_RBT_WANT_LOOKUP +#endif #define ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING #define ERTS_RBT_UNDEF #include "erl_rbtree.h" -#endif /* ERTS_BTM_ACCESSOR_SUPPORT */ +#endif /* !ERTS_MAGIC_REF_BIF_TIMERS */ #ifdef ERTS_SMP static void init_canceled_queue(ErtsHLTCncldTmrQ *cq); @@ -680,7 +740,9 @@ erts_create_timer_service(void) srv = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_TIMER_SERVICE, sizeof(ErtsHLTimerService)); srv->time_tree = NULL; +#ifndef ERTS_MAGIC_REF_BIF_TIMERS srv->btm_tree = NULL; +#endif srv->next_timeout = NULL; srv->yield = init_yield; erts_twheel_init_timer(&srv->service_timer); @@ -697,11 +759,8 @@ erts_timer_type_size(ErtsAlcType_t type) { switch (type) { case ERTS_ALC_T_LL_PTIMER: return sizeof(ErtsTWTimer); - case ERTS_ALC_T_HL_PTIMER: return ERTS_HL_PTIMER_SIZE; - case ERTS_ALC_T_BIF_TIMER: return ERTS_BIF_TIMER_SIZE; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - case ERTS_ALC_T_ABIF_TIMER: return ERTS_ABIF_TIMER_SIZE; -#endif + case ERTS_ALC_T_HL_PTIMER: return sizeof(ErtsHLTimer); + case ERTS_ALC_T_BIF_TIMER: return sizeof(ErtsBifTimer); default: ERTS_INTERNAL_ERROR("Unknown type"); } return 0; @@ -760,6 +819,111 @@ port_timeout_common(Port *port, void *tmr) return 0; } +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + +static erts_smp_atomic_t * +mbin_to_btmref__(ErtsMagicBinary *mbin) +{ + return erts_smp_binary_to_magic_indirection((Binary *) mbin); +} + +static ERTS_INLINE void +magic_binary_init(ErtsMagicBinary *mbin, ErtsBifTimer *tmr) +{ + erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin); + erts_smp_atomic_init_nob(aptr, (erts_aint_t) tmr); +} + +static ERTS_INLINE ErtsBifTimer * +magic_binary_to_btm(ErtsMagicBinary *mbin) +{ + erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin); + ErtsBifTimer *tmr = (ErtsBifTimer *) erts_smp_atomic_read_nob(aptr); + ERTS_HLT_ASSERT(!tmr || tmr->btm.mbin == mbin); + return tmr; +} + +#endif /* ERTS_MAGIC_REF_BIF_TIMERS */ + +static ERTS_INLINE erts_aint_t +init_btm_specifics(ErtsSchedulerData *esdp, + ErtsBifTimer *tmr, Eterm msg, +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsMagicBinary *mbin +#else + Uint32 *refn +#endif + ) +{ + Uint hsz = is_immed(msg) ? ((Uint) 0) : size_object(msg); + int refc; + if (!hsz) { + tmr->btm.message = msg; + tmr->btm.bp = NULL; + } + else { + ErlHeapFragment *bp = new_message_buffer(hsz); + Eterm *hp = bp->mem; + tmr->btm.message = copy_struct(msg, hsz, &hp, &bp->off_heap); + tmr->btm.bp = bp; + } +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + refc = 1; + tmr->btm.mbin = mbin; + erts_refc_inc(&mbin->refc, 1); + magic_binary_init(mbin, tmr); + tmr->btm.proc_list.next = NULL; +#else + refc = 0; + tmr->btm.refn[0] = refn[0]; + tmr->btm.refn[1] = refn[1]; + tmr->btm.refn[2] = refn[2]; + + tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + + btm_rbt_insert(&esdp->timer_service->btm_tree, tmr); +#endif + + erts_smp_atomic32_init_nob(&tmr->btm.state, ERTS_TMR_STATE_ACTIVE); + return refc; /* refc from magic binary... */ +} + +static void tw_bif_timer_timeout(void *vbtmp); + +static ERTS_INLINE void +timer_destroy(ErtsTimer *tmr, int twt, int btm) +{ + if (!btm) { + if (twt) + tw_timer_free(&tmr->twt); + else + erts_free(ERTS_ALC_T_HL_PTIMER, tmr); + } + else { +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + Binary *bp = (Binary *) tmr->btm.btm.mbin; + if (erts_refc_dectest(&bp->refc, 0) == 0) + erts_bin_free(bp); +#endif + if (tmr->head.roflgs & ERTS_TMR_ROFLG_PRE_ALC) + bif_timer_pre_free(&tmr->btm); + else + erts_free(ERTS_ALC_T_BIF_TIMER, &tmr->btm); + } +} + +static ERTS_INLINE void +timer_pre_dec_refc(ErtsTimer *tmr) +{ +#ifdef ERTS_HLT_DEBUG + erts_aint_t refc; + refc = erts_smp_atomic32_dec_read_nob(&tmr->head.refc); + ERTS_HLT_ASSERT(refc > 0); +#else + erts_smp_atomic32_dec_nob(&tmr->head.refc); +#endif +} + /* * Basic timer wheel timer stuff */ @@ -767,26 +931,39 @@ port_timeout_common(Port *port, void *tmr) static void scheduled_tw_timer_destroy(void *vtmr) { - tw_timer_free((ErtsTWTimer *) vtmr); + ErtsTimer * tmr = (ErtsTimer *) vtmr; + int btm = !!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR); + timer_destroy((ErtsTimer *) vtmr, 1, btm); } static void schedule_tw_timer_destroy(ErtsTWTimer *tmr) { + Uint size; /* * Reference to process/port can be * dropped at once... */ if (tmr->head.roflgs & ERTS_TMR_ROFLG_PROC) - erts_proc_dec_refc((Process *) tmr->u.p); + erts_proc_dec_refc(tmr->head.receiver.proc); else if (tmr->head.roflgs & ERTS_TMR_ROFLG_PORT) - erts_port_dec_refc((Port *) tmr->u.p); + erts_port_dec_refc(tmr->head.receiver.port); + + if (!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) + size = sizeof(ErtsHLTimer); + else { + /* Message buffer already dropped... */ + size = sizeof(ErtsBifTimer); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + size += sizeof(ErtsMagicIndirectionWord); +#endif + } erts_schedule_thr_prgr_later_cleanup_op( scheduled_tw_timer_destroy, (void *) tmr, - &tmr->tw_tmr.u.cleanup, - sizeof(ErtsTWTimer)); + &tmr->u.cleanup, + size); } static ERTS_INLINE void @@ -802,7 +979,7 @@ static void tw_proc_timeout(void *vtwtp) { ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; - Process *proc = (Process *) twtp->u.p; + Process *proc = twtp->head.receiver.proc; if (proc_timeout_common(proc, vtwtp)) tw_timer_dec_refc(twtp); tw_timer_dec_refc(twtp); @@ -812,84 +989,126 @@ static void tw_port_timeout(void *vtwtp) { ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; - Port *port = (Port *) twtp->u.p; + Port *port = twtp->head.receiver.port; if (port_timeout_common(port, vtwtp)) tw_timer_dec_refc(twtp); tw_timer_dec_refc(twtp); } static void -tw_ptimer_cancel(void *vtwtp) -{ - tw_timer_dec_refc((ErtsTWTimer *) vtwtp); -} - -static void cancel_tw_timer(ErtsSchedulerData *esdp, ErtsTWTimer *tmr) { ERTS_HLT_ASSERT((tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK) == (Uint32) esdp->no); - erts_twheel_cancel_timer(esdp->timer_wheel, &tmr->tw_tmr); + erts_twheel_cancel_timer(esdp->timer_wheel, &tmr->u.tw_tmr); + tw_timer_dec_refc(tmr); } static void tw_callback_timeout(void *vtwtp) { ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; - void (*callback)(void *) = twtp->u.callback; + void (*callback)(void *) = twtp->head.receiver.callback; void *arg = twtp->head.u.arg; tw_timer_dec_refc(twtp); (*callback)(arg); } -static ErtsTWTimer * -create_tw_timer(ErtsSchedulerData *esdp, - ErtsTmrType type, void *p, - void (*callback)(void *), void *arg, - ErtsMonotonicTime timeout_pos) +static ErtsTimer * +create_tw_timer(ErtsSchedulerData *esdp, + ErtsMonotonicTime timeout_pos, + int short_time, ErtsTmrType type, + void *rcvrp, Eterm rcvr, + Eterm msg, +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsMagicBinary *mbin, +#else + Uint32 *refn, +#endif + void (*callback)(void *), void *arg) { ErtsTWTimer *tmr; void (*timeout_func)(void *); - void (*cancel_func)(void *); erts_aint32_t refc; - tmr = tw_timer_alloc(); - erts_twheel_init_timer(&tmr->tw_tmr); - - tmr->head.roflgs = (Uint32) esdp->no; - ERTS_HLT_ASSERT((tmr->head.roflgs - & ~ERTS_TMR_ROFLG_SID_MASK) == 0); + if (type != ERTS_TMR_BIF) { + tmr = tw_timer_alloc(); + tmr->head.roflgs = 0; + } + else { + if (short_time) { + tmr = (ErtsTWTimer *) bif_timer_pre_alloc(); + if (!tmr) + goto alloc_bif_timer; + tmr->head.roflgs = (ERTS_TMR_ROFLG_BIF_TMR + | ERTS_TMR_ROFLG_PRE_ALC); + } + else { + alloc_bif_timer: + tmr = (ErtsTWTimer *) erts_alloc(ERTS_ALC_T_BIF_TIMER, + sizeof(ErtsBifTimer)); + tmr->head.roflgs = ERTS_TMR_ROFLG_BIF_TMR; + } + } + + erts_twheel_init_timer(&tmr->u.tw_tmr); + tmr->head.roflgs |= (Uint32) esdp->no; + ERTS_HLT_ASSERT((((Uint32) esdp->no) + & ~ERTS_TMR_ROFLG_SID_MASK) == 0); switch (type) { case ERTS_TMR_PROC: - tmr->u.p = p; + tmr->head.receiver.proc = (Process *) rcvrp; tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC; timeout_func = tw_proc_timeout; - cancel_func = tw_ptimer_cancel; - erts_proc_inc_refc((Process *) p); + erts_proc_inc_refc((Process *) rcvrp); refc = 2; break; case ERTS_TMR_PORT: - tmr->u.p = p; + tmr->head.receiver.port = (Port *) rcvrp; tmr->head.roflgs |= ERTS_TMR_ROFLG_PORT; timeout_func = tw_port_timeout; - cancel_func = tw_ptimer_cancel; - erts_port_inc_refc((Port *) p); + erts_port_inc_refc((Port *) rcvrp); refc = 2; break; case ERTS_TMR_CALLBACK: tmr->head.u.arg = arg; - tmr->u.callback = callback; + tmr->head.receiver.callback = callback; tmr->head.roflgs |= ERTS_TMR_ROFLG_CALLBACK; timeout_func = tw_callback_timeout; - cancel_func = NULL; refc = 1; break; + case ERTS_TMR_BIF: + + timeout_func = tw_bif_timer_timeout; + if (is_internal_pid(rcvr)) { + tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC; + tmr->head.receiver.proc = (Process *) rcvrp; + refc = 2; + } + else { + ERTS_HLT_ASSERT(is_atom(rcvr)); + tmr->head.roflgs |= ERTS_TMR_ROFLG_REG_NAME; + tmr->head.receiver.name = (Eterm) rcvr; + refc = 1; + } + + refc += init_btm_specifics(esdp, + (ErtsBifTimer *) tmr, + msg, +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + mbin +#else + refn +#endif + ); + break; + default: ERTS_INTERNAL_ERROR("Unsupported timer type"); return NULL; @@ -898,41 +1117,24 @@ create_tw_timer(ErtsSchedulerData *esdp, erts_smp_atomic32_init_nob(&tmr->head.refc, refc); erts_twheel_set_timer(esdp->timer_wheel, - &tmr->tw_tmr, + &tmr->u.tw_tmr, timeout_func, - cancel_func, tmr, timeout_pos); - return tmr; + return (ErtsTimer *) tmr; } /* * Basic high level timer stuff */ -static ERTS_INLINE void -hl_timer_destroy(ErtsHLTimer *tmr) -{ - Uint32 roflgs = tmr->head.roflgs; - if (!(roflgs & ERTS_TMR_ROFLG_BIF_TMR)) - erts_free(ERTS_ALC_T_HL_PTIMER, tmr); - else { - if (roflgs & ERTS_TMR_ROFLG_PRE_ALC) - bif_timer_pre_free(tmr); -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - else if (roflgs & ERTS_TMR_ROFLG_ABIF_TMR) - erts_free(ERTS_ALC_T_ABIF_TIMER, tmr); -#endif - else - erts_free(ERTS_ALC_T_BIF_TIMER, tmr); - } -} - static void scheduled_hl_timer_destroy(void *vtmr) { - hl_timer_destroy((ErtsHLTimer *) vtmr); + ErtsTimer * tmr = (ErtsTimer *) vtmr; + int btm = !!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR); + timer_destroy((ErtsTimer *) vtmr, 0, btm); } static void @@ -948,25 +1150,25 @@ schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs) ERTS_HLT_ASSERT(erts_smp_atomic32_read_nob(&tmr->head.refc) == 0); if (roflgs & ERTS_TMR_ROFLG_REG_NAME) { - ERTS_HLT_ASSERT(is_atom(tmr->receiver.name)); + ERTS_HLT_ASSERT(is_atom(tmr->head.receiver.name)); } else if (roflgs & ERTS_TMR_ROFLG_PROC) { - ERTS_HLT_ASSERT(tmr->receiver.proc); - erts_proc_dec_refc(tmr->receiver.proc); + ERTS_HLT_ASSERT(tmr->head.receiver.proc); + erts_proc_dec_refc(tmr->head.receiver.proc); } else if (roflgs & ERTS_TMR_ROFLG_PORT) { - ERTS_HLT_ASSERT(tmr->receiver.port); - erts_port_dec_refc(tmr->receiver.port); + ERTS_HLT_ASSERT(tmr->head.receiver.port); + erts_port_dec_refc(tmr->head.receiver.port); } if (!(roflgs & ERTS_TMR_ROFLG_BIF_TMR)) - size = ERTS_HL_PTIMER_SIZE; - else { - /* - * Message buffer can be dropped at - * once... - */ size = sizeof(ErtsHLTimer); + else { + /* Message buffer already dropped... */ + size = sizeof(ErtsBifTimer); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + size += sizeof(ErtsMagicIndirectionWord); +#endif } erts_schedule_thr_prgr_later_cleanup_op( @@ -975,18 +1177,6 @@ schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs) } static ERTS_INLINE void -hl_timer_pre_dec_refc(ErtsHLTimer *tmr) -{ -#ifdef ERTS_HLT_DEBUG - erts_aint_t refc; - refc = erts_smp_atomic32_dec_read_nob(&tmr->head.refc); - ERTS_HLT_ASSERT(refc > 0); -#else - erts_smp_atomic32_dec_nob(&tmr->head.refc); -#endif -} - -static ERTS_INLINE void hl_timer_dec_refc(ErtsHLTimer *tmr, Uint32 roflgs) { if (erts_smp_atomic32_dec_read_relb(&tmr->head.refc) == 0) { @@ -1018,39 +1208,135 @@ check_canceled_queue(ErtsSchedulerData *esdp, ErtsHLTimerService *srv) #endif } -#ifdef ERTS_BTM_ACCESSOR_SUPPORT +#ifdef ERTS_MAGIC_REF_BIF_TIMERS -static void -hlt_delete_abtm(ErtsHLTimer *tmr) +static int +bif_timer_ref_destructor(Binary *unused) { - Process *proc; + return 1; +} - ERTS_HLT_ASSERT(tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR); +static ERTS_INLINE void +btm_clear_magic_binary(ErtsBifTimer *tmr) +{ + erts_smp_atomic_t *aptr = mbin_to_btmref__(tmr->btm.mbin); + Uint32 roflgs = tmr->type.head.roflgs; +#ifdef ERTS_HLT_DEBUG + erts_aint_t tval = erts_smp_atomic_xchg_nob(aptr, + (erts_aint_t) NULL); + ERTS_HLT_ASSERT(tval == (erts_aint_t) tmr); +#else + erts_smp_atomic_set_nob(aptr, (erts_aint_t) NULL); +#endif + if (roflgs & ERTS_TMR_ROFLG_HLT) + hl_timer_dec_refc(&tmr->type.hlt, roflgs); + else + tw_timer_dec_refc(&tmr->type.twt); +} - proc = erts_proc_lookup(tmr->abtm.accessor); +#endif /* ERTS_MAGIC_REF_BIF_TIMERS */ - if (proc) { - int deref = 0; - erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM); - if (tmr->abtm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { - abtm_rbt_delete(&proc->accessor_bif_timers, tmr); - deref = 1; - tmr->abtm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - } - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); - if (deref) - hl_timer_pre_dec_refc(tmr); +static ERTS_INLINE void +bif_timer_timeout(ErtsHLTimerService *srv, + ErtsBifTimer *tmr, + Uint32 roflgs) +{ + erts_aint32_t state; + + ERTS_HLT_ASSERT(tmr->type.head.roflgs == roflgs); + ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR); + + state = erts_smp_atomic32_cmpxchg_acqb(&tmr->btm.state, + ERTS_TMR_STATE_TIMED_OUT, + ERTS_TMR_STATE_ACTIVE); + + ERTS_HLT_ASSERT(state == ERTS_TMR_STATE_CANCELED + || state == ERTS_TMR_STATE_ACTIVE); + + if (state == ERTS_TMR_STATE_ACTIVE) { + Process *proc; + +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + btm_clear_magic_binary(tmr); +#endif + + if (roflgs & ERTS_TMR_ROFLG_REG_NAME) { + Eterm term; + term = tmr->type.head.receiver.name; + ERTS_HLT_ASSERT(is_atom(term)); + term = erts_whereis_name_to_id(NULL, term); + proc = erts_proc_lookup(term); + } + else { + ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_PROC); + proc = tmr->type.head.receiver.proc; + ERTS_HLT_ASSERT(proc); + } + if (proc) { + if (!ERTS_PROC_IS_EXITING(proc)) { + int dec_refc = 0; + ErtsMessage *mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = tmr->btm.bp; + tmr->btm.bp = NULL; + erts_queue_message(proc, 0, mp, tmr->btm.message, + am_clock_service); + erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + if (tmr->btm.proc_list.next) { + proc_btm_list_delete(&proc->bif_timers, tmr); + dec_refc = 1; + } +#else + if (tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + proc_btm_rbt_delete(&proc->bif_timers, tmr); + tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + dec_refc = 1; + } +#endif + erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); + if (dec_refc) + timer_pre_dec_refc((ErtsTimer *) tmr); + } + } + if (tmr->btm.bp) + free_message_buffer(tmr->btm.bp); } + +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + btm_rbt_delete(&srv->btm_tree, tmr); + tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + } +#endif + + } +static void +tw_bif_timer_timeout(void *vbtmp) +{ +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsHLTimerService *srv = NULL; +#else + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsHLTimerService *srv = esdp->timer_service; #endif + ErtsBifTimer *btmp = (ErtsBifTimer *) vbtmp; + bif_timer_timeout(srv, btmp, btmp->type.head.roflgs); + tw_timer_dec_refc(&btmp->type.twt); +} -static ErtsHLTimer * +static ErtsTimer * create_hl_timer(ErtsSchedulerData *esdp, ErtsMonotonicTime timeout_pos, int short_time, ErtsTmrType type, - void *rcvrp, Eterm rcvr, Eterm acsr, - Eterm msg, Uint32 *refn, + void *rcvrp, Eterm rcvr, + Eterm msg, +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsMagicBinary *mbin, +#else + Uint32 *refn, +#endif void (*callback)(void *), void *arg) { ErtsHLTimerService *srv = esdp->timer_service; @@ -1069,7 +1355,7 @@ create_hl_timer(ErtsSchedulerData *esdp, if (type != ERTS_TMR_BIF) { tmr = erts_alloc(ERTS_ALC_T_HL_PTIMER, - ERTS_HL_PTIMER_SIZE); + sizeof(ErtsHLTimer)); tmr->timeout = timeout_pos; switch (type) { @@ -1078,7 +1364,7 @@ create_hl_timer(ErtsSchedulerData *esdp, ERTS_HLT_ASSERT(is_internal_pid(rcvr)); erts_proc_inc_refc((Process *) rcvrp); - tmr->receiver.proc = (Process *) rcvrp; + tmr->head.receiver.proc = (Process *) rcvrp; roflgs |= ERTS_TMR_ROFLG_PROC; refc = 2; break; @@ -1086,14 +1372,14 @@ create_hl_timer(ErtsSchedulerData *esdp, case ERTS_TMR_PORT: ERTS_HLT_ASSERT(is_internal_port(rcvr)); erts_port_inc_refc((Port *) rcvrp); - tmr->receiver.port = (Port *) rcvrp; + tmr->head.receiver.port = (Port *) rcvrp; roflgs |= ERTS_TMR_ROFLG_PORT; refc = 2; break; case ERTS_TMR_CALLBACK: roflgs |= ERTS_TMR_ROFLG_CALLBACK; - tmr->receiver.callback = callback; + tmr->head.receiver.callback = callback; tmr->head.u.arg = arg; refc = 1; break; @@ -1105,84 +1391,47 @@ create_hl_timer(ErtsSchedulerData *esdp, } else { /* ERTS_TMR_BIF */ - Uint hsz; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - int is_abif_tmr = is_value(acsr) && acsr != rcvr; -#endif if (short_time) { - tmr = bif_timer_pre_alloc(); + tmr = (ErtsHLTimer *) bif_timer_pre_alloc(); if (!tmr) goto alloc_bif_timer; roflgs |= ERTS_TMR_ROFLG_PRE_ALC; } else { alloc_bif_timer: -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - if (is_abif_tmr) - tmr = erts_alloc(ERTS_ALC_T_ABIF_TIMER, - ERTS_ABIF_TIMER_SIZE); - else -#endif - tmr = erts_alloc(ERTS_ALC_T_BIF_TIMER, - ERTS_BIF_TIMER_SIZE); - } + tmr = (ErtsHLTimer *) erts_alloc(ERTS_ALC_T_BIF_TIMER, + sizeof(ErtsBifTimer)); + } tmr->timeout = timeout_pos; roflgs |= ERTS_TMR_ROFLG_BIF_TMR; if (is_internal_pid(rcvr)) { roflgs |= ERTS_TMR_ROFLG_PROC; - tmr->receiver.proc = (Process *) rcvrp; + tmr->head.receiver.proc = (Process *) rcvrp; refc = 2; } else { ERTS_HLT_ASSERT(is_atom(rcvr)); roflgs |= ERTS_TMR_ROFLG_REG_NAME; - tmr->receiver.name = rcvr; + tmr->head.receiver.name = rcvr; refc = 1; } - hsz = is_immed(msg) ? ((Uint) 0) : size_object(msg); - if (!hsz) { - tmr->btm.message = msg; - tmr->btm.bp = NULL; - } - else { - ErlHeapFragment *bp = new_message_buffer(hsz); - Eterm *hp = bp->mem; - tmr->btm.message = copy_struct(msg, hsz, &hp, &bp->off_heap); - tmr->btm.bp = bp; - } - tmr->btm.refn[0] = refn[0]; - tmr->btm.refn[1] = refn[1]; - tmr->btm.refn[2] = refn[2]; - - tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - if (is_abif_tmr) { - Process *aproc; - roflgs |= ERTS_TMR_ROFLG_ABIF_TMR; - tmr->abtm.accessor = acsr; - aproc = erts_proc_lookup(acsr); - if (!aproc) - tmr->abtm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - else { - refc++; - erts_smp_proc_lock(aproc, ERTS_PROC_LOCK_BTM); - abtm_rbt_insert(&aproc->accessor_bif_timers, tmr); - erts_smp_proc_unlock(aproc, ERTS_PROC_LOCK_BTM); - } - } + refc += init_btm_specifics(esdp, + (ErtsBifTimer *) tmr, + msg, +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + mbin +#else + refn #endif - - btm_rbt_insert(&srv->btm_tree, tmr); + ); } tmr->head.roflgs = roflgs; erts_smp_atomic32_init_nob(&tmr->head.refc, refc); - erts_smp_atomic32_init_nob(&tmr->state, ERTS_TMR_STATE_ACTIVE); if (!srv->next_timeout || tmr->timeout < srv->next_timeout->timeout) { @@ -1192,7 +1441,6 @@ create_hl_timer(ErtsSchedulerData *esdp, erts_twheel_set_timer(esdp->timer_wheel, &srv->service_timer, hlt_service_timeout, - NULL, (void *) esdp, tmr->timeout); srv->next_timeout = tmr; @@ -1209,79 +1457,20 @@ create_hl_timer(ErtsSchedulerData *esdp, ERTS_HLT_HDBG_CHK_SRV(srv); - return tmr; -} - -static ERTS_INLINE void -hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs) -{ - ErtsProcLocks proc_locks = ERTS_PROC_LOCKS_MSG_SEND; - Process *proc; - int queued_message = 0; - int dec_refc = 0; - Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME); - ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR); - -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - if (tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR) - hlt_delete_abtm(tmr); -#endif - - if (is_reg_name) { - Eterm pid; - ERTS_HLT_ASSERT(is_atom(tmr->receiver.name)); - pid = erts_whereis_name_to_id(NULL, tmr->receiver.name); - proc = erts_proc_lookup(pid); - } - else { - ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_PROC); - ERTS_HLT_ASSERT(tmr->receiver.proc); - - proc = tmr->receiver.proc; - proc_locks |= ERTS_PROC_LOCK_BTM; - } - if (proc) { - erts_smp_proc_lock(proc, proc_locks); - /* - * If process is exiting, let it clean up - * the btm tree by itself (it may be in - * the middle of tree destruction). - */ - if (!ERTS_PROC_IS_EXITING(proc)) { - ErtsMessage *mp = erts_alloc_message(0, NULL); - mp->data.heap_frag = tmr->btm.bp; - erts_queue_message(proc, proc_locks, mp, - tmr->btm.message, am_clock_service); - erts_smp_proc_unlock(proc, ERTS_PROC_LOCKS_MSG_SEND); - queued_message = 1; - proc_locks &= ~ERTS_PROC_LOCKS_MSG_SEND; - tmr->btm.bp = NULL; - if (tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { - proc_btm_rbt_delete(&proc->bif_timers, tmr); - tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - dec_refc = 1; - } - } - if (proc_locks) - erts_smp_proc_unlock(proc, proc_locks); - if (dec_refc) - hl_timer_pre_dec_refc(tmr); - } - if (!queued_message && tmr->btm.bp) - free_message_buffer(tmr->btm.bp); + return (ErtsTimer *) tmr; } static ERTS_INLINE void hlt_proc_timeout(ErtsHLTimer *tmr) { - if (proc_timeout_common(tmr->receiver.proc, (void *) tmr)) + if (proc_timeout_common(tmr->head.receiver.proc, (void *) tmr)) hl_timer_dec_refc(tmr, tmr->head.roflgs); } static ERTS_INLINE void hlt_port_timeout(ErtsHLTimer *tmr) { - if (port_timeout_common(tmr->receiver.port, (void *) tmr)) + if (port_timeout_common(tmr->head.receiver.port, (void *) tmr)) hl_timer_dec_refc(tmr, tmr->head.roflgs); } @@ -1289,41 +1478,24 @@ static void hlt_timeout(ErtsHLTimer *tmr, void *vsrv) { ErtsHLTimerService *srv = (ErtsHLTimerService *) vsrv; Uint32 roflgs; - erts_aint32_t state; ERTS_HLT_HDBG_CHK_SRV(srv); roflgs = tmr->head.roflgs; ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_HLT); - state = erts_smp_atomic32_cmpxchg_acqb(&tmr->state, - ERTS_TMR_STATE_TIMED_OUT, - ERTS_TMR_STATE_ACTIVE); - - ERTS_HLT_ASSERT(state == ERTS_TMR_STATE_CANCELED - || state == ERTS_TMR_STATE_ACTIVE); - - if (state == ERTS_TMR_STATE_ACTIVE) { - - if (roflgs & ERTS_TMR_ROFLG_BIF_TMR) - hlt_bif_timer_timeout(tmr, roflgs); - else if (roflgs & ERTS_TMR_ROFLG_PROC) - hlt_proc_timeout(tmr); - else if (roflgs & ERTS_TMR_ROFLG_PORT) - hlt_port_timeout(tmr); - else { - ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_CALLBACK); - (*tmr->receiver.callback)(tmr->head.u.arg); - } - + if (roflgs & ERTS_TMR_ROFLG_BIF_TMR) + bif_timer_timeout(srv, (ErtsBifTimer *) tmr, roflgs); + else if (roflgs & ERTS_TMR_ROFLG_PROC) + hlt_proc_timeout(tmr); + else if (roflgs & ERTS_TMR_ROFLG_PORT) + hlt_port_timeout(tmr); + else { + ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_CALLBACK); + (*tmr->head.receiver.callback)(tmr->head.u.arg); } tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - if ((roflgs & ERTS_TMR_ROFLG_BIF_TMR) - && tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { - btm_rbt_delete(&srv->btm_tree, tmr); - tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - } ERTS_HLT_HDBG_CHK_SRV(srv); @@ -1390,7 +1562,6 @@ hlt_service_timeout(void *vesdp) erts_twheel_set_timer(esdp->timer_wheel, &srv->service_timer, hlt_service_timeout, - NULL, vesdp, tmr->timeout); } @@ -1402,19 +1573,6 @@ hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr) ERTS_HLT_HDBG_CHK_SRV(srv); - if (tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR) { - - if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { - btm_rbt_delete(&srv->btm_tree, tmr); - tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - } - -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - if (tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR) - hlt_delete_abtm(tmr); -#endif - } - if (tmr->time.tree.parent == ERTS_HLT_PFIELD_NOT_IN_TABLE) { /* Already removed... */ ERTS_HLT_HDBG_CHK_SRV(srv); @@ -1460,7 +1618,6 @@ hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr) erts_twheel_set_timer(esdp->timer_wheel, &srv->service_timer, hlt_service_timeout, - NULL, (void *) esdp, smlst->timeout); } @@ -1485,6 +1642,17 @@ cleanup_sched_local_canceled_timer(ErtsSchedulerData *esdp, ERTS_HLT_ASSERT(esdp == erts_get_scheduler_data()); ERTS_HLT_ASSERT((tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK) == (Uint32) esdp->no); + +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + if (roflgs & ERTS_TMR_ROFLG_BIF_TMR) { + ErtsBifTimer *btm = (ErtsBifTimer *) tmr; + if (btm->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + btm_rbt_delete(&esdp->timer_service->btm_tree, btm); + btm->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + } + } +#endif + if (roflgs & ERTS_TMR_ROFLG_HLT) { hlt_delete_timer(esdp, &tmr->hlt); hl_timer_dec_refc(&tmr->hlt, roflgs); @@ -1750,57 +1918,86 @@ continue_cancel_ptimer(ErtsSchedulerData *esdp, ErtsTimer *tmr) * BIF timer specific */ + Uint erts_bif_timer_memory_size(void) { return (Uint) 0; } static BIF_RETTYPE -setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos, - int short_time, Eterm rcvr, Eterm acsr, - Eterm msg, int wrap) +setup_bif_timer(Process *c_p, int twheel, ErtsMonotonicTime timeout_pos, + int short_time, Eterm rcvr, Eterm msg, int wrap) { BIF_RETTYPE ret; Eterm ref, tmo_msg, *hp; - ErtsHLTimer *tmr; + ErtsBifTimer *tmr; ErtsSchedulerData *esdp; - DeclareTmpHeap(tmp_hp, 4, c_p); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + Binary *mbin; +#endif + Eterm tmp_hp[4]; + ErtsCreateTimerFunc create_timer; if (is_not_internal_pid(rcvr) && is_not_atom(rcvr)) goto badarg; esdp = erts_proc_sched_data(c_p); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + mbin = erts_create_magic_indirection(bif_timer_ref_destructor); + hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE); + ref = erts_mk_magic_ref(&hp, &c_p->off_heap, mbin); + ASSERT(erts_get_ref_numbers_thr_id(((ErtsMagicBinary *)mbin)->refn) + == (Uint32) esdp->no); +#else hp = HAlloc(c_p, ERTS_REF_THING_SIZE); ref = erts_sched_make_ref_in_buffer(esdp, hp); - - ASSERT(erts_get_ref_numbers_thr_id( - internal_ref_numbers(ref)) == (Uint32) esdp->no); - - UseTmpHeap(4, c_p); + ASSERT(erts_get_ref_numbers_thr_id(internal_ordinary_ref_numbers(ref)) + == (Uint32) esdp->no); +#endif tmo_msg = wrap ? TUPLE3(tmp_hp, am_timeout, ref, msg) : msg; - tmr = create_hl_timer(esdp, timeout_pos, short_time, - ERTS_TMR_BIF, NULL, rcvr, acsr, tmo_msg, - internal_ref_numbers(ref), NULL, NULL); - - UnUseTmpHeap(4, c_p); + create_timer = twheel ? create_tw_timer : create_hl_timer; + tmr = (ErtsBifTimer *) create_timer(esdp, timeout_pos, + short_time, ERTS_TMR_BIF, + NULL, rcvr, tmo_msg, +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + (ErtsMagicBinary *) mbin, +#else + internal_ordinary_ref_numbers(ref), +#endif + NULL, NULL); if (is_internal_pid(rcvr)) { Process *proc = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, rcvr, ERTS_PROC_LOCK_BTM, ERTS_P2P_FLG_INC_REFC); if (!proc) { +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + btm_clear_magic_binary(tmr); +#else + if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + btm_rbt_delete(&esdp->timer_service->btm_tree, tmr); + tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + } +#endif if (tmr->btm.bp) free_message_buffer(tmr->btm.bp); - hlt_delete_timer(esdp, tmr); - hl_timer_destroy(tmr); + if (twheel) + cancel_tw_timer(esdp, &tmr->type.twt); + else + hlt_delete_timer(esdp, &tmr->type.hlt); + timer_destroy((ErtsTimer *) tmr, twheel, 1); } else { +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + proc_btm_list_insert(&proc->bif_timers, tmr); +#else proc_btm_rbt_insert(&proc->bif_timers, tmr); +#endif erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); - tmr->receiver.proc = proc; + tmr->type.head.receiver.proc = proc; } } @@ -1814,27 +2011,33 @@ badarg: } static int -cancel_bif_timer(ErtsHLTimer *tmr) +cancel_bif_timer(ErtsBifTimer *tmr) { erts_aint_t state; Uint32 roflgs; int res; - state = erts_smp_atomic32_cmpxchg_acqb(&tmr->state, + state = erts_smp_atomic32_cmpxchg_acqb(&tmr->btm.state, ERTS_TMR_STATE_CANCELED, ERTS_TMR_STATE_ACTIVE); if (state != ERTS_TMR_STATE_ACTIVE) return 0; +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + btm_clear_magic_binary(tmr); +#endif + if (tmr->btm.bp) free_message_buffer(tmr->btm.bp); res = -1; - roflgs = tmr->head.roflgs; + roflgs = tmr->type.head.roflgs; if (roflgs & ERTS_TMR_ROFLG_PROC) { - Process *proc = tmr->receiver.proc; - ERTS_HLT_ASSERT(!(tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME)); + Process *proc; + + proc = tmr->type.head.receiver.proc; + ERTS_HLT_ASSERT(!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME)); erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM); /* @@ -1842,29 +2045,238 @@ cancel_bif_timer(ErtsHLTimer *tmr) * the btm tree by itself (it may be in * the middle of tree destruction). */ +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + if (!ERTS_PROC_IS_EXITING(proc) && tmr->btm.proc_list.next) { + proc_btm_list_delete(&proc->bif_timers, tmr); + res = 1; + } +#else if (!ERTS_PROC_IS_EXITING(proc) && tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { proc_btm_rbt_delete(&proc->bif_timers, tmr); tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; res = 1; } +#endif erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); } return res; } +static ERTS_INLINE Sint64 +access_btm(ErtsBifTimer *tmr, Uint32 sid, ErtsSchedulerData *esdp, int cancel) +{ + int cncl_res; + Sint64 time_left; + ErtsMonotonicTime timeout; + int is_hlt; + + if (!tmr) + return -1; + + is_hlt = !!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT); + timeout = (is_hlt + ? tmr->type.hlt.timeout + : erts_tweel_read_timeout(&tmr->type.twt.u.tw_tmr)); + + if (!cancel) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->btm.state); + if (state == ERTS_TMR_STATE_ACTIVE) + return get_time_left(esdp, timeout); + return -1; + } + + cncl_res = cancel_bif_timer(tmr); + if (!cncl_res) + return -1; + + time_left = get_time_left(esdp, timeout); + + if (sid != (Uint32) esdp->no) { + if (cncl_res > 0) + queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr); + } + else { +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + btm_rbt_delete(&esdp->timer_service->btm_tree, tmr); + tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + } +#endif + if (is_hlt) { + if (cncl_res > 0) + hl_timer_dec_refc(&tmr->type.hlt, tmr->type.hlt.head.roflgs); + hlt_delete_timer(esdp, &tmr->type.hlt); + } + else { + if (cncl_res > 0) + tw_timer_dec_refc(&tmr->type.twt); + cancel_tw_timer(esdp, &tmr->type.twt); + } + } + + return time_left; +} + +static ERTS_INLINE Eterm +return_info(Process *c_p, Sint64 time_left) +{ + Uint hsz; + Eterm *hp; + + if (time_left < 0) + return am_false; + + if (time_left <= (Sint64) MAX_SMALL) + return make_small((Sint) time_left); + + hsz = ERTS_SINT64_HEAP_SIZE(time_left); + hp = HAlloc(c_p, hsz); + return erts_sint64_to_big(time_left, &hp); +} + +static ERTS_INLINE Eterm +send_async_info(Process *proc, ErtsProcLocks initial_locks, + Eterm tref, int cancel, Sint64 time_left) +{ + ErtsProcLocks locks = initial_locks; + ErtsMessage *mp; + Eterm tag, res, msg, ref; + Uint hsz; + Eterm *hp; + ErlOffHeap *ohp; + + hsz = 4; + hsz += NC_HEAP_SIZE(tref); + + if (time_left > (Sint64) MAX_SMALL) + hsz += ERTS_SINT64_HEAP_SIZE(time_left); + + mp = erts_alloc_message_heap(proc, &locks, hsz, &hp, &ohp); + + if (cancel) + tag = am_cancel_timer; + else + tag = am_read_timer; + + ref = STORE_NC(&hp, ohp, tref); + + if (time_left < 0) + res = am_false; + else if (time_left <= (Sint64) MAX_SMALL) + res = make_small((Sint) time_left); + else + res = erts_sint64_to_big(time_left, &hp); + + msg = TUPLE3(hp, tag, ref, res); + + erts_queue_message(proc, locks, mp, msg, am_clock_service); + + locks &= ~initial_locks; + if (locks) + erts_smp_proc_unlock(proc, locks); + + return am_ok; +} + +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + +static BIF_RETTYPE +access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info) +{ + BIF_RETTYPE ret; + Eterm res; + Sint64 time_left; + + if (!is_internal_magic_ref(tref)) { + if (is_not_ref(tref)) { + ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); + return ret; + } + time_left = -1; + } + else { + ErtsMagicBinary *mbin; + mbin = (ErtsMagicBinary *) erts_magic_ref2bin(tref); + if (mbin->destructor != bif_timer_ref_destructor) + time_left = -1; + else { + ErtsBifTimer *tmr; + Uint32 sid; + tmr = magic_binary_to_btm(mbin); + sid = erts_get_ref_numbers_thr_id(internal_magic_ref_numbers(tref)); + ASSERT(1 <= sid && sid <= erts_no_schedulers); + time_left = access_btm(tmr, sid, erts_proc_sched_data(c_p), cancel); + } + } + + if (!info) + res = am_ok; + else if (!async) + res = return_info(c_p, time_left); + else + res = send_async_info(c_p, ERTS_PROC_LOCK_MAIN, + tref, cancel, time_left); + + ERTS_BIF_PREP_RET(ret, res); + + return ret; +} + +#else /* !ERTS_MAGIC_REF_BIF_TIMERS */ + +static ERTS_INLINE Eterm +send_sync_info(Process *proc, ErtsProcLocks initial_locks, + Uint32 *refn, int cancel, Sint64 time_left) +{ + ErtsProcLocks locks = initial_locks; + ErtsMessage *mp; + Eterm res, msg, ref; + Uint hsz; + Eterm *hp; + ErlOffHeap *ohp; + + hsz = 3 + ERTS_REF_THING_SIZE; + + if (time_left > (Sint64) MAX_SMALL) + hsz += ERTS_SINT64_HEAP_SIZE(time_left); + + mp = erts_alloc_message_heap(proc, &locks, hsz, &hp, &ohp); + + write_ref_thing(hp, refn[0], refn[1], refn[2]); + ref = make_internal_ref(hp); + hp += ERTS_REF_THING_SIZE; + + if (time_left < 0) + res = am_false; + else if (time_left <= (Sint64) MAX_SMALL) + res = make_small((Sint) time_left); + else + res = erts_sint64_to_big(time_left, &hp); + + msg = TUPLE2(hp, ref, res); + + erts_queue_message(proc, locks, mp, msg, am_clock_service); + + locks &= ~initial_locks; + if (locks) + erts_smp_proc_unlock(proc, locks); + + return am_ok; +} + static ERTS_INLINE Eterm access_sched_local_btm(Process *c_p, Eterm pid, - Eterm tref, Uint32 *trefn, - Uint32 *rrefn, - int async, int cancel, - int return_res, - int info) + Eterm tref, Uint32 *trefn, + Uint32 *rrefn, + int async, int cancel, + int return_res, + int info) { ErtsSchedulerData *esdp; ErtsHLTimerService *srv; - ErtsHLTimer *tmr; + ErtsBifTimer *tmr; Sint64 time_left; Process *proc; ErtsProcLocks proc_locks; @@ -1884,111 +2296,40 @@ access_sched_local_btm(Process *c_p, Eterm pid, srv = esdp->timer_service; tmr = btm_rbt_lookup(srv->btm_tree, trefn); - if (tmr) { - if (!cancel) { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->state); - if (state == ERTS_TMR_STATE_ACTIVE) - time_left = get_time_left(esdp, tmr->timeout); - } - else { - int cncl_res = cancel_bif_timer(tmr); - if (cncl_res) { - - time_left = get_time_left(esdp, tmr->timeout); - if (cncl_res > 0) - hl_timer_dec_refc(tmr, tmr->head.roflgs); - - hlt_delete_timer(esdp, tmr); - } - } - } + time_left = access_btm(tmr, (Uint32) esdp->no, esdp, cancel); if (!info) - return am_ok; - - if (return_res) { - ERTS_HLT_ASSERT(c_p); - if (time_left < 0) - return am_false; - else if (time_left <= (Sint64) MAX_SMALL) - return make_small((Sint) time_left); - else { - Uint hsz = ERTS_SINT64_HEAP_SIZE(time_left); - Eterm *hp = HAlloc(c_p, hsz); - return erts_sint64_to_big(time_left, &hp); - } - } + return am_ok; if (c_p) { - proc = c_p; - proc_locks = ERTS_PROC_LOCK_MAIN; + proc = c_p; + proc_locks = ERTS_PROC_LOCK_MAIN; } else { - proc = erts_proc_lookup(pid); - proc_locks = 0; + proc = erts_proc_lookup(pid); + proc_locks = 0; } - if (proc) { - Uint hsz; - ErtsMessage *mp; - Eterm *hp, msg, ref, result; - ErlOffHeap *ohp; - Uint32 *refn; -#ifdef ERTS_HLT_DEBUG - Eterm *hp_end; -#endif - - hsz = ERTS_REF_THING_SIZE; - if (async) { - refn = trefn; /* timer ref */ - hsz += 4; /* 3-tuple */ - } - else { - refn = rrefn; /* request ref */ - hsz += 3; /* 2-tuple */ - } - - ERTS_HLT_ASSERT(refn); - - if (time_left > (Sint64) MAX_SMALL) - hsz += ERTS_SINT64_HEAP_SIZE(time_left); - - mp = erts_alloc_message_heap(proc, &proc_locks, - hsz, &hp, &ohp); - -#ifdef ERTS_HLT_DEBUG - hp_end = hp + hsz; -#endif - - if (time_left < 0) - result = am_false; - else if (time_left <= (Sint64) MAX_SMALL) - result = make_small((Sint) time_left); - else - result = erts_sint64_to_big(time_left, &hp); - - write_ref_thing(hp, - refn[0], - refn[1], - refn[2]); - ref = make_internal_ref(hp); - hp += ERTS_REF_THING_SIZE; - - msg = (async - ? TUPLE3(hp, (cancel - ? am_cancel_timer - : am_read_timer), ref, result) - : TUPLE2(hp, ref, result)); - - ERTS_HLT_ASSERT(hp + (async ? 4 : 3) == hp_end); - - erts_queue_message(proc, proc_locks, mp, msg, am_clock_service); - - if (c_p) - proc_locks &= ~ERTS_PROC_LOCK_MAIN; - if (proc_locks) - erts_smp_proc_unlock(proc, proc_locks); + if (!async) { + if (c_p) + return return_info(c_p, time_left); + + if (proc) + return send_sync_info(proc, proc_locks, + rrefn, cancel, time_left); + } + else if (proc) { + Eterm ref; + Eterm heap[ERTS_REF_THING_SIZE]; + if (is_value(tref)) + ref = tref; + else { + write_ref_thing(&heap[0], trefn[0], trefn[1], trefn[2]); + ref = make_internal_ref(&heap[0]); + } + return send_async_info(proc, proc_locks, + ref, cancel, time_left); } return am_ok; @@ -2021,108 +2362,64 @@ bif_timer_access_request(void *vreq) static int try_access_sched_remote_btm(ErtsSchedulerData *esdp, Process *c_p, Uint32 sid, - Uint32 *trefn, + Eterm tref, Uint32 *trefn, int async, int cancel, int info, Eterm *resp) { - ErtsHLTimer *tmr; + ErtsBifTimer *tmr; Sint64 time_left; ERTS_HLT_ASSERT(c_p); /* * Check if the timer is aimed at current - * process of if this process is an accessor - * of the timer... + * process... */ erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_BTM); tmr = proc_btm_rbt_lookup(c_p->bif_timers, trefn); -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - if (!tmr) - tmr = abtm_rbt_lookup(c_p->accessor_bif_timers, trefn); -#endif erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_BTM); if (!tmr) return 0; - if (!cancel) { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->state); - if (state == ERTS_TMR_STATE_ACTIVE) - time_left = get_time_left(esdp, tmr->timeout); - else - time_left = -1; - } - else { - int cncl_res = cancel_bif_timer(tmr); - if (!cncl_res) - time_left = -1; - else { - time_left = get_time_left(esdp, tmr->timeout); - if (cncl_res > 0) - queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr); - } - } + time_left = access_btm(tmr, sid, esdp, cancel); - if (!info) { + if (!info) *resp = am_ok; - return 1; - } - - if (!async) { - if (time_left < 0) - *resp = am_false; - else if (time_left <= (Sint64) MAX_SMALL) - *resp = make_small((Sint) time_left); - else { - Uint hsz = ERTS_SINT64_HEAP_SIZE(time_left); - Eterm *hp = HAlloc(c_p, hsz); - *resp = erts_sint64_to_big(time_left, &hp); - } - } - else { - ErtsMessage *mp; - Eterm tag, res, msg, tref; - Uint hsz; - Eterm *hp; - ErtsProcLocks proc_locks = ERTS_PROC_LOCK_MAIN; - ErlOffHeap *ohp; - - hsz = 4 + ERTS_REF_THING_SIZE; - if (time_left > (Sint64) MAX_SMALL) - hsz += ERTS_SINT64_HEAP_SIZE(time_left); - - mp = erts_alloc_message_heap(c_p, &proc_locks, - hsz, &hp, &ohp); - if (cancel) - tag = am_cancel_timer; - else - tag = am_read_timer; - - write_ref_thing(hp, - trefn[0], - trefn[1], - trefn[2]); - tref = make_internal_ref(hp); - hp += ERTS_REF_THING_SIZE; - - if (time_left < 0) - res = am_false; - else if (time_left <= (Sint64) MAX_SMALL) - res = make_small((Sint) time_left); - else - res = erts_sint64_to_big(time_left, &hp); - - msg = TUPLE3(hp, tag, tref, res); + else if (!async) + *resp = return_info(c_p, time_left); + else + *resp = send_async_info(c_p, ERTS_PROC_LOCK_MAIN, + tref, cancel, time_left); - erts_queue_message(c_p, proc_locks, mp, msg, am_clock_service); + return 1; +} - proc_locks &= ~ERTS_PROC_LOCK_MAIN; - if (proc_locks) - erts_smp_proc_unlock(c_p, proc_locks); +static Eterm +no_timer_result(Process *c_p, Eterm tref, int cancel, int async, int info) +{ + ErtsMessage *mp; + Uint hsz; + Eterm *hp, msg, ref, tag; + ErlOffHeap *ohp; + ErtsProcLocks locks; - *resp = am_ok; - } - return 1; + if (!async) + return am_false; + if (!info) + return am_ok; + + hsz = 4; + hsz += NC_HEAP_SIZE(tref); + locks = ERTS_PROC_LOCK_MAIN; + mp = erts_alloc_message_heap(c_p, &locks, hsz, &hp, &ohp); + ref = STORE_NC(&hp, ohp, tref); + tag = cancel ? am_cancel_timer : am_read_timer; + msg = TUPLE3(hp, tag, ref, am_false); + erts_queue_message(c_p, locks, mp, msg, am_clock_service); + locks &= ~ERTS_PROC_LOCK_MAIN; + if (locks) + erts_smp_proc_unlock(c_p, locks); + return am_ok; } static BIF_RETTYPE @@ -2156,7 +2453,7 @@ access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info) ERTS_BIF_PREP_RET(ret, res); } else if (try_access_sched_remote_btm(esdp, c_p, - sid, trefn, + sid, tref, trefn, async, cancel, info, &res)) { ERTS_BIF_PREP_RET(ret, res); @@ -2235,11 +2532,11 @@ badarg: return ret; no_timer: - ERTS_BIF_PREP_RET(ret, am_false); - return ret; - + return no_timer_result(c_p, tref, cancel, async, info); } +#endif /* !ERTS_MAGIC_REF_BIF_TIMERS */ + static ERTS_INLINE int bool_arg(Eterm val, int *argp) { @@ -2251,8 +2548,8 @@ bool_arg(Eterm val, int *argp) } static ERTS_INLINE int -parse_bif_timer_options(Eterm option_list, int *async, int *info, - int *abs, Eterm *accessor) +parse_bif_timer_options(Eterm option_list, int *async, + int *info, int *abs) { Eterm list = option_list; @@ -2262,8 +2559,6 @@ parse_bif_timer_options(Eterm option_list, int *async, int *info, *info = 1; if (abs) *abs = 0; - if (accessor) - *accessor = THE_NON_VALUE; while (is_list(list)) { Eterm *consp, *tp, opt; @@ -2290,13 +2585,6 @@ parse_bif_timer_options(Eterm option_list, int *async, int *info, if (!abs || !bool_arg(tp[2], abs)) return 0; break; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - case am_accessor: - if (!accessor || is_not_internal_pid(tp[2])) - return 0; - *accessor = tp[2]; - break; -#endif default: return 0; } @@ -2310,42 +2598,57 @@ parse_bif_timer_options(Eterm option_list, int *async, int *info, } static void -exit_cancel_bif_timer(ErtsHLTimer *tmr, void *vesdp) +exit_cancel_bif_timer(ErtsBifTimer *tmr, void *vesdp) { ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp; Uint32 sid, roflgs; erts_aint_t state; + int is_hlt; - state = erts_smp_atomic32_cmpxchg_acqb(&tmr->state, + state = erts_smp_atomic32_cmpxchg_acqb(&tmr->btm.state, ERTS_TMR_STATE_CANCELED, ERTS_TMR_STATE_ACTIVE); - roflgs = tmr->head.roflgs; + roflgs = tmr->type.head.roflgs; sid = roflgs & ERTS_TMR_ROFLG_SID_MASK; + is_hlt = !!(roflgs & ERTS_TMR_ROFLG_HLT); - ERTS_HLT_ASSERT(sid == erts_get_ref_numbers_thr_id(tmr->btm.refn)); + ERTS_HLT_ASSERT(sid == erts_get_ref_numbers_thr_id(ERTS_BTM_HLT2REFN(tmr))); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ERTS_HLT_ASSERT(tmr->btm.proc_list.next); +#else ERTS_HLT_ASSERT(tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE); - tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; +#endif - if (sid == (Uint32) esdp->no) { - if (state == ERTS_TMR_STATE_ACTIVE) { - if (tmr->btm.bp) - free_message_buffer(tmr->btm.bp); - hlt_delete_timer(esdp, tmr); - } - hl_timer_dec_refc(tmr, roflgs); - } - else { - if (state == ERTS_TMR_STATE_ACTIVE) { - if (tmr->btm.bp) - free_message_buffer(tmr->btm.bp); - queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr); + if (state == ERTS_TMR_STATE_ACTIVE) { +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + btm_clear_magic_binary(tmr); +#endif + if (tmr->btm.bp) + free_message_buffer(tmr->btm.bp); + + if (sid != (Uint32) esdp->no) { + queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr); + return; + } + +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + btm_rbt_delete(&esdp->timer_service->btm_tree, tmr); + tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; } - else - hl_timer_dec_refc(tmr, roflgs); +#endif + if (is_hlt) + hlt_delete_timer(esdp, &tmr->type.hlt); + else + cancel_tw_timer(esdp, &tmr->type.twt); } + if (is_hlt) + hl_timer_dec_refc(&tmr->type.hlt, roflgs); + else + tw_timer_dec_refc(&tmr->type.twt); } #ifdef ERTS_HLT_DEBUG @@ -2354,20 +2657,29 @@ exit_cancel_bif_timer(ErtsHLTimer *tmr, void *vesdp) # define ERTS_BTM_MAX_DESTROY_LIMIT 50 #endif +#ifndef ERTS_MAGIC_REF_BIF_TIMERS typedef struct { ErtsBifTimers *bif_timers; union { proc_btm_rbt_yield_state_t proc_btm_yield_state; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - abtm_rbt_yield_state_t abtm_yield_state; -#endif } u; } ErtsBifTimerYieldState; +#endif -int erts_cancel_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp) +int erts_cancel_bif_timers(Process *p, ErtsBifTimers **btm, void **vyspp) { ErtsSchedulerData *esdp = erts_proc_sched_data(p); - ErtsBifTimerYieldState ys = {btm, {ERTS_RBT_YIELD_STAT_INITER}}; + +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + + return proc_btm_list_foreach_destroy_yielding(btm, + exit_cancel_bif_timer, + (void *) esdp, + ERTS_BTM_MAX_DESTROY_LIMIT); + +#else /* !ERTS_MAGIC_REF_BIF_TIMERS */ + + ErtsBifTimerYieldState ys = {*btm, {ERTS_RBT_YIELD_STAT_INITER}}; ErtsBifTimerYieldState *ysp; int res; @@ -2399,63 +2711,18 @@ int erts_cancel_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp) } return res; -} - -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - -static void -detach_bif_timer(ErtsHLTimer *tmr, void *vesdp) -{ - tmr->abtm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - hl_timer_dec_refc(tmr, tmr->head.roflgs); -} - -int erts_detach_accessor_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp) -{ - ErtsSchedulerData *esdp = erts_proc_sched_data(p); - ErtsBifTimerYieldState ys = {btm, {ERTS_RBT_YIELD_STAT_INITER}}; - ErtsBifTimerYieldState *ysp; - int res; - - ysp = (ErtsBifTimerYieldState *) *vyspp; - if (!ysp) - ysp = &ys; - - res = abtm_rbt_foreach_destroy_yielding(&ysp->bif_timers, - detach_bif_timer, - (void *) esdp, - &ysp->u.abtm_yield_state, - ERTS_BTM_MAX_DESTROY_LIMIT); - - if (res == 0) { - if (ysp != &ys) - erts_free(ERTS_ALC_T_BTM_YIELD_STATE, ysp); - *vyspp = NULL; - } - else { - - if (ysp == &ys) { - ysp = erts_alloc(ERTS_ALC_T_BTM_YIELD_STATE, - sizeof(ErtsBifTimerYieldState)); - sys_memcpy((void *) ysp, (void *) &ys, - sizeof(ErtsBifTimerYieldState)); - } - - *vyspp = (void *) ysp; - } - return res; +#endif /* !ERTS_MAGIC_REF_BIF_TIMERS */ } -#endif /* ERTS_BTM_ACCESSOR_SUPPORT */ - static ERTS_INLINE int parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg, ErtsMonotonicTime *conv_arg, int abs, - ErtsMonotonicTime *tposp, int *stimep) + ErtsMonotonicTime *tposp, int *stimep, + ErtsMonotonicTime *msp) { - ErtsMonotonicTime t; - + ErtsMonotonicTime t, now; + if (!term_to_Sint64(arg, &t)) { ERTS_HLT_ASSERT(!is_small(arg)); if (!is_big(arg)) @@ -2470,22 +2737,30 @@ parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg, if (conv_arg) *conv_arg = t; + now = erts_get_monotonic_time(esdp); + if (abs) { t += -1*ERTS_MONOTONIC_OFFSET_MSEC; /* external to internal */ if (t < ERTS_MONOTONIC_TO_MSEC(ERTS_MONOTONIC_BEGIN)) return 1; if (t > ERTS_MONOTONIC_TO_MSEC(ERTS_MONOTONIC_END)) return 1; + if (msp) + *msp = t - ERTS_MONOTONIC_TO_MSEC(now); + *stimep = (t - ERTS_MONOTONIC_TO_MSEC(esdp->last_monotonic_time) < ERTS_BIF_TIMER_SHORT_TIME); *tposp = ERTS_MSEC_TO_CLKTCKS(t); } else { - ErtsMonotonicTime now, ticks; + ErtsMonotonicTime ticks; if (t < 0) return -1; + if (msp) + *msp = t; + ticks = ERTS_MSEC_TO_CLKTCKS(t); if (ERTS_CLKTCK_RESOLUTION > 1000 && ticks < 0) @@ -2493,7 +2768,6 @@ parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg, ERTS_HLT_ASSERT(ticks >= 0); - now = erts_get_monotonic_time(esdp); ticks += ERTS_MONOTONIC_TO_CLKTCKS(now-1); ticks += 1; @@ -2516,66 +2790,68 @@ parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg, BIF_RETTYPE send_after_3(BIF_ALIST_3) { - ErtsMonotonicTime timeout_pos; + ErtsMonotonicTime timeout_pos, tmo; int short_time, tres; - tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL, - 0, &timeout_pos, &short_time); + tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, + NULL, 0, &timeout_pos, &short_time, &tmo); if (tres != 0) BIF_ERROR(BIF_P, BADARG); - return setup_bif_timer(BIF_P, timeout_pos, short_time, - BIF_ARG_2, BIF_ARG_2, BIF_ARG_3, 0); + return setup_bif_timer(BIF_P, tmo < ERTS_TIMER_WHEEL_MSEC, + timeout_pos, short_time, BIF_ARG_2, + BIF_ARG_3, 0); } BIF_RETTYPE send_after_4(BIF_ALIST_4) { - ErtsMonotonicTime timeout_pos; - Eterm accessor; + ErtsMonotonicTime timeout_pos, tmo; int short_time, abs, tres; - if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs, &accessor)) + if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs)) BIF_ERROR(BIF_P, BADARG); tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL, - abs, &timeout_pos, &short_time); + abs, &timeout_pos, &short_time, &tmo); if (tres != 0) BIF_ERROR(BIF_P, BADARG); - return setup_bif_timer(BIF_P, timeout_pos, short_time, - BIF_ARG_2, accessor, BIF_ARG_3, 0); + return setup_bif_timer(BIF_P, tmo < ERTS_TIMER_WHEEL_MSEC, + timeout_pos, short_time, BIF_ARG_2, + BIF_ARG_3, 0); } BIF_RETTYPE start_timer_3(BIF_ALIST_3) { - ErtsMonotonicTime timeout_pos; + ErtsMonotonicTime timeout_pos, tmo; int short_time, tres; tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL, - 0, &timeout_pos, &short_time); + 0, &timeout_pos, &short_time, &tmo); if (tres != 0) BIF_ERROR(BIF_P, BADARG); - return setup_bif_timer(BIF_P, timeout_pos, short_time, - BIF_ARG_2, BIF_ARG_2, BIF_ARG_3, !0); + return setup_bif_timer(BIF_P, tmo < ERTS_TIMER_WHEEL_MSEC, + timeout_pos, short_time, BIF_ARG_2, + BIF_ARG_3, !0); } BIF_RETTYPE start_timer_4(BIF_ALIST_4) { - ErtsMonotonicTime timeout_pos; - Eterm accessor; + ErtsMonotonicTime timeout_pos, tmo; int short_time, abs, tres; - if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs, &accessor)) + if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs)) BIF_ERROR(BIF_P, BADARG); tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL, - abs, &timeout_pos, &short_time); + abs, &timeout_pos, &short_time, &tmo); if (tres != 0) BIF_ERROR(BIF_P, BADARG); - return setup_bif_timer(BIF_P, timeout_pos, short_time, - BIF_ARG_2, accessor, BIF_ARG_3, !0); + return setup_bif_timer(BIF_P, tmo < ERTS_TIMER_WHEEL_MSEC, + timeout_pos, short_time, BIF_ARG_2, + BIF_ARG_3, !0); } BIF_RETTYPE cancel_timer_1(BIF_ALIST_1) @@ -2588,7 +2864,7 @@ BIF_RETTYPE cancel_timer_2(BIF_ALIST_2) BIF_RETTYPE ret; int async, info; - if (parse_bif_timer_options(BIF_ARG_2, &async, &info, NULL, NULL)) + if (parse_bif_timer_options(BIF_ARG_2, &async, &info, NULL)) return access_bif_timer(BIF_P, BIF_ARG_1, 1, async, info); ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); @@ -2605,7 +2881,7 @@ BIF_RETTYPE read_timer_2(BIF_ALIST_2) BIF_RETTYPE ret; int async; - if (parse_bif_timer_options(BIF_ARG_2, &async, NULL, NULL, NULL)) + if (parse_bif_timer_options(BIF_ARG_2, &async, NULL, NULL)) return access_bif_timer(BIF_P, BIF_ARG_1, 0, async, 1); ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); @@ -2620,14 +2896,13 @@ start_callback_timer(ErtsSchedulerData *esdp, void *arg) { - if (twt) - create_tw_timer(esdp, ERTS_TMR_CALLBACK, NULL, - callback, arg, timeout_pos); - else - create_hl_timer(esdp, timeout_pos, 0, - ERTS_TMR_CALLBACK, NULL, - NIL, THE_NON_VALUE, NIL, - NULL, callback, arg); + ErtsCreateTimerFunc create_timer = (twt + ? create_tw_timer + : create_hl_timer); + (void) create_timer(esdp, timeout_pos, 0, + ERTS_TMR_CALLBACK, NULL, + NIL, THE_NON_VALUE, NULL, + callback, arg); } typedef struct { @@ -2704,18 +2979,18 @@ set_proc_timer_common(Process *c_p, ErtsSchedulerData *esdp, Sint64 tmo, if (tmo == 0) c_p->flags |= F_TIMO; else { + ErtsCreateTimerFunc create_timer; c_p->flags |= F_INSLPQUEUE; c_p->flags &= ~F_TIMO; - if (tmo < ERTS_TIMER_WHEEL_MSEC) - tmr = (void *) create_tw_timer(esdp, ERTS_TMR_PROC, (void *) c_p, - NULL, NULL, timeout_pos); - else - tmr = (void *) create_hl_timer(esdp, timeout_pos, short_time, - ERTS_TMR_PROC, (void *) c_p, - c_p->common.id, THE_NON_VALUE, - NIL, NULL, NULL, NULL); + create_timer = (tmo < ERTS_TIMER_WHEEL_MSEC + ? create_tw_timer + : create_hl_timer); + tmr = (void *) create_timer(esdp, timeout_pos, short_time, + ERTS_TMR_PROC, (void *) c_p, + c_p->common.id, THE_NON_VALUE, + NULL, NULL, NULL); erts_smp_atomic_set_relb(&c_p->common.timer, (erts_aint_t) tmr); } } @@ -2731,7 +3006,7 @@ erts_set_proc_timer_term(Process *c_p, Eterm etmo) == ERTS_PTMR_NONE); tres = parse_timeout_pos(esdp, etmo, &tmo, 0, - &timeout_pos, &short_time); + &timeout_pos, &short_time, NULL); if (tres != 0) return tres; @@ -2789,6 +3064,7 @@ erts_set_port_timer(Port *c_prt, Sint64 tmo) void *tmr; ErtsSchedulerData *esdp = erts_get_scheduler_data(); ErtsMonotonicTime timeout_pos; + ErtsCreateTimerFunc create_timer; if (erts_smp_atomic_read_nob(&c_prt->common.timer) != ERTS_PTMR_NONE) erts_cancel_port_timer(c_prt); @@ -2797,13 +3073,12 @@ erts_set_port_timer(Port *c_prt, Sint64 tmo) timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp), tmo); - if (tmo < ERTS_TIMER_WHEEL_MSEC) - tmr = (void *) create_tw_timer(esdp, ERTS_TMR_PORT, (void *) c_prt, - NULL, NULL, timeout_pos); - else - tmr = (void *) create_hl_timer(esdp, timeout_pos, 0, ERTS_TMR_PORT, - (void *) c_prt, c_prt->common.id, - THE_NON_VALUE, NIL, NULL, NULL, NULL); + create_timer = (tmo < ERTS_TIMER_WHEEL_MSEC + ? create_tw_timer + : create_hl_timer); + tmr = (void *) create_timer(esdp, timeout_pos, 0, ERTS_TMR_PORT, + (void *) c_prt, c_prt->common.id, + THE_NON_VALUE, NULL, NULL, NULL); erts_smp_atomic_set_relb(&c_prt->common.timer, (erts_aint_t) tmr); } @@ -2842,7 +3117,7 @@ erts_read_port_timer(Port *c_prt) if (tmr->head.roflgs & ERTS_TMR_ROFLG_HLT) timeout_pos = tmr->hlt.timeout; else - timeout_pos = tmr->twt.tw_tmr.timeout_pos; + timeout_pos = erts_tweel_read_timeout(&tmr->twt.u.tw_tmr); return get_time_left(NULL, timeout_pos); } @@ -2857,20 +3132,35 @@ typedef struct { } ErtsBTMPrint; static void -btm_print(ErtsHLTimer *tmr, void *vbtmp) +btm_print(ErtsBifTimer *tmr, void *vbtmp, ErtsMonotonicTime tpos, int is_hlt) { ErtsBTMPrint *btmp = (ErtsBTMPrint *) vbtmp; ErtsMonotonicTime left; Eterm receiver; - if (tmr->timeout <= btmp->now) - left = 0; - else - left = ERTS_CLKTCKS_TO_MSEC(tmr->timeout - btmp->now); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + if (!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) + return; +#endif - receiver = ((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME) - ? tmr->receiver.name - : tmr->receiver.proc->common.id); + if (is_hlt) { + ERTS_HLT_ASSERT(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT); + if (tmr->type.hlt.timeout <= btmp->now) + left = 0; + else + left = ERTS_CLKTCKS_TO_MSEC(tmr->type.hlt.timeout - btmp->now); + } + else { + ERTS_HLT_ASSERT(!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT)); + if (tpos <= btmp->now) + left = 0; + else + left = ERTS_CLKTCKS_TO_MSEC(tpos - btmp->now); + } + + receiver = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) + ? tmr->type.head.receiver.name + : tmr->type.head.receiver.proc->common.id); erts_print(btmp->to, btmp->to_arg, "=timer:%T\n" @@ -2881,6 +3171,36 @@ btm_print(ErtsHLTimer *tmr, void *vbtmp) (Sint64) left); } +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + +static void +hlt_btm_print(ErtsHLTimer *tmr, void *vbtmp) +{ + btm_print((ErtsBifTimer *) tmr, vbtmp, 0, 1); +} + +static void +twt_btm_print(void *vbtmp, ErtsMonotonicTime tpos, void *vtwtp) +{ + btm_print((ErtsBifTimer *) vtwtp, vbtmp, tpos, 0); +} + +#else + +static void +btm_tree_print(ErtsBifTimer *tmr, void *vbtmp) +{ + int is_hlt = !!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT); + ErtsMonotonicTime tpos; + if (is_hlt) + tpos = 0; + else + tpos = erts_tweel_read_timeout(&tmr->type.twt.u.tw_tmr); + btm_print(tmr, vbtmp, tpos, is_hlt); +} + +#endif + void erts_print_bif_timer_info(fmtfn_t to, void *to_arg) { @@ -2898,7 +3218,15 @@ erts_print_bif_timer_info(fmtfn_t to, void *to_arg) for (six = 0; six < erts_no_schedulers; six++) { ErtsHLTimerService *srv = erts_aligned_scheduler_data[six].esd.timer_service; - btm_rbt_foreach(srv->btm_tree, btm_print, (void *) &btmp); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsTimerWheel *twheel = + erts_aligned_scheduler_data[six].esd.timer_wheel; + erts_twheel_debug_foreach(twheel, tw_bif_timer_timeout, + twt_btm_print, (void *) &btmp); + time_rbt_foreach(srv->time_tree, hlt_btm_print, (void *) &btmp); +#else + btm_rbt_foreach(srv->btm_tree, btm_tree_print, (void *) &btmp); +#endif } } @@ -2911,19 +3239,37 @@ typedef struct { } ErtsBTMForeachDebug; static void -debug_btm_foreach(ErtsHLTimer *tmr, void *vbtmfd) +debug_btm_foreach(ErtsBifTimer *tmr, void *vbtmfd) { - if (erts_smp_atomic32_read_nob(&tmr->state) == ERTS_TMR_STATE_ACTIVE) { +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + if (!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) + return; +#endif + if (erts_smp_atomic32_read_nob(&tmr->btm.state) == ERTS_TMR_STATE_ACTIVE) { ErtsBTMForeachDebug *btmfd = (ErtsBTMForeachDebug *) vbtmfd; - (*btmfd->func)(((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME) - ? tmr->receiver.name - : tmr->receiver.proc->common.id), - tmr->btm.message, - tmr->btm.bp, - btmfd->arg); + Eterm id = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) + ? tmr->type.head.receiver.name + : tmr->type.head.receiver.proc->common.id); + (*btmfd->func)(id, tmr->btm.message, tmr->btm.bp, btmfd->arg); } } +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + +static void +hlt_debug_btm_foreach(ErtsHLTimer *tmr, void *vbtmfd) +{ + debug_btm_foreach((ErtsBifTimer *) tmr, vbtmfd); +} + +static void +twt_debug_btm_foreach(void *vbtmfd, ErtsMonotonicTime tpos, void *vtwtp) +{ + debug_btm_foreach((ErtsBifTimer *) vtwtp, vbtmfd); +} + +#endif + void erts_debug_bif_timer_foreach(void (*func)(Eterm, Eterm, @@ -2943,9 +3289,20 @@ erts_debug_bif_timer_foreach(void (*func)(Eterm, for (six = 0; six < erts_no_schedulers; six++) { ErtsHLTimerService *srv = erts_aligned_scheduler_data[six].esd.timer_service; +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsTimerWheel *twheel = + erts_aligned_scheduler_data[six].esd.timer_wheel; + erts_twheel_debug_foreach(twheel, tw_bif_timer_timeout, + twt_debug_btm_foreach, + (void *) &btmfd); + time_rbt_foreach(srv->time_tree, + hlt_debug_btm_foreach, + (void *) &btmfd); +#else btm_rbt_foreach(srv->btm_tree, debug_btm_foreach, (void *) &btmfd); +#endif } } @@ -2964,7 +3321,7 @@ debug_callback_timer_foreach_list(ErtsHLTimer *tmr, void *vdfct) = (ErtsDebugForeachCallbackTimer *) vdfct; if ((tmr->head.roflgs & ERTS_TMR_ROFLG_CALLBACK) - && (tmr->receiver.callback == dfct->tclbk)) + && (tmr->head.receiver.callback == dfct->tclbk)) (*dfct->func)(dfct->arg, tmr->timeout, tmr->head.u.arg); @@ -2982,7 +3339,7 @@ debug_callback_timer_foreach(ErtsHLTimer *tmr, void *vdfct) vdfct); if ((tmr->head.roflgs & ERTS_TMR_ROFLG_CALLBACK) - && (tmr->receiver.callback == dfct->tclbk)) + && (tmr->head.receiver.callback == dfct->tclbk)) (*dfct->func)(dfct->arg, tmr->timeout, tmr->head.u.arg); @@ -2997,7 +3354,7 @@ debug_tw_callback_timer(void *vdfct, ErtsDebugForeachCallbackTimer *dfct = (ErtsDebugForeachCallbackTimer *) vdfct; - if (twtp->u.callback == dfct->tclbk) + if (twtp->head.receiver.callback == dfct->tclbk) (*dfct->func)(dfct->arg, timeout_pos, twtp->head.u.arg); @@ -3068,7 +3425,9 @@ st_hdbg_func(ErtsHLTimer *tmr, void *vhdbg) } ERTS_HLT_ASSERT(tmr->time.tree.u.l.next->time.tree.u.l.prev == tmr); ERTS_HLT_ASSERT(tmr->time.tree.u.l.prev->time.tree.u.l.next == tmr); - ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, tmr->btm.refn) == tmr); +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, ERTS_BTM_HLT2REFN(tmr)) == tmr); +#endif } static void @@ -3097,8 +3456,10 @@ tt_hdbg_func(ErtsHLTimer *tmr, void *vhdbg) & ~ERTS_HLT_PFLGS_MASK); ERTS_HLT_ASSERT(tmr == prnt); } +#ifndef ERTS_MAGIC_REF_BIF_TIMERS if (tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR) - ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, tmr->btm.refn) == tmr); + ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, ERTS_BTM_HLT2REFN(tmr)) == tmr); +#endif if (tmr->time.tree.same_time) { ErtsHdbgHLT st_hdbg; st_hdbg.srv = hdbg->srv; @@ -3164,6 +3525,7 @@ hdbg_chk_srv(ErtsHLTimerService *srv) time_rbt_foreach(srv->time_tree, tt_hdbg_func, (void *) &hdbg); ERTS_HLT_ASSERT(hdbg.found_root); } +#ifndef ERTS_MAGIC_REF_BIF_TIMERS if (srv->btm_tree) { ErtsHdbgHLT hdbg; hdbg.srv = srv; @@ -3172,6 +3534,7 @@ hdbg_chk_srv(ErtsHLTimerService *srv) btm_rbt_foreach(srv->btm_tree, bt_hdbg_func, (void *) &hdbg); ERTS_HLT_ASSERT(hdbg.found_root); } +#endif } #endif /* ERTS_HLT_HARD_DEBUG */ diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h index 9cdcd581a0..f70fcdd1c0 100644 --- a/erts/emulator/beam/erl_hl_timer.h +++ b/erts/emulator/beam/erl_hl_timer.h @@ -21,7 +21,7 @@ #ifndef ERL_HL_TIMER_H__ #define ERL_HL_TIMER_H__ -typedef struct ErtsHLTimer_ ErtsBifTimers; +typedef struct ErtsBifTimer_ ErtsBifTimers; typedef struct ErtsHLTimerService_ ErtsHLTimerService; #include "sys.h" @@ -56,7 +56,7 @@ void erts_cancel_proc_timer(Process *); void erts_set_port_timer(Port *, Sint64); void erts_cancel_port_timer(Port *); Sint64 erts_read_port_timer(Port *); -int erts_cancel_bif_timers(Process *, ErtsBifTimers *, void **); +int erts_cancel_bif_timers(Process *, ErtsBifTimers **, void **); int erts_detach_accessor_bif_timers(Process *, ErtsBifTimers *, void **); ErtsHLTimerService *erts_create_timer_service(void); void erts_hl_timer_init(void); diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 541bfec532..eaaf5c911a 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -61,8 +61,9 @@ #define ERTS_DEFAULT_NO_ASYNC_THREADS 10 -#define ERTS_DEFAULT_SCHED_STACK_SIZE 256 -#define ERTS_MIN_SCHED_STACK_SIZE 20 +#define ERTS_DEFAULT_SCHED_STACK_SIZE 128 +#define ERTS_DEFAULT_DCPU_SCHED_STACK_SIZE 40 +#define ERTS_DEFAULT_DIO_SCHED_STACK_SIZE 40 /* * The variables below (prefixed with etp_) are for erts/etc/unix/etp-commands @@ -641,9 +642,22 @@ void erts_usage(void) erts_fprintf(stderr, "-swt val set scheduler wakeup threshold, valid values are:\n"); erts_fprintf(stderr, " very_low|low|medium|high|very_high.\n"); erts_fprintf(stderr, "-sss size suggested stack size in kilo words for scheduler threads,\n"); - erts_fprintf(stderr, " valid range is [%d-%d]\n", + erts_fprintf(stderr, " valid range is [%d-%d] (default %d)\n", + ERTS_SCHED_THREAD_MIN_STACK_SIZE, + ERTS_SCHED_THREAD_MAX_STACK_SIZE, + ERTS_DEFAULT_SCHED_STACK_SIZE); +#ifdef ERTS_DIRTY_SCHEDULERS + erts_fprintf(stderr, "-sssdcpu size suggested stack size in kilo words for dirty CPU scheduler\n"); + erts_fprintf(stderr, " threads, valid range is [%d-%d] (default %d)\n", + ERTS_SCHED_THREAD_MIN_STACK_SIZE, + ERTS_SCHED_THREAD_MAX_STACK_SIZE, + ERTS_DEFAULT_DCPU_SCHED_STACK_SIZE); + erts_fprintf(stderr, "-sssdio size suggested stack size in kilo words for dirty IO scheduler\n"); + erts_fprintf(stderr, " threads, valid range is [%d-%d] (default %d)\n", ERTS_SCHED_THREAD_MIN_STACK_SIZE, - ERTS_SCHED_THREAD_MAX_STACK_SIZE); + ERTS_SCHED_THREAD_MAX_STACK_SIZE, + ERTS_DEFAULT_DIO_SCHED_STACK_SIZE); +#endif erts_fprintf(stderr, "-spp Bool set port parallelism scheduling hint\n"); erts_fprintf(stderr, "-S n1:n2 set number of schedulers (n1), and number of\n"); erts_fprintf(stderr, " schedulers online (n2), maximum for both\n"); @@ -1327,6 +1341,10 @@ erl_start(int argc, char **argv) * a lot of stack. */ erts_sched_thread_suggested_stack_size = ERTS_DEFAULT_SCHED_STACK_SIZE; +#ifdef ERTS_DIRTY_SCHEDULERS + erts_dcpu_sched_thread_suggested_stack_size = ERTS_DEFAULT_DCPU_SCHED_STACK_SIZE; + erts_dio_sched_thread_suggested_stack_size = ERTS_DEFAULT_DIO_SCHED_STACK_SIZE; +#endif #ifdef DEBUG verbose = DEBUG_DEFAULT; @@ -1945,6 +1963,42 @@ erl_start(int argc, char **argv) VERBOSE(DEBUG_SYSTEM, ("scheduler wakeup threshold: %s\n", arg)); } +#ifdef ERTS_DIRTY_SCHEDULERS + else if (has_prefix("ssdcpu", sub_param)) { + /* suggested stack size (Kilo Words) for dirty CPU scheduler threads */ + arg = get_arg(sub_param+6, argv[i+1], &i); + erts_dcpu_sched_thread_suggested_stack_size = atoi(arg); + + if ((erts_dcpu_sched_thread_suggested_stack_size + < ERTS_SCHED_THREAD_MIN_STACK_SIZE) + || (erts_dcpu_sched_thread_suggested_stack_size > + ERTS_SCHED_THREAD_MAX_STACK_SIZE)) { + erts_fprintf(stderr, "bad stack size for dirty CPU scheduler threads %s\n", + arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, + ("suggested dirty CPU scheduler thread stack size %d kilo words\n", + erts_dcpu_sched_thread_suggested_stack_size)); + } + else if (has_prefix("ssdio", sub_param)) { + /* suggested stack size (Kilo Words) for dirty IO scheduler threads */ + arg = get_arg(sub_param+5, argv[i+1], &i); + erts_dio_sched_thread_suggested_stack_size = atoi(arg); + + if ((erts_dio_sched_thread_suggested_stack_size + < ERTS_SCHED_THREAD_MIN_STACK_SIZE) + || (erts_dio_sched_thread_suggested_stack_size > + ERTS_SCHED_THREAD_MAX_STACK_SIZE)) { + erts_fprintf(stderr, "bad stack size for dirty IO scheduler threads %s\n", + arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, + ("suggested dirty IO scheduler thread stack size %d kilo words\n", + erts_dio_sched_thread_suggested_stack_size)); + } +#endif else if (has_prefix("ss", sub_param)) { /* suggested stack size (Kilo Words) for scheduler threads */ arg = get_arg(sub_param+2, argv[i+1], &i); @@ -2259,8 +2313,14 @@ erl_start(int argc, char **argv) boot_argc = argc - i; /* Number of arguments to init */ boot_argv = &argv[i]; - if (erts_sched_thread_suggested_stack_size < ERTS_MIN_SCHED_STACK_SIZE) - erts_sched_thread_suggested_stack_size = ERTS_MIN_SCHED_STACK_SIZE; + if (erts_sched_thread_suggested_stack_size < ERTS_SCHED_THREAD_MIN_STACK_SIZE) + erts_sched_thread_suggested_stack_size = ERTS_SCHED_THREAD_MIN_STACK_SIZE; +#ifdef ERTS_DIRTY_SCHEDULERS + if (erts_dcpu_sched_thread_suggested_stack_size < ERTS_SCHED_THREAD_MIN_STACK_SIZE) + erts_dcpu_sched_thread_suggested_stack_size = ERTS_SCHED_THREAD_MIN_STACK_SIZE; + if (erts_dio_sched_thread_suggested_stack_size < ERTS_SCHED_THREAD_MIN_STACK_SIZE) + erts_dio_sched_thread_suggested_stack_size = ERTS_SCHED_THREAD_MIN_STACK_SIZE; +#endif erl_init(ncpu, proc_tab_sz, @@ -2334,6 +2394,7 @@ erl_start(int argc, char **argv) set_main_stack_size(); erts_sched_init_time_sup(esdp); erts_ets_sched_spec_data_init(esdp); + erts_aux_work_timeout_late_init(esdp); process_main(esdp->x_reg_array, esdp->f_reg_array); } #endif diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index f181c1e3cb..17982a2d14 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -167,9 +167,7 @@ erts_cleanup_offheap(ErlOffHeap *offheap) for (u.hdr = offheap->first; u.hdr; u.hdr = u.hdr->next) { switch (thing_subtag(u.hdr->thing_word)) { case REFC_BINARY_SUBTAG: - if (erts_refc_dectest(&u.pb->val->refc, 0) == 0) { - erts_bin_free(u.pb->val); - } + erts_bin_release(u.pb->val); break; case FUN_SUBTAG: if (erts_smp_refc_dectest(&u.fun->fe->refc, 0) == 0) { @@ -178,8 +176,7 @@ erts_cleanup_offheap(ErlOffHeap *offheap) break; case REF_SUBTAG: ASSERT(is_magic_ref_thing(u.hdr)); - if (erts_refc_dectest(&u.mref->mb->refc, 0) == 0) - erts_bin_free((Binary *)u.mref->mb); + erts_bin_release((Binary *)u.mref->mb); break; default: ASSERT(is_external_header(u.hdr->thing_word)); diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index f86b9739fa..872b58d1ef 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1074,7 +1074,6 @@ int enif_alloc_binary(size_t size, ErlNifBinary* bin) if (refbin == NULL) { return 0; /* The NIF must take action */ } - erts_refc_init(&refbin->refc, 1); bin->size = size; bin->data = (unsigned char*) refbin->orig_bytes; @@ -1113,9 +1112,7 @@ void enif_release_binary(ErlNifBinary* bin) if (bin->ref_bin != NULL) { Binary* refbin = bin->ref_bin; ASSERT(bin->bin_term == THE_NON_VALUE); - if (erts_refc_dectest(&refbin->refc, 0) == 0) { - erts_bin_free(refbin); - } + erts_bin_release(refbin); } #ifdef DEBUG bin->data = NULL; @@ -1279,7 +1276,7 @@ Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin) OH_OVERHEAD(&(MSO(env->proc)), pb->size / sizeof(Eterm)); bin_term = make_binary(pb); - if (erts_refc_read(&bptr->refc, 1) == 1) { + if (erts_refc_read(&bptr->intern.refc, 1) == 1) { /* Total ownership transfer */ bin->ref_bin = NULL; bin->bin_term = bin_term; @@ -2249,7 +2246,7 @@ static int nif_resource_dtor(Binary* bin) ASSERT(type->down); erts_smp_mtx_lock(&rm->lock); - ASSERT(erts_refc_read(&bin->refc, 0) == 0); + ASSERT(erts_refc_read(&bin->intern.refc, 0) == 0); if (rm->root) { ASSERT(!rm->is_dying); destroy_all_monitors(rm->root, resource); @@ -2323,13 +2320,13 @@ void erts_fire_nif_monitor(ErtsResource* resource, Eterm pid, Eterm ref) erts_smp_mtx_unlock(&rmp->lock); if (free_me) { - ASSERT(erts_refc_read(&bin->binary.refc, 0) == 0); + ASSERT(erts_refc_read(&bin->binary.intern.refc, 0) == 0); erts_bin_free(&bin->binary); } return; } ASSERT(!rmp->is_dying); - if (erts_refc_inc_unless(&bin->binary.refc, 0, 0) == 0) { + if (erts_refc_inc_unless(&bin->binary.intern.refc, 0, 0) == 0) { /* * Racing resource destruction. * To avoid a more complex refc-dance with destructing thread @@ -2348,9 +2345,7 @@ void erts_fire_nif_monitor(ErtsResource* resource, Eterm pid, Eterm ref) resource->type->down(&msg_env.env, resource->data, &nif_pid, &nif_monitor); post_nif_noproc(&msg_env); - if (erts_refc_dectest(&bin->binary.refc, 0) == 0) { - erts_bin_free(&bin->binary); - } + erts_bin_release(&bin->binary); } erts_destroy_monitor(rmon); } @@ -2379,7 +2374,7 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t data_sz) ASSERT(type->owner && type->next && type->prev); /* not allowed in load/upgrade */ resource->type = type; - erts_refc_inc(&bin->refc, 1); + erts_refc_inc(&bin->intern.refc, 1); #ifdef DEBUG erts_refc_init(&resource->nif_refc, 1); #endif @@ -2408,9 +2403,7 @@ void enif_release_resource(void* obj) #ifdef DEBUG erts_refc_dec(&resource->nif_refc, 0); #endif - if (erts_refc_dectest(&bin->binary.refc, 0) == 0) { - erts_bin_free(&bin->binary); - } + erts_bin_release(&bin->binary); } void enif_keep_resource(void* obj) @@ -2423,7 +2416,7 @@ void enif_keep_resource(void* obj) #ifdef DEBUG erts_refc_inc(&resource->nif_refc, 1); #endif - erts_refc_inc(&bin->binary.refc, 2); + erts_refc_inc(&bin->binary.intern.refc, 2); } Eterm erts_bld_resource_ref(Eterm** hpp, ErlOffHeap* oh, ErtsResource* resource) @@ -2460,7 +2453,7 @@ ERL_NIF_TERM enif_make_resource_binary(ErlNifEnv* env, void* obj, pb->flags = 0; OH_OVERHEAD(ohp, size / sizeof(Eterm)); - erts_refc_inc(&bin->binary.refc, 1); + erts_refc_inc(&bin->binary.intern.refc, 1); return make_binary(hp); } @@ -2485,7 +2478,7 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ } */ mbin = ((ProcBin *) hp)->val; - if (!(mbin->flags & BIN_FLAG_MAGIC)) + if (!(mbin->intern.flags & BIN_FLAG_MAGIC)) return 0; } resource = (ErtsResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(mbin); @@ -3658,11 +3651,11 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ci = *get_func_pp(this_mi->code_hdr, f_atom, f->arity); code_ptr = erts_codeinfo_to_code(ci); - if (ci->native == 0) { + if (ci->u.gen_bp == NULL) { code_ptr[0] = (BeamInstr) BeamOp(op_call_nif); } else { /* Function traced, patch the original instruction word */ - GenericBp* g = (GenericBp *) ci->native; + GenericBp* g = ci->u.gen_bp; ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)); g->orig_instr = (BeamInstr) BeamOp(op_call_nif); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 894e0ee582..9947e33f47 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -194,7 +194,10 @@ static UWord thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_O ErtsPTab erts_proc erts_align_attribute(ERTS_CACHE_LINE_SIZE); int erts_sched_thread_suggested_stack_size = -1; - +#ifdef ERTS_DIRTY_SCHEDULERS +int erts_dcpu_sched_thread_suggested_stack_size = -1; +int erts_dio_sched_thread_suggested_stack_size = -1; +#endif #ifdef ERTS_ENABLE_LOCK_CHECK ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE]; #endif @@ -562,7 +565,6 @@ static int stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg); static void aux_work_timeout(void *unused); static void aux_work_timeout_early_init(int no_schedulers); -static void aux_work_timeout_late_init(void); static void setup_aux_work_timer(ErtsSchedulerData *esdp); static int execute_sys_tasks(Process *c_p, @@ -2792,6 +2794,9 @@ typedef struct { int initialized; erts_atomic32_t refc; +#ifdef DEBUG + erts_atomic32_t used; +#endif erts_atomic32_t type[1]; } ErtsAuxWorkTmo; @@ -2801,6 +2806,13 @@ static ERTS_INLINE void start_aux_work_timer(ErtsSchedulerData *esdp) { ErtsMonotonicTime tmo = erts_get_monotonic_time(esdp); +#ifdef DEBUG + Uint no = (Uint) erts_atomic32_xchg_mb(&aux_work_tmo->used, + (erts_aint32_t) esdp->no); + ASSERT(esdp->type == ERTS_SCHED_NORMAL); + ASSERT(!no); +#endif + tmo = ERTS_MONOTONIC_TO_CLKTCKS(tmo-1); tmo += ERTS_MSEC_TO_CLKTCKS(1000) + 1; erts_twheel_init_timer(&aux_work_tmo->timer.data); @@ -2808,7 +2820,6 @@ start_aux_work_timer(ErtsSchedulerData *esdp) erts_twheel_set_timer(esdp->timer_wheel, &aux_work_tmo->timer.data, aux_work_timeout, - NULL, (void *) esdp, tmo); } @@ -2837,16 +2848,19 @@ aux_work_timeout_early_init(int no_schedulers) aux_work_tmo = (ErtsAuxWorkTmo *) p; aux_work_tmo->initialized = 0; erts_atomic32_init_nob(&aux_work_tmo->refc, 0); +#ifdef DEBUG + erts_atomic32_init_nob(&aux_work_tmo->used, 0); +#endif for (i = 0; i <= no_schedulers; i++) erts_atomic32_init_nob(&aux_work_tmo->type[i], 0); } void -aux_work_timeout_late_init(void) +erts_aux_work_timeout_late_init(ErtsSchedulerData *esdp) { aux_work_tmo->initialized = 1; - if (erts_atomic32_read_nob(&aux_work_tmo->refc)) - start_aux_work_timer(erts_get_scheduler_data()); + if (erts_atomic32_read_acqb(&aux_work_tmo->refc)) + start_aux_work_timer(esdp); } static void @@ -2854,6 +2868,13 @@ aux_work_timeout(void *vesdp) { erts_aint32_t refc; int i; +#ifdef DEBUG + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + Uint no = (Uint) erts_atomic32_xchg_mb(&aux_work_tmo->used, 0); + ASSERT(no == esdp->no); + ASSERT(esdp == (ErtsSchedulerData *) vesdp); +#endif + #ifdef ERTS_SMP i = 0; #else @@ -6073,6 +6094,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, runq->scheduler = esdp; esdp->run_queue = runq; esdp->no = (Uint) num; + esdp->type = ERTS_SCHED_NORMAL; #endif esdp->ssi = ssi; @@ -6467,8 +6489,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online /* init port tasks */ erts_port_task_init(); - aux_work_timeout_late_init(); - #ifndef ERTS_SMP #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC erts_scheduler_data->verify_unused_temp_alloc @@ -8754,6 +8774,9 @@ sched_thread_func(void *vesdp) erts_sched_init_time_sup(esdp); + if (no == 1) + erts_aux_work_timeout_late_init(esdp); + (void) ERTS_RUNQ_FLGS_SET_NOB(esdp->run_queue, ERTS_RUNQ_FLG_EXEC); @@ -8979,6 +9002,7 @@ erts_start_schedulers(void) for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) { ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); erts_snprintf(opts.name, 16, "%d_dirty_cpu_scheduler", ix + 1); + opts.suggested_stack_size = erts_dcpu_sched_thread_suggested_stack_size; res = ethr_thr_create(&esdp->tid,sched_dirty_cpu_thread_func,(void*)esdp,&opts); if (res != 0) erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty cpu scheduler thread %d\n", ix); @@ -8986,6 +9010,7 @@ erts_start_schedulers(void) for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix); erts_snprintf(opts.name, 16, "%d_dirty_io_scheduler", ix + 1); + opts.suggested_stack_size = erts_dio_sched_thread_suggested_stack_size; res = ethr_thr_create(&esdp->tid,sched_dirty_io_thread_func,(void*)esdp,&opts); if (res != 0) erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty io scheduler thread %d\n", ix); @@ -12429,9 +12454,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->msg_inq.len = 0; #endif p->bif_timers = NULL; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - p->accessor_bif_timers = NULL; -#endif p->mbuf = NULL; p->msg_frag = NULL; p->mbuf_sz = 0; @@ -12630,9 +12652,6 @@ void erts_init_empty_process(Process *p) p->msg.save = &p->msg.first; p->msg.len = 0; p->bif_timers = NULL; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - p->accessor_bif_timers = NULL; -#endif p->dictionary = NULL; p->seq_trace_clock = 0; p->seq_trace_lastcnt = 0; @@ -12733,9 +12752,6 @@ erts_debug_verify_clean_empty_process(Process* p) ASSERT(p->msg.first == NULL); ASSERT(p->msg.len == 0); ASSERT(p->bif_timers == NULL); -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - ASSERT(p->accessor_bif_timers == NULL); -#endif ASSERT(p->dictionary == NULL); ASSERT(p->catches == 0); ASSERT(p->cp == NULL); @@ -13801,7 +13817,7 @@ erts_continue_exit_process(Process *p) ASSERT(erts_proc_read_refc(p) > 0); if (p->bif_timers) { - if (erts_cancel_bif_timers(p, p->bif_timers, &p->u.terminate)) { + if (erts_cancel_bif_timers(p, &p->bif_timers, &p->u.terminate)) { ASSERT(erts_proc_read_refc(p) > 0); goto yield; } @@ -13809,19 +13825,6 @@ erts_continue_exit_process(Process *p) p->bif_timers = NULL; } -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - if (p->accessor_bif_timers) { - if (erts_detach_accessor_bif_timers(p, - p->accessor_bif_timers, - &p->u.terminate)) { - ASSERT(erts_proc_read_refc(p) > 0); - goto yield; - } - ASSERT(erts_proc_read_refc(p) > 0); - p->accessor_bif_timers = NULL; - } -#endif - #ifdef ERTS_SMP if (p->flags & F_SCHDLR_ONLN_WAITQ) abort_sched_onln_chng_waitq(p); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 883d9f2a4c..5b35dc3c78 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -117,7 +117,11 @@ extern Uint erts_no_dirty_io_schedulers; #endif extern Uint erts_no_run_queues; extern int erts_sched_thread_suggested_stack_size; -#define ERTS_SCHED_THREAD_MIN_STACK_SIZE 4 /* Kilo words */ +#ifdef ERTS_DIRTY_SCHEDULERS +extern int erts_dcpu_sched_thread_suggested_stack_size; +extern int erts_dio_sched_thread_suggested_stack_size; +#endif +#define ERTS_SCHED_THREAD_MIN_STACK_SIZE 20 /* Kilo words */ #define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192 /* Kilo words */ #ifdef ERTS_SMP @@ -1026,9 +1030,6 @@ struct process { ErlMessageQueue msg; /* Message queue */ ErtsBifTimers *bif_timers; /* Bif timers aiming at this process */ -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - ErtsBifTimers *accessor_bif_timers; /* Accessor bif timers */ -#endif ProcDict *dictionary; /* Process dictionary, may be NULL */ @@ -1830,6 +1831,7 @@ void erts_schedule_multi_misc_aux_work(int ignore_self, void (*func)(void *), void *arg); erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int); +void erts_aux_work_timeout_late_init(ErtsSchedulerData *esdp); void erts_sched_notify_check_cpu_bind(void); Uint erts_active_schedulers(void); void erts_init_process(int, int, int); diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 77c7c5e73c..fbf14df92b 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -447,8 +447,8 @@ heap_dump(fmtfn_t to, void *to_arg, Eterm x) ProcBin* pb = (ProcBin *) binary_val(x); Binary* val = pb->val; - if (erts_atomic_xchg_nob(&val->refc, 0) != 0) { - val->flags = (UWord) all_binaries; + if (erts_atomic_xchg_nob(&val->intern.refc, 0) != 0) { + val->intern.flags = (UWord) all_binaries; all_binaries = val; } erts_print(to, to_arg, @@ -529,7 +529,7 @@ dump_binaries(fmtfn_t to, void *to_arg, Binary* current) erts_print(to, to_arg, "%02X", bytes[i]); } erts_putc(to, to_arg, '\n'); - current = (Binary *) current->flags; + current = (Binary *) current->intern.flags; } } diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index a1c4220633..46d6da6448 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -21,19 +21,52 @@ #ifndef ERL_TIME_H__ #define ERL_TIME_H__ -/* timer wheel size NEED to be a power of 2 */ -#ifdef SMALL_MEMORY -#define ERTS_TIW_SIZE (1 << 13) -#else -#define ERTS_TIW_SIZE (1 << 16) +#if 0 +# define ERTS_TW_DEBUG +#endif +#if defined(DEBUG) && !defined(ERTS_TW_DEBUG) +# define ERTS_TW_DEBUG #endif -#if defined(DEBUG) || 0 +#if defined(ERTS_TW_DEBUG) #define ERTS_TIME_ASSERT(B) ERTS_ASSERT(B) #else #define ERTS_TIME_ASSERT(B) ((void) 1) #endif +#ifdef ERTS_TW_DEBUG +/* + * Soon wheel will handle about 1 seconds + * Later wheel will handle about 8 minutes + */ +# define ERTS_TW_SOON_WHEEL_BITS 10 +# define ERTS_TW_LATER_WHEEL_BITS 10 +#else +# ifdef SMALL_MEMORY +/* + * Soon wheel will handle about 4 seconds + * Later wheel will handle about 2 hours and 19 minutes + */ +# define ERTS_TW_SOON_WHEEL_BITS 12 +# define ERTS_TW_LATER_WHEEL_BITS 12 +# else +/* + * Soon wheel will handle about 16 seconds + * Later wheel will handle about 37 hours and 16 minutes + */ +# define ERTS_TW_SOON_WHEEL_BITS 14 +# define ERTS_TW_LATER_WHEEL_BITS 14 +# endif +#endif + +/* + * Number of slots in each timer wheel... + * + * These *need* to be a power of 2 + */ +#define ERTS_TW_SOON_WHEEL_SIZE (1 << ERTS_TW_SOON_WHEEL_BITS) +#define ERTS_TW_LATER_WHEEL_SIZE (1 << ERTS_TW_LATER_WHEEL_BITS) + typedef enum { ERTS_NO_TIME_WARP_MODE, ERTS_SINGLE_TIME_WARP_MODE, @@ -103,7 +136,10 @@ Eterm erts_system_time_source(struct process*c_p); #define ERTS_CLKTCK_RESOLUTION (erts_time_sup__.r.o.clktck_resolution) #endif -#define ERTS_TIMER_WHEEL_MSEC (ERTS_TIW_SIZE/(ERTS_CLKTCK_RESOLUTION/1000)) +#define ERTS_TW_SOON_WHEEL_MSEC (ERTS_TW_SOON_WHEEL_SIZE/(ERTS_CLKTCK_RESOLUTION/1000)) +#define ERTS_TW_LATER_WHEEL_MSEC (ERTS_TW_LATER_WHEEL_SIZE*ERTS_TW_SOON_WHEEL_MSEC/2) + +#define ERTS_TIMER_WHEEL_MSEC ERTS_TW_LATER_WHEEL_MSEC struct erts_time_sup_read_only__ { ErtsMonotonicTime monotonic_time_unit; @@ -412,34 +448,25 @@ erts_time_unit_conversion(Uint64 value, void erts_sched_init_time_sup(ErtsSchedulerData *esdp); -#define ERTS_TWHEEL_SLOT_AT_ONCE -1 -#define ERTS_TWHEEL_SLOT_INACTIVE -2 +#define ERTS_TW_SLOT_INACTIVE (-2) /* ** Timer entry: */ typedef struct erl_timer { - struct erl_timer* next; /* next entry tiw slot or chain */ - struct erl_timer* prev; /* prev entry tiw slot or chain */ - union { - struct { - void (*timeout)(void*); /* called when timeout */ - void (*cancel)(void*); /* called when cancel (may be NULL) */ - void* arg; /* argument to timeout/cancel procs */ - } func; - ErtsThrPrgrLaterOp cleanup; - } u; ErtsMonotonicTime timeout_pos; /* Timeout in absolute clock ticks */ + struct erl_timer* next; /* next entry tiw slot or chain */ + struct erl_timer* prev; /* prev entry tiw slot or chain */ + void (*timeout)(void*); /* called when timeout */ + void* arg; /* argument to timeout/cancel procs */ int slot; } ErtsTWheelTimer; typedef void (*ErlTimeoutProc)(void*); -typedef void (*ErlCancelProc)(void*); void erts_twheel_set_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p, ErlTimeoutProc timeout, - ErlCancelProc cancel, void *arg, - ErtsMonotonicTime timeout_pos); + void *arg, ErtsMonotonicTime timeout_pos); void erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p); ErtsTimerWheel *erts_create_timer_wheel(ErtsSchedulerData *esdp); @@ -447,12 +474,13 @@ ErtsMonotonicTime erts_check_next_timeout_time(ErtsSchedulerData *); ERTS_GLB_INLINE void erts_twheel_init_timer(ErtsTWheelTimer *p); ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef); +ERTS_GLB_INLINE ErtsMonotonicTime erts_tweel_read_timeout(ErtsTWheelTimer *twt); #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void erts_twheel_init_timer(ErtsTWheelTimer *p) { - p->slot = ERTS_TWHEEL_SLOT_INACTIVE; + p->slot = ERTS_TW_SLOT_INACTIVE; } ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef nxt_tmo_ref) @@ -460,6 +488,12 @@ ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef nxt_ return *((ErtsMonotonicTime *) nxt_tmo_ref); } +ERTS_GLB_INLINE ErtsMonotonicTime +erts_tweel_read_timeout(ErtsTWheelTimer *twt) +{ + return twt->timeout_pos; +} + #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ void diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index cf9d3adc86..c69fec3c80 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -678,7 +678,6 @@ check_time_correction(void *vesdp) erts_twheel_set_timer(esdp->timer_wheel, &time_sup.inf.c.parmon.timer, check_time_correction, - NULL, (void *) esdp, timeout_pos); } @@ -729,7 +728,6 @@ check_time_offset(void *vesdp) erts_twheel_set_timer(esdp->timer_wheel, &time_sup.inf.c.parmon.timer, check_time_offset, - NULL, vesdp, timeout_pos); } @@ -836,7 +834,6 @@ late_init_time_correction(ErtsSchedulerData *esdp) erts_twheel_set_timer(esdp->timer_wheel, &time_sup.inf.c.parmon.timer, check_func, - NULL, (quick_init_drift_adj ? NULL : esdp), diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 04f3160d42..870f1f142d 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1875,7 +1875,6 @@ trace_port_tmp_binary(char *bin, Sint sz, Binary **bptrp, Eterm **hp) } else { ProcBin* pb = (ProcBin *)*hp; Binary *bptr = erts_bin_nrml_alloc(sz); - erts_refc_init(&bptr->refc, 1); sys_memcpy(bptr->orig_bytes, bin, sz); pb->thing_word = HEADER_PROC_BIN; pb->size = sz; @@ -2000,8 +1999,8 @@ trace_port_receive(Port *t_p, Eterm caller, Eterm what, ...) TRACE_FUN_T_RECEIVE, am_receive, data, THE_NON_VALUE, am_true); - if (bptr && erts_refc_dectest(&bptr->refc, 1) == 0) - erts_bin_free(bptr); + if (bptr) + erts_bin_release(bptr); if (orig_hp) erts_free(ERTS_ALC_T_TMP, orig_hp); @@ -2051,8 +2050,8 @@ void trace_port_send_binary(Port *t_p, Eterm to, Eterm what, char *bin, Sint sz) send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_SEND, am_send, msg, to, am_true); - if (bptr && erts_refc_dectest(&bptr->refc, 1) == 0) - erts_bin_free(bptr); + if (bptr) + erts_bin_release(bptr); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 33ed6d7ec1..ecfef28c57 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -132,7 +132,7 @@ export_alloc(struct export_entry* tmpl_e) erts_smp_atomic_add_nob(&total_entries_bytes, sizeof(*blob)); obj = &blob->exp; obj->info.op = 0; - obj->info.native = 0; + obj->info.u.gen_bp = NULL; obj->info.mfa.module = tmpl->info.mfa.module; obj->info.mfa.function = tmpl->info.mfa.function; obj->info.mfa.arity = tmpl->info.mfa.arity; diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 285ae4ac78..f5a5da981c 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1821,7 +1821,7 @@ static int ttb_context_destructor(Binary *context_bin) case TTBEncode: DESTROY_SAVED_WSTACK(&context->s.ec.wstack); if (context->s.ec.result_bin != NULL) { /* Set to NULL if ever made alive! */ - ASSERT(erts_refc_read(&(context->s.ec.result_bin->refc),0) == 0); + ASSERT(erts_refc_read(&(context->s.ec.result_bin->intern.refc),1)); erts_bin_free(context->s.ec.result_bin); context->s.ec.result_bin = NULL; } @@ -1830,13 +1830,13 @@ static int ttb_context_destructor(Binary *context_bin) erl_zlib_deflate_finish(&(context->s.cc.stream)); if (context->s.cc.destination_bin != NULL) { /* Set to NULL if ever made alive! */ - ASSERT(erts_refc_read(&(context->s.cc.destination_bin->refc),0) == 0); + ASSERT(erts_refc_read(&(context->s.cc.destination_bin->intern.refc),1)); erts_bin_free(context->s.cc.destination_bin); context->s.cc.destination_bin = NULL; } if (context->s.cc.result_bin != NULL) { /* Set to NULL if ever made alive! */ - ASSERT(erts_refc_read(&(context->s.cc.result_bin->refc),0) == 0); + ASSERT(erts_refc_read(&(context->s.cc.result_bin->intern.refc),1)); erts_bin_free(context->s.cc.result_bin); context->s.cc.result_bin = NULL; } @@ -1920,7 +1920,6 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla } result_bin = erts_bin_nrml_alloc(size); - erts_refc_init(&result_bin->refc, 0); result_bin->orig_bytes[0] = VERSION_MAGIC; /* Next state immediately, no need to export context */ context->state = TTBEncode; @@ -1960,8 +1959,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla pb->bytes = (byte*) result_bin->orig_bytes; pb->flags = 0; OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); - erts_refc_inc(&result_bin->refc, 1); - if (context_b && erts_refc_read(&context_b->refc,0) == 0) { + if (context_b && erts_refc_read(&context_b->intern.refc,0) == 0) { erts_bin_free(context_b); } return make_binary(pb); @@ -1980,7 +1978,6 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla context->s.cc.result_bin = result_bin; result_bin = erts_bin_nrml_alloc(real_size); - erts_refc_init(&result_bin->refc, 0); result_bin->orig_bytes[0] = VERSION_MAGIC; context->s.cc.destination_bin = result_bin; @@ -2028,15 +2025,15 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla pb->next = MSO(p).first; MSO(p).first = (struct erl_off_heap_header*)pb; pb->val = result_bin; + ASSERT(erts_refc_read(&result_bin->intern.refc, 1)); pb->bytes = (byte*) result_bin->orig_bytes; pb->flags = 0; OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); - erts_refc_inc(&result_bin->refc, 1); erts_bin_free(context->s.cc.result_bin); context->s.cc.result_bin = NULL; context->alive = 0; BUMP_REDS(p, (this_time * CONTEXT_REDS) / TERM_TO_BINARY_COMPRESS_CHUNK); - if (context_b && erts_refc_read(&context_b->refc,0) == 0) { + if (context_b && erts_refc_read(&context_b->intern.refc,0) == 0) { erts_bin_free(context_b); } return make_binary(pb); @@ -2055,13 +2052,13 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla pb->bytes = (byte*) result_bin->orig_bytes; pb->flags = 0; OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); - erts_refc_inc(&result_bin->refc, 1); + ASSERT(erts_refc_read(&result_bin->intern.refc, 1)); erl_zlib_deflate_finish(&(context->s.cc.stream)); erts_bin_free(context->s.cc.destination_bin); context->s.cc.destination_bin = NULL; context->alive = 0; BUMP_REDS(p, (this_time * CONTEXT_REDS) / TERM_TO_BINARY_COMPRESS_CHUNK); - if (context_b && erts_refc_read(&context_b->refc,0) == 0) { + if (context_b && erts_refc_read(&context_b->intern.refc,0) == 0) { erts_bin_free(context_b); } return make_binary(pb); @@ -2749,7 +2746,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, erts_emasculate_writable_binary(pb); bytes += (pb->val->orig_bytes - before_realloc); } - erts_refc_inc(&pb->val->refc, 2); + erts_refc_inc(&pb->val->intern.refc, 2); sys_memcpy(&tmp, pb, sizeof(ProcBin)); tmp.next = *off_heap; @@ -3509,7 +3506,6 @@ dec_term_atom_common: } else { Binary* dbin = erts_bin_nrml_alloc(n); ProcBin* pb; - erts_refc_init(&dbin->refc, 1); pb = (ProcBin *) hp; hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; @@ -3562,7 +3558,6 @@ dec_term_atom_common: Binary* dbin = erts_bin_nrml_alloc(n); ProcBin* pb; - erts_refc_init(&dbin->refc, 1); pb = (ProcBin *) hp; pb->thing_word = HEADER_PROC_BIN; pb->size = n; @@ -3865,7 +3860,7 @@ dec_term_atom_common: sys_memcpy(pb, ep, sizeof(ProcBin)); ep += sizeof(ProcBin); - erts_refc_inc(&pb->val->refc, 1); + erts_refc_inc(&pb->val->intern.refc, 1); hp += PROC_BIN_SIZE; pb->next = factory->off_heap->first; factory->off_heap->first = (struct erl_off_heap_header*)pb; @@ -3883,7 +3878,7 @@ dec_term_atom_common: sys_memcpy(pb, ep, sizeof(ProcBin)); ep += sizeof(ProcBin); - erts_refc_inc(&pb->val->refc, 1); + erts_refc_inc(&pb->val->intern.refc, 1); hp += PROC_BIN_SIZE; pb->next = factory->off_heap->first; factory->off_heap->first = (struct erl_off_heap_header*)pb; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 139394680a..bb4d442240 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1425,14 +1425,14 @@ Eterm erts_msacc_request(Process *c_p, int action, Eterm *threads); #define MatchSetRef(MPSP) \ do { \ if ((MPSP) != NULL) { \ - erts_refc_inc(&(MPSP)->refc, 1); \ + erts_refc_inc(&(MPSP)->intern.refc, 1); \ } \ } while (0) #define MatchSetUnref(MPSP) \ do { \ - if (((MPSP) != NULL) && erts_refc_dectest(&(MPSP)->refc, 0) <= 0) { \ - erts_bin_free(MPSP); \ + if (((MPSP) != NULL)) { \ + erts_bin_release(MPSP); \ } \ } while(0) diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index ddff862607..2f3117223f 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -3791,7 +3791,6 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, Binary* bptr; bptr = erts_bin_nrml_alloc(len); - erts_refc_init(&bptr->refc, 1); sys_memcpy(bptr->orig_bytes, buf, len); pb = (ProcBin *) hp; @@ -4558,8 +4557,7 @@ static void cleanup_scheduled_control(Binary *binp, char *bufp) { if (binp) { - if (erts_refc_dectest(&binp->refc, 0) == 0) - erts_bin_free(binp); + erts_bin_release(binp); } else { if (bufp) @@ -4903,7 +4901,7 @@ erts_port_control(Process* c_p, ASSERT(bufp <= bufp + size); ASSERT(binp->orig_bytes <= bufp && bufp + size <= binp->orig_bytes + binp->orig_size); - erts_refc_inc(&binp->refc, 1); + erts_refc_inc(&binp->intern.refc, 1); } } @@ -6400,7 +6398,6 @@ driver_deliver_term(Port *prt, Eterm to, ErlDrvTermData* data, int len) ProcBin* pbp; Binary* bp = erts_bin_nrml_alloc(size); ASSERT(bufp); - erts_refc_init(&bp->refc, 1); sys_memcpy((void *) bp->orig_bytes, (void *) bufp, size); pbp = (ProcBin *) erts_produce_heap(&factory, PROC_BIN_SIZE, HEAP_EXTRA); @@ -6910,21 +6907,21 @@ ErlDrvSInt driver_binary_get_refc(ErlDrvBinary *dbp) { Binary* bp = ErlDrvBinary2Binary(dbp); - return (ErlDrvSInt) erts_refc_read(&bp->refc, 1); + return (ErlDrvSInt) erts_refc_read(&bp->intern.refc, 1); } ErlDrvSInt driver_binary_inc_refc(ErlDrvBinary *dbp) { Binary* bp = ErlDrvBinary2Binary(dbp); - return (ErlDrvSInt) erts_refc_inctest(&bp->refc, 2); + return (ErlDrvSInt) erts_refc_inctest(&bp->intern.refc, 2); } ErlDrvSInt driver_binary_dec_refc(ErlDrvBinary *dbp) { Binary* bp = ErlDrvBinary2Binary(dbp); - return (ErlDrvSInt) erts_refc_dectest(&bp->refc, 1); + return (ErlDrvSInt) erts_refc_dectest(&bp->intern.refc, 1); } @@ -6940,7 +6937,6 @@ driver_alloc_binary(ErlDrvSizeT size) bin = erts_bin_drv_alloc_fnf((Uint) size); if (!bin) return NULL; /* The driver write must take action */ - erts_refc_init(&bin->refc, 1); return Binary2ErlDrvBinary(bin); } @@ -6970,8 +6966,7 @@ void driver_free_binary(ErlDrvBinary* dbin) return; bin = ErlDrvBinary2Binary(dbin); - if (erts_refc_dectest(&bin->refc, 0) == 0) - erts_bin_free(bin); + erts_bin_release(bin); } diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 6f15082130..a7e5a64b22 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -17,57 +17,157 @@ * * %CopyrightEnd% */ - + /* - * TIMING WHEEL + * TIMER WHEEL + * + * + * The time scale used for timers is Erlang monotonic time. The + * time unit used is ERTS specific clock ticks. A clock tick is + * currently defined to 1 millisecond. That is, the resolution of + * timers triggered by the runtime system is 1 millisecond. * - * Timeouts kept in an wheel. A timeout is measured relative to the - * current slot (tiw_pos) in the wheel, and inserted at slot - * (tiw_pos + timeout) % TIW_SIZE. Each timeout also has a count - * equal to timeout/TIW_SIZE, which is needed since the time axis - * is wrapped arount the wheel. + * When a timer is set, it is determined at what Erlang monotonic + * time, in clock ticks, it should be triggered. * - * Several slots may be processed in one operation. If the number of - * slots is greater that the wheel size, the wheel is only traversed - * once, + * The 'pos' field of the wheel corresponds to current time of + * the wheel. That is, it corresponds to Erlang monotonic time in + * clock tick time unit. The 'pos' field of the wheel is + * monotonically increased when erts_bump_timers() is called. All + * timers in the wheel that have a time less than or equal to + * 'pos' are triggered by the bump operation. The bump operation + * may however be spread over multiple calls to erts_bump_timers() + * if there are a lots of timers to trigger. * - * The following example shows a time axis where there is one timeout - * at each "tick", and where 1, 2, 3 ... wheel slots are released in - * one operation. The notation "<x" means "release all items with - * counts less than x". + * Each scheduler thread maintains its own timer wheel. The timer + * wheel of a scheduler, however, actually consists of two wheels. + * A soon wheel and a later wheel. + * + * + * -- The Soon Wheel -- + * + * The soon wheel contain timers that should be triggered soon. + * That is, they are soon to be triggered. Each slot in the soon + * wheel is 1 clock tick wide. The number of slots in the soon + * wheel is currently 2¹⁴. That is, it contains timers in the + * range ('pos', 'pos' + 2¹⁴] which corresponds to a bit more + * than 16 seconds. + * + * When the bump operation is started, 'pos' is moved forward to a + * position that corresponds to current Erlang monotonic time. Then + * all timers that are in the range (old 'pos', new 'pos'] are + * triggered. During a bump operation, the soon wheel may contain + * timers in the two, possibly overlapping, ranges (old 'pos', + * old 'pos' + 2¹⁴], and (new 'pos', new 'pos' + 2¹⁴]. This may + * occur even if the bump operation doesn't yield, due to timeout + * callbacks inserting new timers. + * + * + * -- The Later Wheel -- + * + * The later wheel contain timers that are further away from 'pos' + * than the width of the soon timer wheel. That is, currently + * timers further away from 'pos' than 2¹⁴ clock ticks. The width + * of each slot in the later wheel is half the width of the soon + * wheel. That is, each slot is currently 2¹³ clock ticks wide + * which corresponds to about 8 seconds. If three timers of the + * times 'pos' + 17000, 'pos' + 18000, and 'pos' + 19000 are + * inserted, they will all end up in the same slot in the later + * wheel. + * + * The number of slots in the later wheel is currently the same as + * in the soon wheel, i.e. 2¹⁴. That is, one revolution of the later + * wheel currently corresponds to 2¹⁴×2¹³ clock ticks which is + * almost 37 ½ hour. Timers even further away than that are put in + * the later slot identified by their time modulo the size of the later + * wheel. Such timers are however very uncommon. Most timers used + * by the runtime system will utilize the high level timer API. + * The high level timer implementation will not insert timers + * further away then one revolution into the later wheel. It will + * instead keep such timers in a tree of very long timers. The + * high level timer implementation utilize one timer wheel timer + * for the management of this tree of timers. This timer is set to + * the closest timeout in the tree. This timer may however be + * further away than one revolution in the later wheel. + * + * The 'later.pos' field identifies next position in the later wheel. + * 'later.pos' is always increased by the width of a later wheel slot. + * That is, currently 2¹³ clock ticks. When 'pos' is moved (during + * a bump operation) closer to 'later.pos' than the width of a later + * wheel slot, i.e. currently when 'pos' + 2¹³ ≥ 'later.pos', we + * inspect the slot identified by 'later.pos' and then move 'later.pos' + * forward. When inspecting the later slot we move all timers in the + * slot, that are in the soon wheel range, from the later wheel to + * the soon wheel. Timers one or more revolutions of the later wheel + * away are kept in the slot. + * + * During normal operation, timers originally located in the later + * wheel will currently be moved into the soon wheel about 8 to + * 16 seconds before they should be triggered. During extremely + * heavy load, the scheduler might however be heavily delayed, so + * the code must be prepared for situations where time for + * triggering the timer has passed when we inspect the later wheel + * slot, and then trigger the timer immediately. We must also be + * prepared to inspect multiple later wheel slots at once due to the + * delay. + * + * + * -- Slot Management -- + * + * All timers of a slot are placed in a circular double linked + * list. This makes insertion and removal of a timer O(1). + * + * While bumping timers in a slot, we move the circular list + * away from the slot, and refer to it from the 'sentinel' + * field. The list will stay there until we are done with it + * even if the bump operation should yield. The cancel operation + * can remove the timer from this position as well as from the + * slot position by just removing it from the circular double + * linked list that it is in. + * + * -- At Once Slot -- + * + * If a timer is set that has a time earlier or equal to 'pos', + * it is not inserted into the wheel. It is instead inserted, + * into a circular double linked list referred to by the "at + * once" slot. When the bump operation is performed these timers + * will be triggered at once. The circular list of the slot will + * be moved to the 'sentinel' field while bumping these timers + * as when bumping an ordinary wheel slot. A yielding bump + * operation and cancelation of timers is handled the same way + * as if the timer was in a wheel slot. + * + * -- Searching for Next Timeout -- + * + * In order to limit the amount of work needed in order to find + * next timeout, we keep track of total amount of timers in the + * wheels, total amount of timers in the later wheel, total amount + * of timers in soon wheel, and the total amount of timers in + * each range of slots. Each slot range currently contain 512 + * slots. + * + * When next timeout is less than the soon wheel width away we + * determine the exact timeout. Due to the timer counts of + * slot ranges, we currently at most need to search 1024 slots + * in the soon wheel. This besides inspecting slot range counts + * and two slots in the later wheel which potentially might trigger + * timeouts for moving timers from the later wheel to the soon wheel + * earlier than timeouts in the soon wheel. We also keep track + * of latest known minimum timeout position in each wheel which + * makes it possible to avoid scanning from current position + * each time. + * + * When next timeout is further away than the soon wheel width + * we settle for the earliest possible timeout in the first + * non-empty slot range. The further away the next timeout is, the + * more likely it is that the next timeout change before we + * actually get there. That is, a change due to another timer is + * set to an earlier time and/or the timer is cancelled. It is + * therefore in this case no point determining next timeout + * exactly. If the state should not change, we will wake up a bit + * early and do a recalculation of next timeout and eventually + * we will be so close to it that we determine it exactly. * - * Size of wheel: 4 - * - * --|----|----|----|----|----|----|----|----|----|----|----|----|---- - * 0.0 0.1 0.2 0.3 1.0 1.1 1.2 1.3 2.0 2.1 2.2 2.3 3.0 - * - * 1 [ ) - * <1 0.1 0.2 0.3 0.0 1.1 1.2 1.3 1.0 2.1 2.2 2.3 2.0 - * - * 2 [ ) - * <1 <1 0.2 0.3 0.0 0.1 1.2 1.3 1.0 1.1 2.2 2.3 2.0 - * - * 3 [ ) - * <1 <1 <1 0.3 0.0 0.1 0.2 1.3 1.0 1.1 1.2 2.3 2.0 - * - * 4 [ ) - * <1 <1 <1 <1 0.0 0.1 0.2 0.3 1.0 1.1 1.2 1.3 2.0 - * - * 5 [ ) - * <2 <1 <1 <1. 0.1 0.2 0.3 0.0 1.1 1.2 1.3 1.0 - * - * 6 [ ) - * <2 <2 <1 <1. 0.2 0.3 0.0 0.1 1.2 1.3 1.0 - * - * 7 [ ) - * <2 <2 <2 <1. 0.3 0.0 0.1 0.2 1.3 1.0 - * - * 8 [ ) - * <2 <2 <2 <2. 0.0 0.1 0.2 0.3 1.0 - * - * 9 [ ) - * <3 <2 <2 <2. 0.1 0.2 0.3 0.0 - * */ #ifdef HAVE_CONFIG_H @@ -80,8 +180,11 @@ #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" -#define ERTS_MONOTONIC_DAY ERTS_SEC_TO_MONOTONIC(60*60*24) -#define ERTS_CLKTCKS_DAY ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY) +#define ERTS_MAX_CLKTCKS \ + ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_TIME_MAX) + +#define ERTS_CLKTCKS_WEEK \ + ERTS_MONOTONIC_TO_CLKTCKS(ERTS_SEC_TO_MONOTONIC(7*60*60*24)) #ifdef ERTS_ENABLE_LOCK_CHECK #define ASSERT_NO_LOCKED_LOCKS erts_lc_check_exact(NULL, 0) @@ -90,6 +193,10 @@ #endif #if 0 +# define ERTS_TW_HARD_DEBUG +#endif + +#if defined(ERTS_TW_HARD_DEBUG) && !defined(ERTS_TW_DEBUG) # define ERTS_TW_DEBUG #endif #if defined(DEBUG) && !defined(ERTS_TW_DEBUG) @@ -97,16 +204,62 @@ #endif #undef ERTS_TW_ASSERT -#if defined(ERTS_TW_DEBUG) +#if defined(ERTS_TW_DEBUG) # define ERTS_TW_ASSERT(E) ERTS_ASSERT(E) #else # define ERTS_TW_ASSERT(E) ((void) 1) #endif #ifdef ERTS_TW_DEBUG -# define ERTS_TWHEEL_BUMP_YIELD_LIMIT 5 +# define ERTS_TWHEEL_BUMP_YIELD_LIMIT 500 #else -# define ERTS_TWHEEL_BUMP_YIELD_LIMIT 100 +# define ERTS_TWHEEL_BUMP_YIELD_LIMIT 10000 +#endif +#define ERTS_TW_COST_SLOT 1 +#define ERTS_TW_COST_SLOT_MOVE 5 +#define ERTS_TW_COST_TIMEOUT 100 + +/* + * Every slot in the soon wheel is a clock tick (as defined + * by ERTS) wide. A clock tick is currently 1 milli second. + */ + +#define ERTS_TW_SOON_WHEEL_FIRST_SLOT 0 +#define ERTS_TW_SOON_WHEEL_END_SLOT \ + (ERTS_TW_SOON_WHEEL_FIRST_SLOT + ERTS_TW_SOON_WHEEL_SIZE) + +#define ERTS_TW_SOON_WHEEL_MASK (ERTS_TW_SOON_WHEEL_SIZE-1) + +/* + * Every slot in the later wheel is as wide as half the size + * of the soon wheel. + */ + +#define ERTS_TW_LATER_WHEEL_SHIFT (ERTS_TW_SOON_WHEEL_BITS - 1) +#define ERTS_TW_LATER_WHEEL_SLOT_SIZE \ + ((ErtsMonotonicTime) (1 << ERTS_TW_LATER_WHEEL_SHIFT)) +#define ERTS_TW_LATER_WHEEL_POS_MASK \ + (~((ErtsMonotonicTime) (1 << ERTS_TW_LATER_WHEEL_SHIFT)-1)) + +#define ERTS_TW_LATER_WHEEL_FIRST_SLOT ERTS_TW_SOON_WHEEL_SIZE +#define ERTS_TW_LATER_WHEEL_END_SLOT \ + (ERTS_TW_LATER_WHEEL_FIRST_SLOT + ERTS_TW_LATER_WHEEL_SIZE) + +#define ERTS_TW_LATER_WHEEL_MASK (ERTS_TW_LATER_WHEEL_SIZE-1) + +#define ERTS_TW_SCNT_BITS 9 +#define ERTS_TW_SCNT_SHIFT +#define ERTS_TW_SCNT_SIZE \ + ((ERTS_TW_SOON_WHEEL_SIZE + ERTS_TW_LATER_WHEEL_SIZE) \ + >> ERTS_TW_SCNT_BITS) + +#ifdef __GNUC__ +#if ERTS_TW_SOON_WHEEL_BITS < ERTS_TW_SCNT_BITS +# warning Consider larger soon timer wheel +#endif +#if ERTS_TW_SOON_WHEEL_BITS < ERTS_TW_SCNT_BITS +# warning Consider larger later timer wheel +#endif #endif /* Actual interval time chosen by sys_init_time() */ @@ -119,95 +272,360 @@ static int tiw_itime; /* Constant after init */ # define TIW_ITIME tiw_itime #endif +const int etp_tw_soon_wheel_size = ERTS_TW_SOON_WHEEL_SIZE; +const ErtsMonotonicTime etp_tw_soon_wheel_mask = ERTS_TW_SOON_WHEEL_MASK; +const int etp_tw_soon_wheel_first_slot = ERTS_TW_SOON_WHEEL_FIRST_SLOT; + +const int etp_tw_later_wheel_size = ERTS_TW_LATER_WHEEL_SIZE; +const ErtsMonotonicTime etp_tw_later_wheel_slot_size = ERTS_TW_LATER_WHEEL_SLOT_SIZE; +const int etp_tw_later_wheel_shift = ERTS_TW_LATER_WHEEL_SHIFT; +const ErtsMonotonicTime etp_tw_later_wheel_mask = ERTS_TW_LATER_WHEEL_MASK; +const ErtsMonotonicTime etp_tw_later_wheel_pos_mask = ERTS_TW_LATER_WHEEL_POS_MASK; +const int etp_tw_later_wheel_first_slot = ERTS_TW_LATER_WHEEL_FIRST_SLOT; + struct ErtsTimerWheel_ { - ErtsTWheelTimer *w[ERTS_TIW_SIZE]; + ErtsTWheelTimer *slots[1 /* At Once Slot */ + + ERTS_TW_SOON_WHEEL_SIZE /* Soon Wheel Slots */ + + ERTS_TW_LATER_WHEEL_SIZE]; /* Later Wheel Slots */ + ErtsTWheelTimer **w; + Sint scnt[ERTS_TW_SCNT_SIZE]; + Sint bump_scnt[ERTS_TW_SCNT_SIZE]; ErtsMonotonicTime pos; Uint nto; struct { - ErtsTWheelTimer *head; - ErtsTWheelTimer *tail; Uint nto; } at_once; + struct { + ErtsMonotonicTime min_tpos; + Uint nto; + } soon; + struct { + ErtsMonotonicTime min_tpos; + int min_tpos_slot; + ErtsMonotonicTime pos; + Uint nto; + } later; int yield_slot; int yield_slots_left; - int yield_start_pos; ErtsTWheelTimer sentinel; int true_next_timeout_time; + ErtsMonotonicTime next_timeout_pos; ErtsMonotonicTime next_timeout_time; }; -static ERTS_INLINE ErtsMonotonicTime -find_next_timeout(ErtsSchedulerData *esdp, - ErtsTimerWheel *tiw, - int search_all, - ErtsMonotonicTime curr_time, /* When !search_all */ - ErtsMonotonicTime max_search_time) /* When !search_all */ +#define ERTS_TW_SLOT_AT_ONCE (-1) + +#define ERTS_TW_BUMP_LATER_WHEEL(TIW) \ + ((tiw)->pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE >= (TIW)->later.pos) + +static int bump_later_wheel(ErtsTimerWheel *tiw, int *yield_count_p); + +#ifdef ERTS_TW_DEBUG +#define ERTS_TW_DBG_VERIFY_EMPTY_SOON_SLOTS(TIW, TO_POS) \ + dbg_verify_empty_soon_slots((TIW), (TO_POS)) +#define ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(TIW, TO_POS) \ + dbg_verify_empty_later_slots((TIW), (TO_POS)) +void dbg_verify_empty_soon_slots(ErtsTimerWheel *, ErtsMonotonicTime); +void dbg_verify_empty_later_slots(ErtsTimerWheel *, ErtsMonotonicTime); +#else +#define ERTS_TW_DBG_VERIFY_EMPTY_SOON_SLOTS(TIW, TO_POS) +#define ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(TIW, TO_POS) +#endif + +static ERTS_INLINE int +scnt_get_ix(int slot) { - int start_ix, tiw_pos_ix; - ErtsTWheelTimer *p; + return slot >> ERTS_TW_SCNT_BITS; +} + +static ERTS_INLINE void +scnt_inc(Sint *scnt, int slot) +{ + scnt[slot >> ERTS_TW_SCNT_BITS]++; +} + +#ifdef ERTS_TW_HARD_DEBUG + +static ERTS_INLINE void +scnt_ix_inc(Sint *scnt, int six) +{ + scnt[six]++; +} + +#endif + +static ERTS_INLINE void +scnt_dec(Sint *scnt, int slot) +{ + scnt[slot >> ERTS_TW_SCNT_BITS]--; + ERTS_TW_ASSERT(scnt[slot >> ERTS_TW_SCNT_BITS] >= 0); +} + +static ERTS_INLINE void +scnt_ix_dec(Sint *scnt, int six) +{ + scnt[six]--; + ERTS_TW_ASSERT(scnt[six] >= 0); +} + +static ERTS_INLINE void +scnt_wheel_next(int *slotp, int *leftp, ErtsMonotonicTime *posp, + int *sixp, Sint *scnt, int first_slot, + int end_slot, ErtsMonotonicTime slot_sz) +{ + int slot = *slotp; + int left = *leftp; + int ix; + + ERTS_TW_ASSERT(*leftp >= 0); + + left--; + slot++; + if (slot == end_slot) + slot = first_slot; + ix = slot >> ERTS_TW_SCNT_BITS; + + while (!scnt[ix] && left > 0) { + int diff, old_slot = slot; + ix++; + slot = (ix << ERTS_TW_SCNT_BITS); + diff = slot - old_slot; + if (left < diff) { + slot = old_slot + left; + diff = left; + } + if (slot < end_slot) + left -= diff; + else { + left -= end_slot - old_slot; + slot = first_slot; + ix = slot >> ERTS_TW_SCNT_BITS; + } + } + + ERTS_TW_ASSERT(left >= -1); + + if (posp) + *posp += slot_sz * ((ErtsMonotonicTime) (*leftp - left)); + if (sixp) + *sixp = slot >> ERTS_TW_SCNT_BITS; + *leftp = left; + *slotp = slot; +} + + +static ERTS_INLINE void +scnt_soon_wheel_next(int *slotp, int *leftp, ErtsMonotonicTime *posp, + int *sixp, Sint *scnt) +{ + scnt_wheel_next(slotp, leftp, posp, sixp, scnt, + ERTS_TW_SOON_WHEEL_FIRST_SLOT, + ERTS_TW_SOON_WHEEL_END_SLOT, 1); +} + +static ERTS_INLINE void +scnt_later_wheel_next(int *slotp, int *leftp, ErtsMonotonicTime *posp, + int *sixp, Sint *scnt) +{ + scnt_wheel_next(slotp, leftp, posp, sixp, scnt, + ERTS_TW_LATER_WHEEL_FIRST_SLOT, + ERTS_TW_LATER_WHEEL_END_SLOT, + ERTS_TW_LATER_WHEEL_SLOT_SIZE); +} + + +static ERTS_INLINE int +soon_slot(ErtsMonotonicTime soon_pos) +{ + ErtsMonotonicTime slot = soon_pos; + slot &= ERTS_TW_SOON_WHEEL_MASK; + + ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT <= slot); + ERTS_TW_ASSERT(slot < ERTS_TW_SOON_WHEEL_END_SLOT); + + return (int) slot; +} + +static ERTS_INLINE int +later_slot(ErtsMonotonicTime later_pos) +{ + ErtsMonotonicTime slot = later_pos; + slot >>= ERTS_TW_LATER_WHEEL_SHIFT; + slot &= ERTS_TW_LATER_WHEEL_MASK; + slot += ERTS_TW_LATER_WHEEL_FIRST_SLOT; + + ERTS_TW_ASSERT(ERTS_TW_LATER_WHEEL_FIRST_SLOT <= slot); + ERTS_TW_ASSERT(slot < ERTS_TW_LATER_WHEEL_END_SLOT); + + return (int) slot; +} + +#ifdef ERTS_TW_HARD_DEBUG +#define ERTS_HARD_DBG_CHK_WHEELS(TIW, CHK_MIN_TPOS) \ + hrd_dbg_check_wheels((TIW), (CHK_MIN_TPOS)) +static void hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos); +#else +#define ERTS_HARD_DBG_CHK_WHEELS(TIW, CHK_MIN_TPOS) +#endif + +static ErtsMonotonicTime +find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw) +{ + int slot, slots; int true_min_timeout = 0; - ErtsMonotonicTime min_timeout, min_timeout_pos, slot_timeout_pos; + ErtsMonotonicTime min_timeout_pos; + + ERTS_TW_ASSERT(tiw->pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE < tiw->later.pos + && tiw->later.pos <= tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); + + ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); + + ERTS_TW_ASSERT(tiw->yield_slot == ERTS_TW_SLOT_INACTIVE); if (tiw->nto == 0) { /* no timeouts in wheel */ - if (!search_all) - min_timeout_pos = tiw->pos; - else { - curr_time = erts_get_monotonic_time(esdp); - tiw->pos = min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); - } - min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY); - goto found_next; + ErtsMonotonicTime curr_time = erts_get_monotonic_time(esdp); + tiw->pos = min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); + tiw->later.pos = min_timeout_pos + ERTS_TW_SOON_WHEEL_SIZE; + tiw->later.pos &= ERTS_TW_LATER_WHEEL_POS_MASK; + min_timeout_pos += ERTS_CLKTCKS_WEEK; + goto done; } - slot_timeout_pos = min_timeout_pos = tiw->pos; - if (search_all) - min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY); - else - min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + max_search_time); + ERTS_TW_ASSERT(tiw->soon.nto || tiw->later.nto); - start_ix = tiw_pos_ix = (int) (tiw->pos & (ERTS_TIW_SIZE-1)); + if (!tiw->soon.nto) { + ErtsMonotonicTime tpos, min_tpos; - do { - if (++slot_timeout_pos >= min_timeout_pos) - break; - - p = tiw->w[tiw_pos_ix]; - - if (p) { - ErtsTWheelTimer *end = p; - - do { - ErtsMonotonicTime timeout_pos; - timeout_pos = p->timeout_pos; - if (min_timeout_pos > timeout_pos) { - true_min_timeout = 1; - min_timeout_pos = timeout_pos; - if (min_timeout_pos <= slot_timeout_pos) - goto found_next; - } - p = p->next; - } while (p != end); - } + /* Search later wheel... */ + + min_tpos = tiw->later.min_tpos & ERTS_TW_LATER_WHEEL_POS_MASK; + + if (min_tpos <= tiw->later.pos) { + tpos = tiw->later.pos; + slots = ERTS_TW_LATER_WHEEL_SIZE; + } + else { + ErtsMonotonicTime tmp; + /* Don't inspect slots we know are empty... */ + tmp = min_tpos - tiw->later.pos; + tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + if (tmp >= ERTS_TW_LATER_WHEEL_SIZE) { + /* Timeout more than one revolution ahead... */ + + /* Pre-timeout for move from later to soon wheel... */ + min_timeout_pos = min_tpos - ERTS_TW_LATER_WHEEL_SLOT_SIZE; + goto done; + } + tpos = min_tpos; + ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, min_tpos); + slots = ERTS_TW_LATER_WHEEL_SIZE - ((int) tmp); + } + + slot = later_slot(tpos); + + /* + * We never search for an exact timeout in the + * later wheel, but instead settle for the first + * scnt range used. + */ + if (tiw->w[slot]) + true_min_timeout = 1; + else + scnt_later_wheel_next(&slot, &slots, &tpos, NULL, tiw->scnt); + + tiw->later.min_tpos = tpos; + tiw->later.min_tpos_slot = slot; + ERTS_TW_ASSERT(slot == later_slot(tpos)); + + /* Pre-timeout for move from later to soon wheel... */ + tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + min_timeout_pos = tpos; + } + else { + ErtsMonotonicTime tpos; + /* Search soon wheel... */ + + min_timeout_pos = tiw->pos + ERTS_TW_SOON_WHEEL_SIZE; + + /* + * Besides inspecting the soon wheel we + * may also have to inspect two slots in the + * later wheel which potentially can trigger + * timeouts before timeouts in soon wheel... + */ + if (tiw->later.min_tpos > (tiw->later.pos + + 2*ERTS_TW_LATER_WHEEL_SLOT_SIZE)) { + ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS( + tiw, 2*ERTS_TW_LATER_WHEEL_SLOT_SIZE); + } + else { + int fslot; + tpos = tiw->later.pos; + tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + fslot = later_slot(tiw->later.pos); + if (tiw->w[fslot]) + min_timeout_pos = tpos; + else { + tpos += ERTS_TW_LATER_WHEEL_SLOT_SIZE; + if (tpos < min_timeout_pos) { + fslot++; + if (fslot == ERTS_TW_LATER_WHEEL_END_SLOT) + fslot = ERTS_TW_LATER_WHEEL_FIRST_SLOT; + if (tiw->w[fslot]) + min_timeout_pos = tpos; + } + } + } + + if (tiw->soon.min_tpos <= tiw->pos) { + tpos = tiw->pos; + slots = ERTS_TW_SOON_WHEEL_SIZE; + } + else { + ErtsMonotonicTime tmp; + /* Don't inspect slots we know are empty... */ + tmp = tiw->soon.min_tpos - tiw->pos; + ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_SIZE > tmp); + ERTS_TW_DBG_VERIFY_EMPTY_SOON_SLOTS(tiw, tiw->soon.min_tpos); + slots = ERTS_TW_SOON_WHEEL_SIZE - ((int) tmp); + tpos = tiw->soon.min_tpos; + } + + slot = soon_slot(tpos); + + /* find next non-empty slot */ + while (tpos < min_timeout_pos) { + if (tiw->w[slot]) { + ERTS_TW_ASSERT(tiw->w[slot]->timeout_pos == tpos); + min_timeout_pos = tpos; + break; + } + scnt_soon_wheel_next(&slot, &slots, &tpos, NULL, tiw->scnt); + } - tiw_pos_ix++; - if (tiw_pos_ix == ERTS_TIW_SIZE) - tiw_pos_ix = 0; - } while (start_ix != tiw_pos_ix); + tiw->soon.min_tpos = min_timeout_pos; + true_min_timeout = 1; + } + +done: { + ErtsMonotonicTime min_timeout; -found_next: + min_timeout = ERTS_CLKTCKS_TO_MONOTONIC(min_timeout_pos); + tiw->next_timeout_pos = min_timeout_pos; + tiw->next_timeout_time = min_timeout; + tiw->true_next_timeout_time = true_min_timeout; - min_timeout = ERTS_CLKTCKS_TO_MONOTONIC(min_timeout_pos); - tiw->next_timeout_time = min_timeout; - tiw->true_next_timeout_time = true_min_timeout; + ERTS_HARD_DBG_CHK_WHEELS(tiw, 1); - return min_timeout; + return min_timeout; + } } static ERTS_INLINE void insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p) { - ERTS_TW_ASSERT(slot >= 0); - ERTS_TW_ASSERT(slot < ERTS_TIW_SIZE); + ERTS_TW_ASSERT(ERTS_TW_SLOT_AT_ONCE <= slot + && slot < ERTS_TW_LATER_WHEEL_END_SLOT); p->slot = slot; if (!tiw->w[slot]) { tiw->w[slot] = p; @@ -223,55 +641,89 @@ insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p) prev->next = p; next->prev = p; } + if (slot == ERTS_TW_SLOT_AT_ONCE) + tiw->at_once.nto++; + else { + ErtsMonotonicTime tpos = p->timeout_pos; + if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) { + ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); + tiw->soon.nto++; + if (tiw->soon.min_tpos > tpos) + tiw->soon.min_tpos = tpos; + } + else { + ERTS_TW_ASSERT(p->timeout_pos >= tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); + tiw->later.nto++; + if (tiw->later.min_tpos > tpos) { + tiw->later.min_tpos = tpos; + tiw->later.min_tpos_slot = slot; + } + } + scnt_inc(tiw->scnt, slot); + } } static ERTS_INLINE void remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) { int slot = p->slot; - ERTS_TW_ASSERT(slot != ERTS_TWHEEL_SLOT_INACTIVE); - - if (slot >= 0) { - /* - * Timer in wheel or in circular - * list of timers currently beeing - * triggered (referred by sentinel). - */ - ERTS_TW_ASSERT(slot < ERTS_TIW_SIZE); - - if (p->next == p) { - ERTS_TW_ASSERT(tiw->w[slot] == p); - tiw->w[slot] = NULL; - } - else { - if (tiw->w[slot] == p) - tiw->w[slot] = p->next; - p->prev->next = p->next; - p->next->prev = p->prev; - } + int empty_slot; + ERTS_TW_ASSERT(slot != ERTS_TW_SLOT_INACTIVE); + + /* + * Timer is in circular list either referred to + * by at once slot, slot in soon wheel, slot + * in later wheel, or by sentinel (timers currently + * being triggered). + */ + ERTS_TW_ASSERT(ERTS_TW_SLOT_AT_ONCE <= slot + && slot < ERTS_TW_LATER_WHEEL_END_SLOT); + + if (p->next == p) { + /* Cannot be referred by sentinel, i.e. must be referred by slot... */ + ERTS_TW_ASSERT(tiw->w[slot] == p); + tiw->w[slot] = NULL; + empty_slot = 1; } else { - /* Timer in "at once" queue... */ - ERTS_TW_ASSERT(slot == ERTS_TWHEEL_SLOT_AT_ONCE); - if (p->prev) - p->prev->next = p->next; - else { - ERTS_TW_ASSERT(tiw->at_once.head == p); - tiw->at_once.head = p->next; - } - if (p->next) - p->next->prev = p->prev; - else { - ERTS_TW_ASSERT(tiw->at_once.tail == p); - tiw->at_once.tail = p->prev; - } + if (tiw->w[slot] == p) + tiw->w[slot] = p->next; + p->prev->next = p->next; + p->next->prev = p->prev; + empty_slot = 0; + } + if (slot == ERTS_TW_SLOT_AT_ONCE) { ERTS_TW_ASSERT(tiw->at_once.nto > 0); tiw->at_once.nto--; } - - p->slot = ERTS_TWHEEL_SLOT_INACTIVE; - - tiw->nto--; + else { + scnt_dec(tiw->scnt, slot); + if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) { + if (empty_slot + && tiw->true_next_timeout_time + && p->timeout_pos == tiw->next_timeout_pos) { + tiw->true_next_timeout_time = 0; + } + if (--tiw->soon.nto == 0) + tiw->soon.min_tpos = ERTS_MAX_CLKTCKS; + } + else { + if (empty_slot + && tiw->true_next_timeout_time + && tiw->later.min_tpos_slot == slot) { + ErtsMonotonicTime tpos = tiw->later.min_tpos; + tpos &= ERTS_TW_LATER_WHEEL_POS_MASK; + tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + if (tpos == tiw->next_timeout_pos) + tiw->true_next_timeout_time = 0; + } + if (--tiw->later.nto == 0) { + tiw->later.min_tpos = ERTS_MAX_CLKTCKS; + tiw->later.min_tpos_slot = ERTS_TW_LATER_WHEEL_END_SLOT; + } + } + } + p->slot = ERTS_TW_SLOT_INACTIVE; } ErtsMonotonicTime @@ -280,58 +732,26 @@ erts_check_next_timeout_time(ErtsSchedulerData *esdp) ErtsTimerWheel *tiw = esdp->timer_wheel; ErtsMonotonicTime time; ERTS_MSACC_DECLARE_CACHE_X(); + ERTS_TW_ASSERT(tiw->next_timeout_time + == ERTS_CLKTCKS_TO_MONOTONIC(tiw->next_timeout_pos)); if (tiw->true_next_timeout_time) - return tiw->next_timeout_time; + return tiw->next_timeout_time; /* known timeout... */ + if (tiw->next_timeout_pos > tiw->pos + ERTS_TW_SOON_WHEEL_SIZE) + return tiw->next_timeout_time; /* sufficiently later away... */ ERTS_MSACC_PUSH_AND_SET_STATE_CACHED_X(ERTS_MSACC_STATE_TIMERS); - time = find_next_timeout(esdp, tiw, 1, 0, 0); + time = find_next_timeout(esdp, tiw); ERTS_MSACC_POP_STATE_M_X(); return time; } -#ifndef ERTS_TW_DEBUG -#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) ((void) 0) -#else -#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) debug_check_safe_to_skip_to((TIW), (TO)) -static void -debug_check_safe_to_skip_to(ErtsTimerWheel *tiw, ErtsMonotonicTime skip_to_pos) -{ - int slots, ix; - ErtsTWheelTimer *tmr; - ErtsMonotonicTime tmp; - - ix = (int) (tiw->pos & (ERTS_TIW_SIZE-1)); - tmp = skip_to_pos - tiw->pos; - ERTS_TW_ASSERT(tmp >= 0); - if (tmp < (ErtsMonotonicTime) ERTS_TIW_SIZE) - slots = (int) tmp; - else - slots = ERTS_TIW_SIZE; - - while (slots > 0) { - tmr = tiw->w[ix]; - if (tmr) { - ErtsTWheelTimer *end = tmr; - do { - ERTS_TW_ASSERT(tmr->timeout_pos > skip_to_pos); - tmr = tmr->next; - } while (tmr != end); - } - ix++; - if (ix == ERTS_TIW_SIZE) - ix = 0; - slots--; - } -} -#endif - static ERTS_INLINE void timeout_timer(ErtsTWheelTimer *p) { ErlTimeoutProc timeout; void *arg; - p->slot = ERTS_TWHEEL_SLOT_INACTIVE; - timeout = p->u.func.timeout; - arg = p->u.func.arg; + p->slot = ERTS_TW_SLOT_INACTIVE; + timeout = p->timeout; + arg = p->arg; (*timeout)(arg); ASSERT_NO_LOCKED_LOCKS; } @@ -339,73 +759,108 @@ timeout_timer(ErtsTWheelTimer *p) void erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) { - int tiw_pos_ix, slots, yielded_slot_restarted, yield_count; - ErtsMonotonicTime bump_to, tmp_slots, old_pos; + int slot, restarted, yield_count, slots, scnt_ix; + ErtsMonotonicTime bump_to; + Sint *scnt, *bump_scnt; ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); yield_count = ERTS_TWHEEL_BUMP_YIELD_LIMIT; + scnt = &tiw->scnt[0]; + bump_scnt = &tiw->bump_scnt[0]; + /* * In order to be fair we always continue with work * where we left off when restarting after a yield. */ - if (tiw->yield_slot >= 0) { - yielded_slot_restarted = 1; - tiw_pos_ix = tiw->yield_slot; - slots = tiw->yield_slots_left; + slot = tiw->yield_slot; + restarted = slot != ERTS_TW_SLOT_INACTIVE; + if (restarted) { bump_to = tiw->pos; - old_pos = tiw->yield_start_pos; - goto restart_yielded_slot; + if (slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) + goto restart_yielded_later_slot; + tiw->yield_slot = ERTS_TW_SLOT_INACTIVE; + if (slot == ERTS_TW_SLOT_AT_ONCE) + goto restart_yielded_at_once_slot; + scnt_ix = scnt_get_ix(slot); + slots = tiw->yield_slots_left; + ASSERT(0 <= slots && slots <= ERTS_TW_SOON_WHEEL_SIZE); + goto restart_yielded_soon_slot; } do { - yielded_slot_restarted = 0; - + restarted = 0; bump_to = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); + tiw->true_next_timeout_time = 1; + tiw->next_timeout_pos = bump_to; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to); while (1) { ErtsTWheelTimer *p; - old_pos = tiw->pos; - if (tiw->nto == 0) { empty_wheel: - ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, bump_to); + ERTS_TW_DBG_VERIFY_EMPTY_SOON_SLOTS(tiw, bump_to); + ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, bump_to); tiw->true_next_timeout_time = 0; - tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY; + tiw->next_timeout_pos = bump_to + ERTS_CLKTCKS_WEEK; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(tiw->next_timeout_pos);; tiw->pos = bump_to; - tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; + tiw->later.pos = bump_to + ERTS_TW_SOON_WHEEL_SIZE; + tiw->later.pos &= ERTS_TW_LATER_WHEEL_POS_MASK; + tiw->yield_slot = ERTS_TW_SLOT_INACTIVE; ERTS_MSACC_POP_STATE_M_X(); return; } - p = tiw->at_once.head; - while (p) { - if (--yield_count <= 0) { - ERTS_TW_ASSERT(tiw->nto > 0); - ERTS_TW_ASSERT(tiw->at_once.nto > 0); - tiw->yield_slot = ERTS_TWHEEL_SLOT_AT_ONCE; - tiw->true_next_timeout_time = 1; - tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(old_pos); - ERTS_MSACC_POP_STATE_M_X(); - return; - } + p = tiw->w[ERTS_TW_SLOT_AT_ONCE]; + + if (p) { + + if (p->next == p) { + ERTS_TW_ASSERT(tiw->sentinel.next == &tiw->sentinel); + ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel); + } + else { + tiw->sentinel.next = p->next; + tiw->sentinel.prev = p->prev; + tiw->sentinel.next->prev = &tiw->sentinel; + tiw->sentinel.prev->next = &tiw->sentinel; + } + tiw->w[ERTS_TW_SLOT_AT_ONCE] = NULL; + + while (1) { + ERTS_TW_ASSERT(tiw->nto > 0); + ERTS_TW_ASSERT(tiw->at_once.nto > 0); + tiw->nto--; + tiw->at_once.nto--; + + timeout_timer(p); + + yield_count -= ERTS_TW_COST_TIMEOUT; - ERTS_TW_ASSERT(tiw->nto > 0); - ERTS_TW_ASSERT(tiw->at_once.nto > 0); - tiw->nto--; - tiw->at_once.nto--; - tiw->at_once.head = p->next; - if (p->next) - p->next->prev = NULL; - else - tiw->at_once.tail = NULL; + restart_yielded_at_once_slot: - timeout_timer(p); + p = tiw->sentinel.next; + if (p == &tiw->sentinel) { + ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel); + break; + } + + if (yield_count <= 0) { + ERTS_TW_ASSERT(tiw->nto > 0); + ERTS_TW_ASSERT(tiw->at_once.nto > 0); + tiw->yield_slot = ERTS_TW_SLOT_AT_ONCE; + ERTS_MSACC_POP_STATE_M_X(); + return; /* Yield! */ + } + + tiw->sentinel.next = p->next; + p->next->prev = &tiw->sentinel; + } - p = tiw->at_once.head; } if (tiw->pos >= bump_to) { @@ -416,39 +871,66 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) if (tiw->nto == 0) goto empty_wheel; - if (tiw->true_next_timeout_time) { - ErtsMonotonicTime skip_until_pos; + /* + * Save slot counts in bump operation local + * array. + * + * The amount of timers to trigger (or move) + * will only decrease from now until we have + * completed this bump operation (even if we + * yield in the middle of it). + * + * The amount of timers in the wheels may + * however increase due to timers being set + * by timeout callbacks. + */ + sys_memcpy((void *) bump_scnt, (void *) scnt, + sizeof(Sint) * ERTS_TW_SCNT_SIZE); + + if (tiw->soon.min_tpos > tiw->pos) { + ErtsMonotonicTime skip_until_pos = tiw->soon.min_tpos; + /* * No need inspecting slots where we know no timeouts * to trigger should reside. */ - skip_until_pos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time); if (skip_until_pos > bump_to) skip_until_pos = bump_to; skip_until_pos--; if (skip_until_pos > tiw->pos) { - ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, skip_until_pos); - + ERTS_TW_DBG_VERIFY_EMPTY_SOON_SLOTS(tiw, skip_until_pos); tiw->pos = skip_until_pos; } } - tiw_pos_ix = (int) ((tiw->pos+1) & (ERTS_TIW_SIZE-1)); - tmp_slots = (bump_to - tiw->pos); - if (tmp_slots < (ErtsMonotonicTime) ERTS_TIW_SIZE) - slots = (int) tmp_slots; - else - slots = ERTS_TIW_SIZE; + { + ErtsMonotonicTime tmp_slots = bump_to - tiw->pos; + tmp_slots = (bump_to - tiw->pos); + if (tmp_slots < ERTS_TW_SOON_WHEEL_SIZE) + slots = (int) tmp_slots; + else + slots = ERTS_TW_SOON_WHEEL_SIZE; + } + slot = soon_slot(tiw->pos+1); tiw->pos = bump_to; + tiw->next_timeout_pos = bump_to; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to); + + scnt_ix = scnt_get_ix(slot); + + /* Timeout timers in soon wheel */ while (slots > 0) { - p = tiw->w[tiw_pos_ix]; + yield_count -= ERTS_TW_COST_SLOT; + + p = tiw->w[slot]; if (p) { + /* timeout callback need tiw->pos to be up to date */ if (p->next == p) { ERTS_TW_ASSERT(tiw->sentinel.next == &tiw->sentinel); ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel); @@ -459,22 +941,28 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) tiw->sentinel.next->prev = &tiw->sentinel; tiw->sentinel.prev->next = &tiw->sentinel; } - tiw->w[tiw_pos_ix] = NULL; + tiw->w[slot] = NULL; while (1) { - if (p->timeout_pos > bump_to) { - /* Very unusual case... */ - ++yield_count; - insert_timer_into_slot(tiw, tiw_pos_ix, p); - } - else { - /* Normal case... */ - timeout_timer(p); - tiw->nto--; - } - - restart_yielded_slot: + ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT <= p->slot + && p->slot < ERTS_TW_SOON_WHEEL_END_SLOT); + if (--tiw->soon.nto == 0) + tiw->soon.min_tpos = ERTS_MAX_CLKTCKS; + scnt_ix_dec(scnt, scnt_ix); + if (p->timeout_pos <= bump_to) { + timeout_timer(p); + tiw->nto--; + scnt_ix_dec(bump_scnt, scnt_ix); + yield_count -= ERTS_TW_COST_TIMEOUT; + } + else { + /* uncommon case */ + insert_timer_into_slot(tiw, slot, p); + yield_count -= ERTS_TW_COST_SLOT_MOVE; + } + + restart_yielded_soon_slot: p = tiw->sentinel.next; if (p == &tiw->sentinel) { @@ -482,12 +970,9 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) break; } - if (--yield_count <= 0) { - tiw->true_next_timeout_time = 1; - tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(old_pos); - tiw->yield_slot = tiw_pos_ix; + if (yield_count <= 0) { + tiw->yield_slot = slot; tiw->yield_slots_left = slots; - tiw->yield_start_pos = old_pos; ERTS_MSACC_POP_STATE_M_X(); return; /* Yield! */ } @@ -496,24 +981,166 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) p->next->prev = &tiw->sentinel; } } - tiw_pos_ix++; - if (tiw_pos_ix == ERTS_TIW_SIZE) - tiw_pos_ix = 0; - slots--; + + scnt_soon_wheel_next(&slot, &slots, NULL, &scnt_ix, bump_scnt); } + + if (ERTS_TW_BUMP_LATER_WHEEL(tiw)) { + restart_yielded_later_slot: + if (bump_later_wheel(tiw, &yield_count)) + return; /* Yield! */ + } } - } while (yielded_slot_restarted); + } while (restarted); - tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; tiw->true_next_timeout_time = 0; - tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY; + ERTS_TW_ASSERT(tiw->next_timeout_pos == bump_to); - /* Search at most two seconds ahead... */ - (void) find_next_timeout(NULL, tiw, 0, curr_time, ERTS_SEC_TO_MONOTONIC(2)); + (void) find_next_timeout(NULL, tiw); ERTS_MSACC_POP_STATE_M_X(); } +static int +bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) +{ + ErtsMonotonicTime cpos = tiw->pos; + ErtsMonotonicTime later_pos = tiw->later.pos; + int ycount = *ycount_p; + int slots, fslot, scnt_ix; + Sint *scnt, *bump_scnt; + + scnt = &tiw->scnt[0]; + bump_scnt = &tiw->bump_scnt[0]; + + ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); + + if (tiw->yield_slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) { + fslot = tiw->yield_slot; + scnt_ix = scnt_get_ix(fslot); + slots = tiw->yield_slots_left; + ASSERT(0 <= slots && slots <= ERTS_TW_LATER_WHEEL_SIZE); + tiw->yield_slot = ERTS_TW_SLOT_INACTIVE; + goto restart_yielded_slot; + } + else { + ErtsMonotonicTime end_later_pos, tmp_slots, min_tpos; + + min_tpos = tiw->later.min_tpos & ERTS_TW_LATER_WHEEL_POS_MASK; + end_later_pos = cpos + ERTS_TW_SOON_WHEEL_SIZE; + end_later_pos &= ERTS_TW_LATER_WHEEL_POS_MASK; + + /* Skip known empty slots... */ + if (min_tpos > later_pos) { + if (min_tpos > end_later_pos) { + ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, end_later_pos); + tiw->later.pos = end_later_pos; + goto done; + } + later_pos = min_tpos; + ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, later_pos); + } + + tmp_slots = end_later_pos; + tmp_slots -= later_pos; + tmp_slots /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + if (tmp_slots < ERTS_TW_LATER_WHEEL_SIZE) + slots = (int) tmp_slots; + else + slots = ERTS_TW_LATER_WHEEL_SIZE; + + fslot = later_slot(later_pos); + scnt_ix = scnt_get_ix(fslot); + + tiw->later.pos = end_later_pos; + } + + while (slots > 0) { + ErtsTWheelTimer *p; + + ycount -= ERTS_TW_COST_SLOT; + + p = tiw->w[fslot]; + + if (p) { + + if (p->next == p) { + ERTS_TW_ASSERT(tiw->sentinel.next == &tiw->sentinel); + ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel); + } + else { + tiw->sentinel.next = p->next; + tiw->sentinel.prev = p->prev; + tiw->sentinel.next->prev = &tiw->sentinel; + tiw->sentinel.prev->next = &tiw->sentinel; + } + tiw->w[fslot] = NULL; + + while (1) { + ErtsMonotonicTime tpos = p->timeout_pos; + + ERTS_TW_ASSERT(p->slot == fslot); + + if (--tiw->later.nto == 0) { + tiw->later.min_tpos = ERTS_MAX_CLKTCKS; + tiw->later.min_tpos_slot = ERTS_TW_LATER_WHEEL_END_SLOT; + } + scnt_ix_dec(scnt, scnt_ix); + + if (tpos >= tiw->later.pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE) { + /* keep in later slot; very uncommon... */ + insert_timer_into_slot(tiw, fslot, p); + ycount -= ERTS_TW_COST_SLOT_MOVE; + } + else { + scnt_ix_dec(bump_scnt, scnt_ix); + ERTS_TW_ASSERT(tpos < cpos + ERTS_TW_SOON_WHEEL_SIZE); + if (tpos > cpos) { + /* move into soon wheel */ + insert_timer_into_slot(tiw, soon_slot(tpos), p); + ycount -= ERTS_TW_COST_SLOT_MOVE; + } + else { + /* trigger at once */ + timeout_timer(p); + tiw->nto--; + ycount -= ERTS_TW_COST_TIMEOUT; + } + } + + restart_yielded_slot: + + p = tiw->sentinel.next; + if (p == &tiw->sentinel) { + ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel); + break; + } + + if (ycount < 0) { + tiw->yield_slot = fslot; + tiw->yield_slots_left = slots; + *ycount_p = 0; + ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); + return 1; /* Yield! */ + } + + tiw->sentinel.next = p->next; + p->next->prev = &tiw->sentinel; + } + } + + scnt_later_wheel_next(&fslot, &slots, NULL, &scnt_ix, bump_scnt); + } + +done: + + ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); + + *ycount_p = ycount; + + return 0; +} + Uint erts_timer_wheel_memory_size(void) { @@ -526,25 +1153,51 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp) ErtsMonotonicTime mtime; int i; ErtsTimerWheel *tiw; + + /* Some compile time sanity checks... */ + + /* Slots... */ + ERTS_CT_ASSERT(ERTS_TW_SLOT_AT_ONCE == -1); + ERTS_CT_ASSERT(ERTS_TW_SLOT_INACTIVE < ERTS_TW_SLOT_AT_ONCE); + ERTS_CT_ASSERT(ERTS_TW_SLOT_AT_ONCE + 1 == ERTS_TW_SOON_WHEEL_FIRST_SLOT); + ERTS_CT_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT < ERTS_TW_SOON_WHEEL_END_SLOT); + ERTS_CT_ASSERT(ERTS_TW_SOON_WHEEL_END_SLOT == ERTS_TW_LATER_WHEEL_FIRST_SLOT); + ERTS_CT_ASSERT(ERTS_TW_LATER_WHEEL_FIRST_SLOT < ERTS_TW_LATER_WHEEL_END_SLOT); + + /* Both wheel sizes should be a powers of 2 */ + ERTS_CT_ASSERT(ERTS_TW_SOON_WHEEL_SIZE + && !(ERTS_TW_SOON_WHEEL_SIZE & (ERTS_TW_SOON_WHEEL_SIZE-1))); + ERTS_CT_ASSERT(ERTS_TW_LATER_WHEEL_SIZE + && !(ERTS_TW_LATER_WHEEL_SIZE & (ERTS_TW_LATER_WHEEL_SIZE-1))); + tiw = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_TIMER_WHEEL, sizeof(ErtsTimerWheel)); - for(i = 0; i < ERTS_TIW_SIZE; i++) + tiw->w = &tiw->slots[1]; + for(i = ERTS_TW_SLOT_AT_ONCE; i < ERTS_TW_LATER_WHEEL_END_SLOT; i++) tiw->w[i] = NULL; + for (i = 0; i < ERTS_TW_SCNT_SIZE; i++) + tiw->scnt[i] = 0; + mtime = erts_get_monotonic_time(esdp); tiw->pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime); tiw->nto = 0; - tiw->at_once.head = NULL; - tiw->at_once.tail = NULL; tiw->at_once.nto = 0; - tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; + tiw->soon.min_tpos = ERTS_MAX_CLKTCKS; + tiw->soon.nto = 0; + tiw->later.min_tpos = ERTS_MAX_CLKTCKS; + tiw->later.min_tpos_slot = ERTS_TW_LATER_WHEEL_END_SLOT; + tiw->later.pos = tiw->pos + ERTS_TW_SOON_WHEEL_SIZE; + tiw->later.pos &= ERTS_TW_LATER_WHEEL_POS_MASK; + tiw->later.nto = 0; + tiw->yield_slot = ERTS_TW_SLOT_INACTIVE; tiw->true_next_timeout_time = 0; - tiw->next_timeout_time = mtime + ERTS_MONOTONIC_DAY; + tiw->next_timeout_pos = tiw->pos + ERTS_CLKTCKS_WEEK; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(tiw->next_timeout_pos); tiw->sentinel.next = &tiw->sentinel; tiw->sentinel.prev = &tiw->sentinel; - tiw->sentinel.u.func.timeout = NULL; - tiw->sentinel.u.func.cancel = NULL; - tiw->sentinel.u.func.arg = NULL; + tiw->sentinel.timeout = NULL; + tiw->sentinel.arg = NULL; return tiw; } @@ -577,53 +1230,56 @@ erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode) void erts_twheel_set_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p, ErlTimeoutProc timeout, - ErlCancelProc cancel, void *arg, - ErtsMonotonicTime timeout_pos) + void *arg, ErtsMonotonicTime timeout_pos) { - ErtsMonotonicTime timeout_time; + int slot; ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); - p->u.func.timeout = timeout; - p->u.func.cancel = cancel; - p->u.func.arg = arg; + p->timeout = timeout; + p->arg = arg; + + ERTS_TW_ASSERT(p->slot == ERTS_TW_SLOT_INACTIVE); - ERTS_TW_ASSERT(p->slot == ERTS_TWHEEL_SLOT_INACTIVE); + tiw->nto++; + /* calculate slot */ if (timeout_pos <= tiw->pos) { - tiw->nto++; - tiw->at_once.nto++; - p->next = NULL; - p->prev = tiw->at_once.tail; - if (tiw->at_once.tail) { - ERTS_TW_ASSERT(tiw->at_once.head); - tiw->at_once.tail->next = p; - } - else { - ERTS_TW_ASSERT(!tiw->at_once.head); - tiw->at_once.head = p; - } - tiw->at_once.tail = p; - p->timeout_pos = tiw->pos; - p->slot = ERTS_TWHEEL_SLOT_AT_ONCE; - timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(tiw->pos); + /* at once */ + p->timeout_pos = timeout_pos = tiw->pos; + slot = ERTS_TW_SLOT_AT_ONCE; + } + else if (timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE) { + /* soon wheel */ + p->timeout_pos = timeout_pos; + slot = soon_slot(timeout_pos); + if (tiw->soon.min_tpos > timeout_pos) + tiw->soon.min_tpos = timeout_pos; } else { - int slot; - - /* calculate slot */ - slot = (int) (timeout_pos & (ERTS_TIW_SIZE-1)); - - insert_timer_into_slot(tiw, slot, p); - - tiw->nto++; - - timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos); - p->timeout_pos = timeout_pos; + /* later wheel */ + p->timeout_pos = timeout_pos; + slot = later_slot(timeout_pos); + + /* + * Next timeout due to this timeout + * should be in good time before the + * actual timeout (one later wheel slot + * size). This, in order to move it + * from the later wheel to the soon + * wheel. + */ + timeout_pos &= ERTS_TW_LATER_WHEEL_POS_MASK; + timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; } - if (timeout_time < tiw->next_timeout_time) { + insert_timer_into_slot(tiw, slot, p); + + if (timeout_pos <= tiw->next_timeout_pos) { tiw->true_next_timeout_time = 1; - tiw->next_timeout_time = timeout_time; + if (timeout_pos < tiw->next_timeout_pos) { + tiw->next_timeout_pos = timeout_pos; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos); + } } ERTS_MSACC_POP_STATE_M_X(); } @@ -631,15 +1287,10 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw, void erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) { - if (p->slot != ERTS_TWHEEL_SLOT_INACTIVE) { - ErlCancelProc cancel; - void *arg; + if (p->slot != ERTS_TW_SLOT_INACTIVE) { ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); remove_timer(tiw, p); - cancel = p->u.func.cancel; - arg = p->u.func.arg; - if (cancel) - (*cancel)(arg); + tiw->nto--; ERTS_MSACC_POP_STATE_M_X(); } } @@ -657,22 +1308,17 @@ erts_twheel_debug_foreach(ErtsTimerWheel *tiw, tmr = tiw->sentinel.next; while (tmr != &tiw->sentinel) { - if (tmr->u.func.timeout == tclbk) - (*func)(arg, tmr->timeout_pos, tmr->u.func.arg); + if (tmr->timeout == tclbk) + (*func)(arg, tmr->timeout_pos, tmr->arg); tmr = tmr->next; } - for (tmr = tiw->at_once.head; tmr; tmr = tmr->next) { - if (tmr->u.func.timeout == tclbk) - (*func)(arg, tmr->timeout_pos, tmr->u.func.arg); - } - - for (ix = 0; ix < ERTS_TIW_SIZE; ix++) { + for (ix = ERTS_TW_SLOT_AT_ONCE; ix < ERTS_TW_LATER_WHEEL_END_SLOT; ix++) { tmr = tiw->w[ix]; if (tmr) { do { - if (tmr->u.func.timeout == tclbk) - (*func)(arg, tmr->timeout_pos, tmr->u.func.arg); + if (tmr->timeout == tclbk) + (*func)(arg, tmr->timeout_pos, tmr->arg); tmr = tmr->next; } while (tmr != tiw->w[ix]); } @@ -680,35 +1326,206 @@ erts_twheel_debug_foreach(ErtsTimerWheel *tiw, } #ifdef ERTS_TW_DEBUG -void erts_p_slpq(void) + +void +dbg_verify_empty_soon_slots(ErtsTimerWheel *tiw, ErtsMonotonicTime to_pos) { - erts_printf("Not yet implemented...\n"); -#if 0 - ErtsMonotonicTime current_time = erts_get_monotonic_time(NULL); - int i; - ErtsTWheelTimer* p; - - /* print the whole wheel, starting at the current position */ - erts_printf("\ncurrent time = %bps tiw_pos = %d tiw_nto %d\n", - current_time, tiw->pos, tiw->nto); - i = tiw->pos; - if (tiw->w[i] != NULL) { - erts_printf("%d:\n", i); - for(p = tiw->w[i]; p != NULL; p = p->next) { - erts_printf(" (timeout time %bps, slot %d)\n", - ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos), - p->slot); - } + int ix; + ErtsMonotonicTime tmp; + + ix = soon_slot(tiw->pos); + tmp = to_pos; + if (tmp > tiw->pos) { + int slots; + tmp -= tiw->pos; + ERTS_TW_ASSERT(tmp > 0); + if (tmp < (ErtsMonotonicTime) ERTS_TW_SOON_WHEEL_SIZE) + slots = (int) tmp; + else + slots = ERTS_TW_SOON_WHEEL_SIZE; + + while (slots > 0) { + ERTS_TW_ASSERT(!tiw->w[ix]); + ix++; + if (ix == ERTS_TW_SOON_WHEEL_END_SLOT) + ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; + slots--; + } + } +} + +void +dbg_verify_empty_later_slots(ErtsTimerWheel *tiw, ErtsMonotonicTime to_pos) +{ + int ix; + ErtsMonotonicTime tmp; + + ix = later_slot(tiw->later.pos); + tmp = to_pos; + tmp &= ERTS_TW_LATER_WHEEL_POS_MASK; + if (tmp > tiw->later.pos) { + ErtsMonotonicTime pos_min; + int slots; + tmp -= tiw->later.pos; + tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + ERTS_TW_ASSERT(tmp > 0); + + pos_min = tiw->later.pos; + + if (tmp < (ErtsMonotonicTime) ERTS_TW_LATER_WHEEL_SIZE) + slots = (int) tmp; + else { + pos_min += ((tmp / ERTS_TW_LATER_WHEEL_SIZE) + * ERTS_TW_LATER_WHEEL_SLOT_SIZE); + slots = ERTS_TW_LATER_WHEEL_SIZE; + } + + while (slots > 0) { + ErtsTWheelTimer *tmr = tiw->w[ix]; + pos_min += ERTS_TW_LATER_WHEEL_SLOT_SIZE; + if (tmr) { + ErtsTWheelTimer *end = tmr; + do { + ERTS_TW_ASSERT(tmr->timeout_pos >= pos_min); + tmr = tmr->next; + } while (tmr != end); + } + ix++; + if (ix == ERTS_TW_LATER_WHEEL_END_SLOT) + ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT; + slots--; + } + } +} + +#endif /* ERTS_TW_DEBUG */ + +#ifdef ERTS_TW_HARD_DEBUG + +static void +hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos) +{ + int ix, six, soon_tmo, later_tmo, at_once_tmo, + scnt_slot, scnt_slots, scnt_six; + ErtsMonotonicTime min_tpos; + Sint scnt[ERTS_TW_SCNT_SIZE]; + ErtsTWheelTimer *p; + + for (six = 0; six < ERTS_TW_SCNT_SIZE; six++) + scnt[six] = 0; + + min_tpos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time); + + at_once_tmo = 0; + p = tiw->w[ERTS_TW_SLOT_AT_ONCE]; + if (p) { + ErtsTWheelTimer *first = p; + do { + at_once_tmo++; + ERTS_TW_ASSERT(p->slot == ERTS_TW_SLOT_AT_ONCE); + ERTS_TW_ASSERT(p->timeout_pos <= tiw->pos); + ERTS_TW_ASSERT(!check_min_tpos || tiw->pos >= min_tpos); + ERTS_TW_ASSERT(p->next->prev == p); + p = p->next; + } while (p != first); } - for(i = ((i+1) & (ERTS_TIW_SIZE-1)); i != (tiw->pos & (ERTS_TIW_SIZE-1)); i = ((i+1) & (ERTS_TIW_SIZE-1))) { - if (tiw->w[i] != NULL) { - erts_printf("%d:\n", i); - for(p = tiw->w[i]; p != NULL; p = p->next) { - erts_printf(" (timeout time %bps, slot %d)\n", - ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos), p->slot); - } - } + + soon_tmo = 0; + scnt_slot = ERTS_TW_SOON_WHEEL_END_SLOT-1; + scnt_slots = ERTS_TW_SOON_WHEEL_SIZE; + scnt_six = 0; + scnt_soon_wheel_next(&scnt_slot, &scnt_slots, + NULL, &scnt_six, tiw->scnt); + for (ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; + ix < ERTS_TW_SOON_WHEEL_END_SLOT; + ix++) { + p = tiw->w[ix]; + six = scnt_get_ix(ix); + ERTS_TW_ASSERT(!p || six == scnt_six); + if (p) { + ErtsTWheelTimer *first = p; + do { + ErtsMonotonicTime tpos = p->timeout_pos; + soon_tmo++; + scnt_ix_inc(scnt, six); + ERTS_TW_ASSERT(p->slot == ix); + ERTS_TW_ASSERT(ix == soon_slot(tpos)); + ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); + ERTS_TW_ASSERT(!check_min_tpos || tpos >= min_tpos); + ERTS_TW_ASSERT(p->next->prev == p); + p = p->next; + } while (p != first); + } + if (ix == scnt_slot) + scnt_soon_wheel_next(&scnt_slot, &scnt_slots, + NULL, &scnt_six, tiw->scnt); } -#endif + + later_tmo = 0; + scnt_slot = ERTS_TW_SOON_WHEEL_END_SLOT-1; + scnt_slots = ERTS_TW_SOON_WHEEL_SIZE; + scnt_six = 0; + scnt_later_wheel_next(&scnt_slot, &scnt_slots, + NULL, &scnt_six, tiw->scnt); + for (ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT; + ix < ERTS_TW_LATER_WHEEL_END_SLOT; + ix++) { + p = tiw->w[ix]; + six = scnt_get_ix(ix); + ERTS_TW_ASSERT(!p || six == scnt_six); + if (p) { + ErtsTWheelTimer *first = p; + six = scnt_get_ix(ix); + do { + ErtsMonotonicTime tpos = p->timeout_pos; + later_tmo++; + scnt_ix_inc(scnt, six); + ERTS_TW_ASSERT(p->slot == ix); + ERTS_TW_ASSERT(later_slot(tpos) == ix); + tpos &= ERTS_TW_LATER_WHEEL_POS_MASK; + tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + ERTS_TW_ASSERT(!check_min_tpos || tpos >= min_tpos); + ERTS_TW_ASSERT(p->next->prev == p); + p = p->next; + } while (p != first); + } + if (ix == scnt_slot) + scnt_later_wheel_next(&scnt_slot, &scnt_slots, + NULL, &scnt_six, tiw->scnt); + } + + if (tiw->yield_slot != ERTS_TW_SLOT_INACTIVE) { + p = tiw->sentinel.next; + ix = tiw->yield_slot; + while (p != &tiw->sentinel) { + ErtsMonotonicTime tpos = p->timeout_pos; + ERTS_TW_ASSERT(ix == p->slot); + if (ix == ERTS_TW_SLOT_AT_ONCE) + at_once_tmo++; + else { + scnt_inc(scnt, ix); + if (ix >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) { + later_tmo++; + ERTS_TW_ASSERT(ix == later_slot(tpos)); + } + else { + soon_tmo++; + ERTS_TW_ASSERT(ix == (tpos & ERTS_TW_SOON_WHEEL_MASK)); + ERTS_TW_ASSERT(tpos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); + } + p = p->next; + } + } + } + + + ERTS_TW_ASSERT(tiw->at_once.nto == at_once_tmo); + ERTS_TW_ASSERT(tiw->soon.nto == soon_tmo); + ERTS_TW_ASSERT(tiw->later.nto == later_tmo); + ERTS_TW_ASSERT(tiw->nto == soon_tmo + later_tmo + at_once_tmo); + + for (six = 0; six < ERTS_TW_SCNT_SIZE; six++) + ERTS_TW_ASSERT(scnt[six] == tiw->scnt[six]); } -#endif /* ERTS_TW_DEBUG */ + +#endif /* ERTS_TW_HARD_DEBUG */ diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 8f3f48f38f..9263798a28 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -3594,7 +3594,7 @@ store_external_or_ref_(Uint **hpp, ErlOffHeap* oh, Eterm ns) ErtsMRefThing *mreft = (ErtsMRefThing *) from_hp; ErtsMagicBinary *mb = mreft->mb; ASSERT(is_magic_ref_thing(from_hp)); - erts_refc_inc(&mb->refc, 2); + erts_refc_inc(&mb->intern.refc, 2); OH_OVERHEAD(oh, mb->orig_size / sizeof(Eterm)); } diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index 0fe5183b42..90114b77f5 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -591,7 +591,7 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) (((unsigned char*) (s))[1] << 8) | \ (((unsigned char*) (s))[0])) -#ifdef HAVE_SYS_UN_H +#if defined(HAVE_SYS_UN_H) || defined(SO_BINDTODEVICE) /* strnlen doesn't exist everywhere */ static size_t my_strnlen(const char *s, size_t maxlen) @@ -602,6 +602,10 @@ static size_t my_strnlen(const char *s, size_t maxlen) return i; } +#endif + +#if defined(HAVE_SYS_UN_H) + /* Check that some character in the buffer != '\0' */ static int is_nonzero(const char *s, size_t n) { @@ -778,6 +782,7 @@ static int is_nonzero(const char *s, size_t n) #define INET_LOPT_TCP_SHOW_ECONNRESET 39 /* tell user about incoming RST */ #define INET_LOPT_LINE_DELIM 40 /* Line delimiting char */ #define INET_OPT_TCLASS 41 /* IPv6 transport class */ +#define INET_OPT_BIND_TO_DEVICE 42 /* get/set network device the socket is bound to */ /* SCTP options: a separate range, from 100: */ #define SCTP_OPT_RTOINFO 100 #define SCTP_OPT_ASSOCINFO 101 @@ -1334,6 +1339,7 @@ static ErlDrvTermData am_tos; static ErlDrvTermData am_tclass; static ErlDrvTermData am_ipv6_v6only; static ErlDrvTermData am_netns; +static ErlDrvTermData am_bind_to_device; #endif static char str_eafnosupport[] = "eafnosupport"; @@ -3725,6 +3731,7 @@ static void inet_init_sctp(void) { INIT_ATOM(tclass); INIT_ATOM(ipv6_v6only); INIT_ATOM(netns); + INIT_ATOM(bind_to_device); /* Option names */ INIT_ATOM(sctp_rtoinfo); @@ -5946,6 +5953,9 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) int ival; char* arg_ptr; int arg_sz; +#ifdef SO_BINDTODEVICE + char ifname[IFNAMSIZ]; +#endif enum PacketParseType old_htype = desc->htype; int old_active = desc->active; int propagate; /* Set to 1 if failure to set this option @@ -6331,6 +6341,29 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) len -= arg_sz; break; +#ifdef SO_BINDTODEVICE + case INET_OPT_BIND_TO_DEVICE: + if (ival < 0) return -1; + if (len < ival) return -1; + if (ival > sizeof(ifname)) { + return -1; + } + memcpy(ifname, ptr, ival); + ifname[ival] = '\0'; + ptr += ival; + len -= ival; + + proto = SOL_SOCKET; + type = SO_BINDTODEVICE; + arg_ptr = (char*)&ifname; + arg_sz = sizeof(ifname); + propagate = 1; /* We do want to know if this fails */ + + DEBUGF(("inet_set_opts(%ld): s=%d, SO_BINDTODEVICE=%s\r\n", + (long)desc->port, desc->s, ifname)); + break; +#endif + default: return -1; } @@ -6463,6 +6496,9 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len) # ifdef SCTP_DELAYED_ACK_TIME struct sctp_assoc_value av; /* Not in SOLARIS10 */ # endif +# ifdef SO_BINDTODEVICE + char ifname[IFNAMSIZ]; +# endif } arg; @@ -6702,6 +6738,23 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len) continue; /* Option not supported -- ignore it */ # endif +#ifdef SO_BINDTODEVICE + case INET_OPT_BIND_TO_DEVICE: + arg_sz = get_int32(curr); curr += 4; + CHKLEN(curr, arg_sz); + if (arg_sz >= sizeof(arg.ifname)) + return -1; + memcpy(arg.ifname, curr, arg_sz); + arg.ifname[arg_sz] = '\0'; + curr += arg_sz; + + proto = SOL_SOCKET; + type = SO_BINDTODEVICE; + arg_ptr = (char*) (&arg.ifname); + arg_sz = sizeof ( arg.ifname); + break; +#endif + case SCTP_OPT_AUTOCLOSE: { arg.ival= get_int32 (curr); curr += 4; @@ -6967,6 +7020,9 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc, ErlDrvSizeT dest_used = 0; ErlDrvSizeT dest_allocated = destlen; char *orig_dest = *dest; +#ifdef SO_BINDTODEVICE + char ifname[IFNAMSIZ]; +#endif /* Ptr is a name parameter */ #define RETURN_ERROR() \ @@ -7302,6 +7358,26 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc, put_int32(arg_sz,ptr); continue; } + +#ifdef SO_BINDTODEVICE + case INET_OPT_BIND_TO_DEVICE: + arg_sz = sizeof(ifname); + TRUNCATE_TO(0,ptr); + PLACE_FOR(5 + arg_sz,ptr); + arg_ptr = ptr + 5; + if (IS_SOCKET_ERROR(sock_getopt(desc->s,SOL_SOCKET,SO_BINDTODEVICE, + arg_ptr,&arg_sz))) { + TRUNCATE_TO(0,ptr); + continue; + } + arg_sz = my_strnlen(arg_ptr, arg_sz); + TRUNCATE_TO(arg_sz + 5,ptr); + *ptr++ = opt; + put_int32(arg_sz,ptr); + ptr += arg_sz; + continue; +#endif + default: RETURN_ERROR(); } @@ -7583,6 +7659,25 @@ static ErlDrvSSizeT sctp_fill_opts(inet_descriptor* desc, i = LOAD_TUPLE (spec, i, 2); break; } + +#ifdef SO_BINDTODEVICE + /* The following option returns a binary: */ + case INET_OPT_BIND_TO_DEVICE: { + char ifname[IFNAMSIZ]; + unsigned int sz = sizeof(ifname); + + if (sock_getopt(desc->s, SOL_SOCKET, SO_BINDTODEVICE, + &ifname, &sz) < 0) continue; + /* Fill in the response: */ + PLACE_FOR(spec, i, + LOAD_ATOM_CNT + LOAD_BUF2BINARY_CNT + LOAD_TUPLE_CNT); + i = LOAD_ATOM (spec, i, am_bind_to_device); + i = LOAD_BUF2BINARY(spec, i, ifname, my_strnlen(ifname, sz)); + i = LOAD_TUPLE (spec, i, 2); + break; + } +#endif + /* The following options just return an integer value: */ case INET_OPT_RCVBUF : case INET_OPT_SNDBUF : diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 8b420b9e9b..072ca19eae 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -603,10 +603,7 @@ static void print_mfa(Eterm mod, Eterm fun, unsigned int ari) } #endif -/* - * Convert {M,F,A} to pointer to first insn after initial func_info. - */ -static Uint *hipe_find_emu_address(Eterm mod, Eterm name, unsigned int arity) +static ErtsCodeInfo* hipe_find_emu_address(Eterm mod, Eterm name, unsigned int arity) { Module *modp; BeamCodeHeader* code_hdr; @@ -617,15 +614,15 @@ static Uint *hipe_find_emu_address(Eterm mod, Eterm name, unsigned int arity) return NULL; n = code_hdr->num_functions; for (i = 0; i < n; ++i) { - Uint *code_ptr = (Uint*)code_hdr->functions[i]; - ASSERT(code_ptr[0] == BeamOpCode(op_i_func_info_IaaI)); - if (code_ptr[3] == name && code_ptr[4] == arity) - return code_ptr+5; + ErtsCodeInfo *ci = code_hdr->functions[i]; + ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); + if (ci->mfa.function == name && ci->mfa.arity == arity) + return ci; } return NULL; } -Uint *hipe_bifs_find_pc_from_mfa(Eterm term) +ErtsCodeInfo* hipe_bifs_find_pc_from_mfa(Eterm term) { struct hipe_mfa mfa; @@ -636,10 +633,10 @@ Uint *hipe_bifs_find_pc_from_mfa(Eterm term) BIF_RETTYPE hipe_bifs_fun_to_address_1(BIF_ALIST_1) { - Eterm *pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); - if (!pc) + ErtsCodeInfo* ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); + if (!ci) BIF_ERROR(BIF_P, BADARG); - BIF_RET(address_to_term(pc, BIF_P)); + BIF_RET(address_to_term(erts_codeinfo_to_code(ci), BIF_P)); } BIF_RETTYPE hipe_bifs_commit_patch_load_1(BIF_ALIST_1) @@ -652,7 +649,7 @@ BIF_RETTYPE hipe_bifs_commit_patch_load_1(BIF_ALIST_1) BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3) { - Eterm *pc; + ErtsCodeInfo *ci; void *address; int is_closure; struct hipe_mfa mfa; @@ -675,11 +672,12 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3) simply have called hipe_bifs_find_pc_from_mfa(). */ if (!term_to_mfa(BIF_ARG_1, &mfa)) BIF_ERROR(BIF_P, BADARG); - pc = hipe_find_emu_address(mfa.mod, mfa.fun, mfa.ari); + ci = hipe_find_emu_address(mfa.mod, mfa.fun, mfa.ari); - if (pc) { - DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "set beam call trap at %p -> %p", pc, address); - hipe_set_call_trap(pc, address, is_closure); + if (ci) { + DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "set beam call trap at %p -> %p", + erts_codeinfo_to_code(ci), address); + hipe_set_call_trap(ci, address, is_closure); BIF_RET(am_true); } DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "failed set call trap to %p, no beam code found", address); @@ -1618,46 +1616,6 @@ static void purge_mfa(struct hipe_mfa_info* p) erts_free(ERTS_ALC_T_HIPE_LL, p); } -/* Called by init:restart after unloading all hipe compiled modules - * to work around old bug that caused execution of deallocated beam code. - * Can be removed now when delete/purge of native modules works better. - * Test: Do init:restart in debug compiled vm with hipe compiled kernel. - */ -static void hipe_purge_all_refs(void) -{ - struct hipe_mfa_info **bucket; - unsigned int i, nrbuckets; - - hipe_mfa_info_table_rwlock(); - - ASSERT(hipe_mfa_info_table.used == 0); - bucket = hipe_mfa_info_table.bucket; - nrbuckets = 1 << hipe_mfa_info_table.log2size; - for (i = 0; i < nrbuckets; ++i) { - ASSERT(bucket[i] == NULL); - while (bucket[i] != NULL) { - struct hipe_mfa_info* mfa = bucket[i]; - bucket[i] = mfa->bucket.next; - - hash_erase(&mod2mfa_tab, mfa); - erts_free(ERTS_ALC_T_HIPE_LL, mfa); - } - } - hipe_mfa_info_table.used = 0; - hipe_mfa_info_table_rwunlock(); -} - -BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1) -{ - if (BIF_ARG_1 == am_all) { - hipe_purge_all_refs(); - BIF_RET(am_ok); - } - - ASSERT(!"hipe_bifs_remove_refs_from_1() called"); - BIF_ERROR(BIF_P, BADARG); -} - int hipe_purge_need_blocking(Module* modp) { /* SVERK: Verify if this is really necessary */ @@ -1978,6 +1936,6 @@ BIF_RETTYPE hipe_bifs_alloc_loader_state_1(BIF_ALIST_1) hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE); res = erts_mk_magic_ref(&hp, &MSO(BIF_P), magic); - erts_refc_dec(&magic->refc, 1); + erts_refc_dec(&magic->intern.refc, 1); BIF_RET(res); } diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h index 811c3801c1..d6be473ff5 100644 --- a/erts/emulator/hipe/hipe_bif0.h +++ b/erts/emulator/hipe/hipe_bif0.h @@ -26,7 +26,7 @@ #ifndef HIPE_BIF0_H #define HIPE_BIF0_H -extern Uint *hipe_bifs_find_pc_from_mfa(Eterm mfa); +extern ErtsCodeInfo *hipe_bifs_find_pc_from_mfa(Eterm mfa); extern void hipe_mfa_info_table_init(void); extern void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a); diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab index 264ea2c34a..078ebc40b7 100644 --- a/erts/emulator/hipe/hipe_bif0.tab +++ b/erts/emulator/hipe/hipe_bif0.tab @@ -82,7 +82,6 @@ bif hipe_bifs:patch_insn/3 bif hipe_bifs:patch_call/3 bif hipe_bifs:add_ref/2 -bif hipe_bifs:remove_refs_from/1 bif hipe_bifs:alloc_loader_state/1 diff --git a/erts/emulator/hipe/hipe_bif1.c b/erts/emulator/hipe/hipe_bif1.c index 0ba0fa5172..0d4c1539b6 100644 --- a/erts/emulator/hipe/hipe_bif1.c +++ b/erts/emulator/hipe/hipe_bif1.c @@ -39,13 +39,15 @@ BIF_RETTYPE hipe_bifs_call_count_on_1(BIF_ALIST_1) { + ErtsCodeInfo *ci; Eterm *pc; struct hipe_call_count *hcc; - pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); - if (!pc) + ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); + if (!ci) BIF_ERROR(BIF_P, BADARG); - ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); + pc = erts_codeinfo_to_code(ci); if (pc[0] == BeamOpCode(op_hipe_trap_call)) BIF_ERROR(BIF_P, BADARG); if (pc[0] == BeamOpCode(op_hipe_call_count)) @@ -53,59 +55,65 @@ BIF_RETTYPE hipe_bifs_call_count_on_1(BIF_ALIST_1) hcc = erts_alloc(ERTS_ALC_T_HIPE_SL, sizeof(*hcc)); hcc->count = 0; hcc->opcode = pc[0]; - pc[-4] = (Eterm)hcc; + ci->u.hcc = hcc; pc[0] = BeamOpCode(op_hipe_call_count); BIF_RET(am_true); } BIF_RETTYPE hipe_bifs_call_count_off_1(BIF_ALIST_1) { + ErtsCodeInfo* ci; Eterm *pc; struct hipe_call_count *hcc; unsigned count; - pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); - if (!pc) + ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); + if (!ci) BIF_ERROR(BIF_P, BADARG); - ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); + pc = erts_codeinfo_to_code(ci); if (pc[0] != BeamOpCode(op_hipe_call_count)) BIF_RET(am_false); - hcc = (struct hipe_call_count*)pc[-4]; + hcc = ci->u.hcc; count = hcc->count; pc[0] = hcc->opcode; - pc[-4] = (Eterm)NULL; + ci->u.hcc = NULL; erts_free(ERTS_ALC_T_HIPE_SL, hcc); BIF_RET(make_small(count)); } BIF_RETTYPE hipe_bifs_call_count_get_1(BIF_ALIST_1) { + ErtsCodeInfo* ci; Eterm *pc; struct hipe_call_count *hcc; - pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); - if (!pc) + ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); + if (!ci) BIF_ERROR(BIF_P, BADARG); - ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); + pc = erts_codeinfo_to_code(ci); if (pc[0] != BeamOpCode(op_hipe_call_count)) BIF_RET(am_false); - hcc = (struct hipe_call_count*)pc[-4]; + hcc = ci->u.hcc; BIF_RET(make_small(hcc->count)); } BIF_RETTYPE hipe_bifs_call_count_clear_1(BIF_ALIST_1) { + ErtsCodeInfo* ci; Eterm *pc; struct hipe_call_count *hcc; unsigned count; - pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); - if (!pc) + ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); + if (!ci) BIF_ERROR(BIF_P, BADARG); - ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); + pc = erts_codeinfo_to_code(ci); if (pc[0] != BeamOpCode(op_hipe_call_count)) BIF_RET(am_false); - hcc = (struct hipe_call_count*)pc[-4]; + hcc = ci->u.hcc; count = hcc->count; hcc->count = 0; BIF_RET(make_small(count)); diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c index 0b53880628..9a9e3c6b12 100644 --- a/erts/emulator/hipe/hipe_load.c +++ b/erts/emulator/hipe/hipe_load.c @@ -81,7 +81,7 @@ Binary *hipe_alloc_loader_state(Eterm module) magic = erts_create_magic_binary(sizeof(HipeLoaderState), hipe_loader_state_dtor); - erts_refc_inc(&magic->refc, 1); + erts_refc_inc(&magic->intern.refc, 1); stp = ERTS_MAGIC_BIN_DATA(magic); stp->module = module; diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c index 712f65f629..1270a94986 100644 --- a/erts/emulator/hipe/hipe_mode_switch.c +++ b/erts/emulator/hipe/hipe_mode_switch.c @@ -176,14 +176,15 @@ void hipe_mode_switch_init(void) hipe_mfa_info_table_init(); } -void hipe_set_call_trap(Uint *bfun, void *nfun, int is_closure) +void hipe_set_call_trap(ErtsCodeInfo* ci, void *nfun, int is_closure) { - HIPE_ASSERT(bfun[-5] == BeamOpCode(op_i_func_info_IaaI)); + BeamInstr* bfun = erts_codeinfo_to_code(ci); + HIPE_ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); bfun[0] = is_closure ? BeamOpCode(op_hipe_trap_call_closure) : BeamOpCode(op_hipe_trap_call); - bfun[-4] = (Uint)nfun; + ci->u.ncallee = (void (*)(void)) nfun; } static __inline__ void diff --git a/erts/emulator/hipe/hipe_mode_switch.h b/erts/emulator/hipe/hipe_mode_switch.h index 334e978307..7b896872a6 100644 --- a/erts/emulator/hipe/hipe_mode_switch.h +++ b/erts/emulator/hipe/hipe_mode_switch.h @@ -55,7 +55,7 @@ extern int hipe_modeswitch_debug; void hipe_mode_switch_init(void); -void hipe_set_call_trap(Uint *bfun, void *nfun, int is_closure); +void hipe_set_call_trap(ErtsCodeInfo*, void *nfun, int is_closure); Process *hipe_mode_switch(Process*, unsigned, Eterm*); void hipe_inc_nstack(Process *p); void hipe_empty_nstack(Process *p); diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c index e581f07f56..342407ef62 100644 --- a/erts/emulator/hipe/hipe_native_bif.c +++ b/erts/emulator/hipe/hipe_native_bif.c @@ -323,7 +323,6 @@ char *hipe_bs_allocate(int len) Binary *bptr; bptr = erts_bin_nrml_alloc(len); - erts_refc_init(&bptr->refc, 1); return bptr->orig_bytes; } diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index 1e05fd3490..d1895f3793 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -87,6 +87,7 @@ static const int debruijn[32] = { #define CACHE_AREAS (32 - MSEG_ALIGN_BITS) +/* FIXME: segment sizes > 2 GB result in bogus negative indices */ #define SIZE_TO_CACHE_AREA_IDX(S) (LOG2((S)) - MSEG_ALIGN_BITS) #define MAX_CACHE_SIZE (30) @@ -396,6 +397,9 @@ static ERTS_INLINE int cache_bless_segment(ErtsMsegAllctr_t *ma, void *seg, UWor if (MSEG_FLG_IS_2POW(flags)) { int ix = SIZE_TO_CACHE_AREA_IDX(size); + if (ix < 0) + return 0; + ASSERT(ix < CACHE_AREAS); ASSERT((1 << (ix + MSEG_ALIGN_BITS)) == size); @@ -471,6 +475,9 @@ static ERTS_INLINE void *cache_get_segment(ErtsMsegAllctr_t *ma, UWord *size_p, ASSERT(IS_2POW(size)); + if (ix < 0) + return NULL; + for( i = ix; i < CACHE_AREAS; i++) { if (erts_circleq_is_empty(&(ma->cache_powered_node[i]))) diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h index bd554238b7..aa373e76bc 100644 --- a/erts/emulator/sys/unix/erl_unix_sys.h +++ b/erts/emulator/sys/unix/erl_unix_sys.h @@ -322,6 +322,7 @@ extern SIGFUNC sys_signal(int, SIGFUNC); extern void sys_sigrelease(int); extern void sys_sigblock(int); extern void sys_init_suspend_handler(void); +extern void erts_sys_unix_later_init(void); /* * Handling of floating point exceptions. diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index f7ee9bace0..3752edc480 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -854,9 +854,14 @@ int erts_set_signal(Eterm signal, Eterm type) { /* Disable break */ void erts_set_ignore_break(void) { - sys_signal(SIGINT, SIG_IGN); - sys_signal(SIGQUIT, SIG_IGN); - sys_signal(SIGTSTP, SIG_IGN); + /* + * Ignore signals that can be sent to the VM by + * typing certain key combinations at the + * controlling terminal... + */ + sys_signal(SIGINT, SIG_IGN); /* Ctrl-C */ + sys_signal(SIGQUIT, SIG_IGN); /* Ctrl-\ */ + sys_signal(SIGTSTP, SIG_IGN); /* Ctrl-Z */ } /* Don't use ctrl-c for break handler but let it be @@ -880,7 +885,6 @@ void erts_replace_intr(void) { void init_break_handler(void) { sys_signal(SIGINT, request_break); - sys_signal(SIGTERM, generic_signal_handler); sys_signal(SIGHUP, generic_signal_handler); #ifndef ETHR_UNUSABLE_SIGUSRX sys_signal(SIGUSR1, generic_signal_handler); @@ -895,6 +899,12 @@ void sys_init_suspend_handler(void) #endif } +void +erts_sys_unix_later_init(void) +{ + sys_signal(SIGTERM, generic_signal_handler); +} + int sys_max_files(void) { return(max_files); diff --git a/erts/emulator/sys/unix/sys_drivers.c b/erts/emulator/sys/unix/sys_drivers.c index 400f163652..93cf64719a 100644 --- a/erts/emulator/sys/unix/sys_drivers.c +++ b/erts/emulator/sys/unix/sys_drivers.c @@ -204,6 +204,7 @@ erl_sys_late_init(void) #ifdef ERTS_SMP erts_mtx_unlock(port->lock); #endif + erts_sys_unix_later_init(); /* Need to be called after forker has been started */ } /* II. Prototypes */ diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile index 8c8c73aa3e..ca81c9915e 100644 --- a/erts/emulator/test/Makefile +++ b/erts/emulator/test/Makefile @@ -121,7 +121,6 @@ MODULES= \ port_trace_SUITE \ unique_SUITE \ z_SUITE \ - old_mod \ long_timers_test \ ignore_cores \ dgawd_handler \ diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl index 6994bfef83..4f98cd6cee 100644 --- a/erts/emulator/test/distribution_SUITE.erl +++ b/erts/emulator/test/distribution_SUITE.erl @@ -19,7 +19,7 @@ %% -module(distribution_SUITE). --compile(r15). +-compile(r16). -define(VERSION_MAGIC, 131). @@ -48,7 +48,7 @@ dist_parallel_send/1, atom_roundtrip/1, unicode_atom_roundtrip/1, - atom_roundtrip_r15b/1, + atom_roundtrip_r16b/1, contended_atom_cache_entry/1, contended_unicode_atom_cache_entry/1, bad_dist_structure/1, @@ -78,7 +78,8 @@ all() -> link_to_dead_new_node, applied_monitor_node, ref_port_roundtrip, nil_roundtrip, stop_dist, {group, trap_bif}, {group, dist_auto_connect}, - dist_parallel_send, atom_roundtrip, unicode_atom_roundtrip, atom_roundtrip_r15b, + dist_parallel_send, atom_roundtrip, unicode_atom_roundtrip, + atom_roundtrip_r16b, contended_atom_cache_entry, contended_unicode_atom_cache_entry, bad_dist_structure, {group, bad_dist_ext}, start_epmd_false, epmd_module]. @@ -1032,21 +1033,21 @@ atom_roundtrip(Config) when is_list(Config) -> stop_node(Node), ok. -atom_roundtrip_r15b(Config) when is_list(Config) -> - case test_server:is_release_available("r15b") of +atom_roundtrip_r16b(Config) when is_list(Config) -> + case test_server:is_release_available("r16b") of true -> ct:timetrap({minutes, 6}), - AtomData = atom_data(), + AtomData = unicode_atom_data(), verify_atom_data(AtomData), - case start_node(Config, [], "r15b") of + case start_node(Config, [], "r16b") of {ok, Node} -> do_atom_roundtrip(Node, AtomData), stop_node(Node); {error, timeout} -> - {skip,"Unable to start OTP R15B release"} + {skip,"Unable to start OTP R16B release"} end; false -> - {skip,"No OTP R15B available"} + {skip,"No OTP R16B available"} end. unicode_atom_roundtrip(Config) when is_list(Config) -> diff --git a/erts/emulator/test/list_bif_SUITE.erl b/erts/emulator/test/list_bif_SUITE.erl index 514dd2f412..08574bff71 100644 --- a/erts/emulator/test/list_bif_SUITE.erl +++ b/erts/emulator/test/list_bif_SUITE.erl @@ -23,7 +23,7 @@ -export([all/0, suite/0]). -export([hd_test/1,tl_test/1,t_length/1,t_list_to_pid/1, - t_list_to_float/1,t_list_to_integer/1]). + t_list_to_port/1,t_list_to_float/1,t_list_to_integer/1]). suite() -> @@ -32,7 +32,7 @@ suite() -> all() -> - [hd_test, tl_test, t_length, t_list_to_pid, + [hd_test, tl_test, t_length, t_list_to_pid, t_list_to_port, t_list_to_float, t_list_to_integer]. %% Tests list_to_integer and string:to_integer @@ -47,9 +47,9 @@ t_list_to_integer(Config) when is_list(Config) -> {12373281903728109372810937209817320981321,"ABC"} = string:to_integer("12373281903728109372810937209817320981321ABC"), {-12373281903728109372810937209817320981321,"ABC"} = string:to_integer("-12373281903728109372810937209817320981321ABC"), {12,[345]} = string:to_integer([$1,$2,345]), - {12,[a]} = string:to_integer([$1,$2,a]), + {error, badarg} = string:to_integer([$1,$2,a]), {error,no_integer} = string:to_integer([$A]), - {error,not_a_list} = string:to_integer($A), + {error,badarg} = string:to_integer($A), ok. %% Test hd/1 with correct and incorrect arguments. @@ -106,10 +106,25 @@ t_list_to_pid(Config) when is_list(Config) -> {'EXIT', {badarg, _}} -> ok; Res -> - ct:fail("list_to_pid/1 with incorrect arg succeeded.~nResult: ~p", [Res]) + ct:fail("list_to_pid/1 with incorrect arg succeeded.~n" + "Result: ~p", [Res]) end, ok. +%% Test list_to_port/1 with correct and incorrect arguments. + +t_list_to_port(Config) when is_list(Config) -> + Me = hd(erlang:ports()), + MyListedPid = port_to_list(Me), + Me = list_to_port(MyListedPid), + case catch list_to_port(id("Incorrect list")) of + {'EXIT', {badarg, _}} -> + ok; + Res -> + ct:fail("list_to_port/1 with incorrect arg succeeded.~n" + "Result: ~p", [Res]) + end, + ok. %% Test list_to_float/1 with correct and incorrect arguments. diff --git a/erts/emulator/test/node_container_SUITE.erl b/erts/emulator/test/node_container_SUITE.erl index 291d3ee30d..30518a5b27 100644 --- a/erts/emulator/test/node_container_SUITE.erl +++ b/erts/emulator/test/node_container_SUITE.erl @@ -45,7 +45,6 @@ ets_refc/1, match_spec_refc/1, timer_refc/1, - otp_4715/1, pid_wrap/1, port_wrap/1, bad_nc/1, @@ -62,7 +61,7 @@ all() -> [term_to_binary_to_term_eq, round_trip_eq, cmp, ref_eq, node_table_gc, dist_link_refc, dist_monitor_refc, node_controller_refc, ets_refc, match_spec_refc, - timer_refc, otp_4715, pid_wrap, port_wrap, bad_nc, + timer_refc, pid_wrap, port_wrap, bad_nc, unique_pid, iter_max_procs, magic_ref]. init_per_suite(Config) -> @@ -684,35 +683,6 @@ timer_refc(Config) when is_list(Config) -> nc_refc_check(node()), ok. -otp_4715(Config) when is_list(Config) -> - case test_server:is_release_available("r9b") of - true -> otp_4715_1(Config); - false -> {skip,"No R9B found"} - end. - -otp_4715_1(Config) -> - case erlang:system_info(compat_rel) of - 9 -> - run_otp_4715(Config); - _ -> - Pa = filename:dirname(code:which(?MODULE)), - test_server:run_on_shielded_node(fun () -> - run_otp_4715(Config) - end, - "+R9 -pa " ++ Pa) - end. - -run_otp_4715(Config) when is_list(Config) -> - erts_debug:set_internal_state(available_internal_state, true), - PidList = [mk_pid({a@b, 1}, 4710, 2), - mk_pid({a@b, 1}, 4712, 1), - mk_pid({c@b, 1}, 4711, 1), - mk_pid({b@b, 3}, 4711, 1), - mk_pid({b@b, 2}, 4711, 1)], - - R9Sorted = old_mod:sort_on_old_node(PidList), - R9Sorted = lists:sort(PidList). - pid_wrap(Config) when is_list(Config) -> pp_wrap(pid). port_wrap(Config) when is_list(Config) -> diff --git a/erts/emulator/test/old_mod.erl b/erts/emulator/test/old_mod.erl deleted file mode 100644 index 866aba79bb..0000000000 --- a/erts/emulator/test/old_mod.erl +++ /dev/null @@ -1,48 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2003-2016. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% - --module(old_mod). --compile(r10). - --export([sort_on_old_node/1, sorter/3]). - --include_lib("common_test/include/ct.hrl"). - -sorter(Receiver, Ref, List) -> - Receiver ! {Ref, lists:sort(List)}. - -sort_on_old_node(List) when is_list(List) -> - OldVersion = "r10", - Pa = filename:dirname(code:which(?MODULE)), - {X, Y, Z} = now(), - NodeName = list_to_atom(OldVersion - ++ "_" - ++ integer_to_list(X) - ++ integer_to_list(Y) - ++ integer_to_list(Z)), - {ok, Node} = test_server:start_node(NodeName, - peer, - [{args, " -pa " ++ Pa}, - {erl, [{release, OldVersion++"b_patched"}]}]), - Ref = make_ref(), - spawn_link(Node, ?MODULE, sorter, [self(), Ref, List]), - SortedPids = receive {Ref, SP} -> SP end, - true = test_server:stop_node(Node), - SortedPids. diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl index 885298ce34..64e51e7d7c 100644 --- a/erts/emulator/test/scheduler_SUITE.erl +++ b/erts/emulator/test/scheduler_SUITE.erl @@ -1087,12 +1087,8 @@ scheduler_threads(Config) when is_list(Config) -> {Sched, SchedOnln, _} = get_sstate(Config, ""), %% Configure half the number of both the scheduler threads and %% the scheduler threads online. - {HalfSched, HalfSchedOnln} = case SmpSupport of - false -> {1,1}; - true -> - {Sched div 2, - SchedOnln div 2} - end, + {HalfSched, HalfSchedOnln} = {lists:max([1,Sched div 2]), + lists:max([1,SchedOnln div 2])}, {HalfSched, HalfSchedOnln, _} = get_sstate(Config, "+SP 50:50"), %% Use +S to configure 4x the number of scheduler threads and %% 4x the number of scheduler threads online, but alter that @@ -1149,12 +1145,8 @@ dirty_scheduler_threads(Config) when is_list(Config) -> dirty_scheduler_threads_test(Config) -> SmpSupport = erlang:system_info(smp_support), {Sched, SchedOnln, _} = get_dsstate(Config, ""), - {HalfSched, HalfSchedOnln} = case SmpSupport of - false -> {1,1}; - true -> - {Sched div 2, - SchedOnln div 2} - end, + {HalfSched, HalfSchedOnln} = {lists:max([1,Sched div 2]), + lists:max([1,SchedOnln div 2])}, Cmd1 = "+SDcpu "++integer_to_list(HalfSched)++":"++ integer_to_list(HalfSchedOnln), {HalfSched, HalfSchedOnln, _} = get_dsstate(Config, Cmd1), diff --git a/erts/emulator/test/smoke_test_SUITE.erl b/erts/emulator/test/smoke_test_SUITE.erl index 5eccdc562b..45b28b28a5 100644 --- a/erts/emulator/test/smoke_test_SUITE.erl +++ b/erts/emulator/test/smoke_test_SUITE.erl @@ -66,17 +66,10 @@ boot_combo(Config) when is_list(Config) -> ok end end, - SMPDisable = fun () -> false = erlang:system_info(smp_support) end, try chk_boot(Config, "+Ktrue", NOOP), chk_boot(Config, "+A42", A42), - chk_boot(Config, "-smp disable", SMPDisable), chk_boot(Config, "+Ktrue +A42", A42), - chk_boot(Config, "-smp disable +A42", - fun () -> SMPDisable(), A42() end), - chk_boot(Config, "-smp disable +Ktrue", SMPDisable), - chk_boot(Config, "-smp disable +Ktrue +A42", - fun () -> SMPDisable(), A42() end), %% A lot more combos could be implemented... ok after diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl index f51244485b..729e86cb4f 100644 --- a/erts/emulator/test/statistics_SUITE.erl +++ b/erts/emulator/test/statistics_SUITE.erl @@ -329,9 +329,9 @@ scheduler_wall_time_test(Type) -> %% 50% load HalfHogs = [StartHog() || _ <- lists:seq(1, (Schedulers-1) div 2)], HalfDirtyCPUHogs = [StartDirtyHog(dirty_cpu) - || _ <- lists:seq(1, DirtyCPUSchedulers div 2)], + || _ <- lists:seq(1, lists:max([1,DirtyCPUSchedulers div 2]))], HalfDirtyIOHogs = [StartDirtyHog(dirty_io) - || _ <- lists:seq(1, DirtyIOSchedulers div 2)], + || _ <- lists:seq(1, lists:max([1,DirtyIOSchedulers div 2]))], HalfLoad = lists:sum(get_load(Type)) div TotLoadSchedulers, if Schedulers < 2, HalfLoad > 80 -> ok; %% Ok only one scheduler online and one hog %% We want roughly 50% load diff --git a/erts/etc/common/Makefile.in b/erts/etc/common/Makefile.in index f656eebc46..b4fa0d4c79 100644 --- a/erts/etc/common/Makefile.in +++ b/erts/etc/common/Makefile.in @@ -143,7 +143,7 @@ MC_OUTPUTS=$(OBJDIR)/erlsrv_logmess.h $(OBJDIR)/erlsrv_logmess.res MT_FLAG="-MD" endif INET_GETHOST = $(BINDIR)/inet_gethost.exe -INSTALL_EMBEDDED_PROGS += $(BINDIR)/dialyzer.exe $(BINDIR)/erlc.exe $(BINDIR)/start_erl.exe $(BINDIR)/escript.exe $(BINDIR)/ct_run.exe +INSTALL_EMBEDDED_PROGS += $(BINDIR)/typer.exe $(BINDIR)/dialyzer.exe $(BINDIR)/erlc.exe $(BINDIR)/start_erl.exe $(BINDIR)/escript.exe $(BINDIR)/ct_run.exe INSTALL_SRC = $(WINETC)/start_erl.c $(WINETC)/Nmakefile.start_erl ERLEXECDIR=. INSTALL_LIBS = @@ -176,7 +176,7 @@ ENTRY_OBJ= ERLSRV_OBJECTS= MC_OUTPUTS= INET_GETHOST = $(BINDIR)/inet_gethost@EXEEXT@ -INSTALL_EMBEDDED_PROGS += $(BINDIR)/dialyzer@EXEEXT@ \ +INSTALL_EMBEDDED_PROGS += $(BINDIR)/typer@EXEEXT@ $(BINDIR)/dialyzer@EXEEXT@ \ $(BINDIR)/erlc@EXEEXT@ $(BINDIR)/escript@EXEEXT@ $(BINDIR)/ct_run@EXEEXT@ \ $(BINDIR)/run_erl $(BINDIR)/to_erl $(BINDIR)/dyn_erl INSTALL_EMBEDDED_DATA = $(UXETC)/start.src $(UXETC)/start_erl.src @@ -238,6 +238,7 @@ endif rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/reclaim.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/run_erl.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/to_erl.o + rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/typer.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/dyn_erl.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/safe_string.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/run_erl.o @@ -432,6 +433,12 @@ $(BINDIR)/dialyzer@EXEEXT@: $(OBJDIR)/dialyzer.o $(ERTS_LIB) $(OBJDIR)/dialyzer.o: dialyzer.c $(RC_GENERATED) $(V_CC) $(CFLAGS) -o $@ -c dialyzer.c +$(BINDIR)/typer@EXEEXT@: $(OBJDIR)/typer.o $(ERTS_LIB) + $(ld_verbose)$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/typer.o -L$(OBJDIR) $(LIBS) $(ERTS_INTERNAL_LIBS) + +$(OBJDIR)/typer.o: typer.c $(RC_GENERATED) + $(V_CC) $(CFLAGS) -o $@ -c typer.c + $(BINDIR)/escript@EXEEXT@: $(OBJDIR)/escript.o $(ERTS_LIB) $(ld_verbose)$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/escript.o -L$(OBJDIR) $(LIBS) $(ERTS_INTERNAL_LIBS) diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index ee59759940..8b75199a1e 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -141,6 +141,8 @@ static char *pluss_val_switches[] = { "wt", "ws", "ss", + "ssdcpu", + "ssdio", "pp", "ub", NULL @@ -203,8 +205,8 @@ void error(char* format, ...); * Local functions. */ -#if !defined(ERTS_HAVE_SMP_EMU) -static void usage_notsup(const char *switchname); +#if !defined(ERTS_HAVE_SMP_EMU) || !defined(ERTS_HAVE_PLAIN_EMU) +static void usage_notsup(const char *switchname, const char *alt); #endif static char **build_args_from_env(char *env_var); static char **build_args_from_string(char *env_var); @@ -487,8 +489,7 @@ int main(int argc, char **argv) cpuinfo = erts_cpu_info_create(); /* '-smp auto' is default */ #ifdef ERTS_HAVE_SMP_EMU - if (erts_get_cpu_configured(cpuinfo) > 1) - emu_type |= EMU_TYPE_SMP; + emu_type |= EMU_TYPE_SMP; #endif #if defined(__WIN32__) && defined(WIN32_ALWAYS_DEBUG) @@ -520,12 +521,11 @@ int main(int argc, char **argv) i++; smp_auto: emu_type_passed |= EMU_TYPE_SMP; -#ifdef ERTS_HAVE_SMP_EMU - if (erts_get_cpu_configured(cpuinfo) > 1) - emu_type |= EMU_TYPE_SMP; - else +#if defined(ERTS_HAVE_PLAIN_EMU) && !defined(ERTS_HAVE_SMP_EMU) + emu_type &= ~EMU_TYPE_SMP; +#else + emu_type |= EMU_TYPE_SMP; #endif - emu_type &= ~EMU_TYPE_SMP; } else if (strcmp(argv[i+1], "enable") == 0) { i++; @@ -534,14 +534,18 @@ int main(int argc, char **argv) #ifdef ERTS_HAVE_SMP_EMU emu_type |= EMU_TYPE_SMP; #else - usage_notsup("-smp enable"); + usage_notsup("-smp enable", ""); #endif } else if (strcmp(argv[i+1], "disable") == 0) { i++; smp_disable: - emu_type_passed |= EMU_TYPE_SMP; + emu_type_passed &= ~EMU_TYPE_SMP; +#ifdef ERTS_HAVE_PLAIN_EMU emu_type &= ~EMU_TYPE_SMP; +#else + usage_notsup("-smp disable", " Use \"+S 1\" instead."); +#endif } else { smp: @@ -550,7 +554,7 @@ int main(int argc, char **argv) #ifdef ERTS_HAVE_SMP_EMU emu_type |= EMU_TYPE_SMP; #else - usage_notsup("-smp"); + usage_notsup("-smp", ""); #endif } } else if (strcmp(argv[i], "-smpenable") == 0) { @@ -1183,14 +1187,14 @@ usage_aux(void) #ifdef __WIN32__ "[-start_erl [datafile]] " #endif - "[-smp " + "[-smp [auto" #ifdef ERTS_HAVE_SMP_EMU - "[enable|" + "|enable" #endif - "auto|disable" -#ifdef ERTS_HAVE_SMP_EMU - "]" +#ifdef ERTS_HAVE_PLAIN_EMU + "|disable" #endif + "]" "] " "[-make] [-man [manopts] MANPAGE] [-x] [-emu_args] [-start_epmd BOOLEAN] " "[-args_file FILENAME] [+A THREADS] [+a SIZE] [+B[c|d|i]] [+c [BOOLEAN]] " @@ -1212,11 +1216,11 @@ usage(const char *switchname) usage_aux(); } -#if !defined(ERTS_HAVE_SMP_EMU) +#if !defined(ERTS_HAVE_SMP_EMU) || !defined(ERTS_HAVE_PLAIN_EMU) static void -usage_notsup(const char *switchname) +usage_notsup(const char *switchname, const char *alt) { - fprintf(stderr, "Argument \'%s\' not supported.\n", switchname); + fprintf(stderr, "Argument \'%s\' not supported.%s\n", switchname, alt); usage_aux(); } #endif diff --git a/erts/etc/common/typer.c b/erts/etc/common/typer.c new file mode 100644 index 0000000000..77a95ccded --- /dev/null +++ b/erts/etc/common/typer.c @@ -0,0 +1,455 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2006-2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/* + * Purpose: Typer front-end. + */ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#ifdef __WIN32__ +#include <winbase.h> +#endif + +#include <ctype.h> + +#define NO 0 +#define YES 1 + +#define ASIZE(a) (sizeof(a)/sizeof(a[0])) + +static int debug = 0; /* Bit flags for debug printouts. */ + +static char** eargv_base; /* Base of vector. */ +static char** eargv; /* First argument for erl. */ + +static int eargc; /* Number of arguments in eargv. */ + +#ifdef __WIN32__ +# define QUOTE(s) possibly_quote(s) +# define IS_DIRSEP(c) ((c) == '/' || (c) == '\\') +# define ERL_NAME "erl.exe" +#else +# define QUOTE(s) s +# define IS_DIRSEP(c) ((c) == '/') +# define ERL_NAME "erl" +#endif + +#define UNSHIFT(s) eargc++, eargv--; eargv[0] = QUOTE(s) +#define PUSH(s) eargv[eargc++] = QUOTE(s) +#define PUSH2(s, t) PUSH(s); PUSH(t) +#define PUSH3(s, t, u) PUSH2(s, t); PUSH(u) + +/* + * Local functions. + */ + +static void error(char* format, ...); +static void* emalloc(size_t size); +static char* strsave(char* string); +static void push_words(char* src); +static int run_erlang(char* name, char** argv); +static char* get_default_emulator(char* progname); +#ifdef __WIN32__ +static char* possibly_quote(char* arg); +static void* erealloc(void *p, size_t size); +#endif + +/* + * Supply a strerror() function if libc doesn't. + */ +#ifndef HAVE_STRERROR + +extern int sys_nerr; + +#ifndef SYS_ERRLIST_DECLARED +extern const char * const sys_errlist[]; +#endif /* !SYS_ERRLIST_DECLARED */ + +char *strerror(int errnum) +{ + static char *emsg[1024]; + + if (errnum != 0) { + if (errnum > 0 && errnum < sys_nerr) + sprintf((char *) &emsg[0], "(%s)", sys_errlist[errnum]); + else + sprintf((char *) &emsg[0], "errnum = %d ", errnum); + } + else { + emsg[0] = '\0'; + } + return (char *) &emsg[0]; +} +#endif /* !HAVE_STRERROR */ + +#ifdef __WIN32__ +int wmain(int argc, wchar_t **wcargv) +{ + char** argv; +#else +int +main(int argc, char** argv) +{ +#endif + int eargv_size; + int eargc_base; /* How many arguments in the base of eargv. */ + char* emulator; + int need_shell = 0; + +#ifdef __WIN32__ + int i; + int len; + /* Convert argv to utf8 */ + argv = emalloc((argc+1) * sizeof(char*)); + for (i=0; i<argc; i++) { + len = WideCharToMultiByte(CP_UTF8, 0, wcargv[i], -1, NULL, 0, NULL, NULL); + argv[i] = emalloc(len*sizeof(char)); + WideCharToMultiByte(CP_UTF8, 0, wcargv[i], -1, argv[i], len, NULL, NULL); + } + argv[argc] = NULL; +#endif + + emulator = get_default_emulator(argv[0]); + + /* + * Allocate the argv vector to be used for arguments to Erlang. + * Arrange for starting to pushing information in the middle of + * the array, to allow easy addition of commands in the beginning. + */ + + eargv_size = argc*4+100; + eargv_base = (char **) emalloc(eargv_size*sizeof(char*)); + eargv = eargv_base; + eargc = 0; + push_words(emulator); + eargc_base = eargc; + eargv = eargv + eargv_size/2; + eargc = 0; + + /* + * Push initial arguments. + */ + + if (argc > 1 && strcmp(argv[1], "-smp") == 0) { + PUSH("-smpauto"); + argc--, argv++; + } + + PUSH("+B"); + PUSH2("-boot", "start_clean"); + PUSH3("-run", "typer", "start"); + PUSH("-extra"); + + /* + * Push everything except --shell. + */ + + while (argc > 1) { + if (strcmp(argv[1], "--shell") == 0) { + need_shell = 1; + } else { + PUSH(argv[1]); + } + argc--, argv++; + } + + if (!need_shell) { + UNSHIFT("-noinput"); + } + + /* + * Move up the commands for invoking the emulator and adjust eargv + * accordingly. + */ + + while (--eargc_base >= 0) { + UNSHIFT(eargv_base[eargc_base]); + } + + /* + * Invoke Erlang with the collected options. + */ + + PUSH(NULL); + return run_erlang(eargv[0], eargv); +} + +static void +push_words(char* src) +{ + char sbuf[MAXPATHLEN]; + char* dst; + + dst = sbuf; + while ((*dst++ = *src++) != '\0') { + if (isspace((int)*src)) { + *dst = '\0'; + PUSH(strsave(sbuf)); + dst = sbuf; + do { + src++; + } while (isspace((int)*src)); + } + } + if (sbuf[0]) + PUSH(strsave(sbuf)); +} +#ifdef __WIN32__ +wchar_t *make_commandline(char **argv) +{ + static wchar_t *buff = NULL; + static int siz = 0; + int num = 0, len; + char **arg; + wchar_t *p; + + if (*argv == NULL) { + return L""; + } + for (arg = argv; *arg != NULL; ++arg) { + num += strlen(*arg)+1; + } + if (!siz) { + siz = num; + buff = (wchar_t *) emalloc(siz*sizeof(wchar_t)); + } else if (siz < num) { + siz = num; + buff = (wchar_t *) erealloc(buff,siz*sizeof(wchar_t)); + } + p = buff; + num=0; + for (arg = argv; *arg != NULL; ++arg) { + len = MultiByteToWideChar(CP_UTF8, 0, *arg, -1, p, siz); + p+=(len-1); + *p++=L' '; + } + *(--p) = L'\0'; + + if (debug) { + printf("Processed command line:%S\n",buff); + } + return buff; +} + +int my_spawnvp(char **argv) +{ + STARTUPINFOW siStartInfo; + PROCESS_INFORMATION piProcInfo; + DWORD ec; + + memset(&siStartInfo,0,sizeof(STARTUPINFOW)); + siStartInfo.cb = sizeof(STARTUPINFOW); + siStartInfo.dwFlags = STARTF_USESTDHANDLES; + siStartInfo.hStdInput = GetStdHandle(STD_INPUT_HANDLE); + siStartInfo.hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE); + siStartInfo.hStdError = GetStdHandle(STD_ERROR_HANDLE); + + if (!CreateProcessW(NULL, + make_commandline(argv), + NULL, + NULL, + TRUE, + 0, + NULL, + NULL, + &siStartInfo, + &piProcInfo)) { + return -1; + } + CloseHandle(piProcInfo.hThread); + + WaitForSingleObject(piProcInfo.hProcess,INFINITE); + if (!GetExitCodeProcess(piProcInfo.hProcess,&ec)) { + return 0; + } + return (int) ec; +} +#endif /* __WIN32__ */ + + +static int +run_erlang(char* progname, char** argv) +{ +#ifdef __WIN32__ + int status; +#endif + + if (debug) { + int i = 0; + while (argv[i] != NULL) + printf(" %s", argv[i++]); + printf("\n"); + } + +#ifdef __WIN32__ + /* + * Alas, we must wait here for the program to finish. + * Otherwise, the shell from which we were executed will think + * we are finished and print a prompt and read keyboard input. + */ + + status = my_spawnvp(argv)/*_spawnvp(_P_WAIT,progname,argv)*/; + if (status == -1) { + fprintf(stderr, "typer: Error executing '%s': %d", progname, + GetLastError()); + } + return status; +#else + execvp(progname, argv); + error("Error %d executing \'%s\'.", errno, progname); + return 2; +#endif +} + +static void +error(char* format, ...) +{ + char sbuf[1024]; + va_list ap; + + va_start(ap, format); + erts_vsnprintf(sbuf, sizeof(sbuf), format, ap); + va_end(ap); + fprintf(stderr, "typer: %s\n", sbuf); + exit(1); +} + +static void* +emalloc(size_t size) +{ + void *p = malloc(size); + if (p == NULL) + error("Insufficient memory"); + return p; +} + +#ifdef __WIN32__ +static void * +erealloc(void *p, size_t size) +{ + void *res = realloc(p, size); + if (res == NULL) + error("Insufficient memory"); + return res; +} +#endif + +static char* +strsave(char* string) +{ + char* p = emalloc(strlen(string)+1); + strcpy(p, string); + return p; +} + +static int +file_exists(char *progname) +{ +#ifdef __WIN32__ + wchar_t wcsbuf[MAXPATHLEN]; + MultiByteToWideChar(CP_UTF8, 0, progname, -1, wcsbuf, MAXPATHLEN); + return (_waccess(wcsbuf, 0) != -1); +#else + return (access(progname, 1) != -1); +#endif +} + +static char* +get_default_emulator(char* progname) +{ + char sbuf[MAXPATHLEN]; + char* s; + + if (strlen(progname) >= sizeof(sbuf)) + return ERL_NAME; + + strcpy(sbuf, progname); + for (s = sbuf+strlen(sbuf); s >= sbuf; s--) { + if (IS_DIRSEP(*s)) { + strcpy(s+1, ERL_NAME); + if(file_exists(sbuf)) + return strsave(sbuf); + break; + } + } + return ERL_NAME; +} + +#ifdef __WIN32__ +static char* +possibly_quote(char* arg) +{ + int mustQuote = NO; + int n = 0; + char* s; + char* narg; + + if (arg == NULL) { + return arg; + } + + /* + * Scan the string to find out if it needs quoting and return + * the original argument if not. + */ + + for (s = arg; *s; s++, n++) { + switch(*s) { + case ' ': + mustQuote = YES; + continue; + case '"': + mustQuote = YES; + n++; + continue; + case '\\': + if(s[1] == '"') + n++; + continue; + default: + continue; + } + } + if (!mustQuote) { + return arg; + } + + /* + * Insert the quotes and put a backslash in front of every quote + * inside the string. + */ + + s = narg = emalloc(n+2+1); + for (*s++ = '"'; *arg; arg++, s++) { + if (*arg == '"' || (*arg == '\\' && arg[1] == '"')) { + *s++ = '\\'; + } + *s = *arg; + } + if (s[-1] == '\\') { + *s++ ='\\'; + } + *s++ = '"'; + *s = '\0'; + return narg; +} +#endif /* __WIN32__ */ diff --git a/erts/etc/unix/Install.src b/erts/etc/unix/Install.src index 8be696b16f..e71308edbe 100644 --- a/erts/etc/unix/Install.src +++ b/erts/etc/unix/Install.src @@ -89,6 +89,7 @@ cd "$ERL_ROOT/bin" cp -p "$ERL_ROOT/erts-%I_VSN%/bin/erl" . cp -p "$ERL_ROOT/erts-%I_VSN%/bin/erlc" . cp -p "$ERL_ROOT/erts-%I_VSN%/bin/dialyzer" . +cp -p "$ERL_ROOT/erts-%I_VSN%/bin/typer" . cp -p "$ERL_ROOT/erts-%I_VSN%/bin/ct_run" . cp -p "$ERL_ROOT/erts-%I_VSN%/bin/escript" . diff --git a/erts/etc/unix/etp-commands.in b/erts/etc/unix/etp-commands.in index c689d495e6..643cfbf323 100644 --- a/erts/etc/unix/etp-commands.in +++ b/erts/etc/unix/etp-commands.in @@ -2871,6 +2871,81 @@ define etp-disasm end end +############################################################################ +# +# Timer Wheel +# + +define etp-timer-wheel +# Args: TimerWheel + if (!erts_initialized) + printf "System not initialized!\n" + else + set $tiw = $arg0 + printf "Number of timers: %d\n", $tiw->nto + printf "Min timeout pos: %d\n", $tiw->next_timeout_pos + printf "\n--- Soon Wheel ---\n" + set $ix = $tiw->pos & etp_tw_soon_wheel_mask + printf "Position: %ld (%d)\n", $tiw->pos, $ix + printf "Min timeout position: %ld (%d)\n", $tiw->soon.min_tpos, $tiw->soon.min_tpos & etp_tw_soon_wheel_mask + printf "Number of timers: %d\n", $tiw->soon.nto + set $slots = etp_tw_soon_wheel_size + while $slots > 0 + set $tmr = $tiw->w[$ix] + if ($tmr != (ErtsTWheelTimer *) 0x0) + printf "---\n" + printf "Slot: %d\n", $ix + printf "\n" + while 1 + printf "- Timeout pos: %ld\n", $tmr->timeout_pos + printf " Pointer: (ErtsTWheelTimer *) %p\n", $tmr + set $tmr = $tmr->next + if ($tmr == $tiw->w[$ix]) + loop_break + end + end + end + set $ix++ + if ($ix == (etp_tw_soon_wheel_first_slot + etp_tw_soon_wheel_size)) + set $ix = etp_tw_soon_wheel_first_slot + end + set $slots-- + end + printf "\n--- Later Wheel ---\n" + set $ix = (($tiw->later.pos >> etp_tw_later_wheel_shift) & etp_tw_later_wheel_mask) + etp_tw_later_wheel_first_slot + printf "Position: %ld (%d)\n", $tiw->later.pos, $ix + printf "Min timeout position: %ld (%d)\n", $tiw->later.min_tpos, (($tiw->later.min_tpos >> etp_tw_later_wheel_shift) & etp_tw_later_wheel_mask) + etp_tw_later_wheel_first_slot + printf "Number of timers: %d\n", $tiw->later.nto + set $slots = etp_tw_later_wheel_size + set $slot_pos = $tiw->later.pos + while $slots > 0 + set $tmr = $tiw->w[$ix] + if ($tmr != (ErtsTWheelTimer *) 0x0) + printf "---\n" + printf "Slot: %d\n", $ix + printf "Slot Range: [%ld, %ld]\n", $slot_pos, $slot_pos + etp_tw_later_wheel_slot_size + printf "Pre timeout pos: %ld\n", $slot_pos - etp_tw_later_wheel_slot_size + printf "\n" + while 1 + printf "- Timeout pos: %ld\n", $tmr->timeout_pos + printf " Pointer: (ErtsTWheelTimer *) %p\n", $tmr + set $tmr = $tmr->next + if ($tmr == $tiw->w[$ix]) + loop_break + end + end + end + set $ix++ + if ($ix == (etp_tw_later_wheel_first_slot + etp_tw_later_wheel_size)) + set $ix = etp_tw_later_wheel_first_slot + end + set $slot_pos = $slot_pos + etp_tw_later_wheel_slot_size + set $slots-- + end + end + printf "---\n" +end + document etp-disasm %--------------------------------------------------------------------------- % etp-disasm StartI EndI diff --git a/erts/etc/win32/Install.c b/erts/etc/win32/Install.c index 04522a0779..5701df35a3 100644 --- a/erts/etc/win32/Install.c +++ b/erts/etc/win32/Install.c @@ -49,6 +49,7 @@ int wmain(int argc, wchar_t **argv) HANDLE module = GetModuleHandle(NULL); wchar_t *binaries[] = { L"erl.exe", L"werl.exe", L"erlc.exe", L"dialyzer.exe", + L"typer.exe", L"escript.exe", L"ct_run.exe", NULL }; wchar_t *scripts[] = { L"start_clean.boot", L"start_sasl.boot", L"no_dot_erlang.boot", NULL }; wchar_t fromname[MAX_PATH]; diff --git a/erts/preloaded/ebin/erl_prim_loader.beam b/erts/preloaded/ebin/erl_prim_loader.beam Binary files differindex 4b2f07f1a4..b17b100261 100644 --- a/erts/preloaded/ebin/erl_prim_loader.beam +++ b/erts/preloaded/ebin/erl_prim_loader.beam diff --git a/erts/preloaded/ebin/erl_tracer.beam b/erts/preloaded/ebin/erl_tracer.beam Binary files differindex c4e1e57c40..33f268737a 100644 --- a/erts/preloaded/ebin/erl_tracer.beam +++ b/erts/preloaded/ebin/erl_tracer.beam diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam Binary files differindex c584fe6f97..5980e23fe2 100644 --- a/erts/preloaded/ebin/erlang.beam +++ b/erts/preloaded/ebin/erlang.beam diff --git a/erts/preloaded/ebin/erts_code_purger.beam b/erts/preloaded/ebin/erts_code_purger.beam Binary files differindex af7028b01c..c109b6c52a 100644 --- a/erts/preloaded/ebin/erts_code_purger.beam +++ b/erts/preloaded/ebin/erts_code_purger.beam diff --git a/erts/preloaded/ebin/erts_dirty_process_code_checker.beam b/erts/preloaded/ebin/erts_dirty_process_code_checker.beam Binary files differindex a6ecba2a17..63e6d5a656 100644 --- a/erts/preloaded/ebin/erts_dirty_process_code_checker.beam +++ b/erts/preloaded/ebin/erts_dirty_process_code_checker.beam diff --git a/erts/preloaded/ebin/erts_internal.beam b/erts/preloaded/ebin/erts_internal.beam Binary files differindex c4b281163e..878563c7e2 100644 --- a/erts/preloaded/ebin/erts_internal.beam +++ b/erts/preloaded/ebin/erts_internal.beam diff --git a/erts/preloaded/ebin/erts_literal_area_collector.beam b/erts/preloaded/ebin/erts_literal_area_collector.beam Binary files differindex 4552a34ae2..5b464e23d1 100644 --- a/erts/preloaded/ebin/erts_literal_area_collector.beam +++ b/erts/preloaded/ebin/erts_literal_area_collector.beam diff --git a/erts/preloaded/ebin/init.beam b/erts/preloaded/ebin/init.beam Binary files differindex e90a988a07..47a086536a 100644 --- a/erts/preloaded/ebin/init.beam +++ b/erts/preloaded/ebin/init.beam diff --git a/erts/preloaded/ebin/otp_ring0.beam b/erts/preloaded/ebin/otp_ring0.beam Binary files differindex 77b0b7a537..198a9f0fcd 100644 --- a/erts/preloaded/ebin/otp_ring0.beam +++ b/erts/preloaded/ebin/otp_ring0.beam diff --git a/erts/preloaded/ebin/prim_eval.beam b/erts/preloaded/ebin/prim_eval.beam Binary files differindex ac9916cc86..eb6409e460 100644 --- a/erts/preloaded/ebin/prim_eval.beam +++ b/erts/preloaded/ebin/prim_eval.beam diff --git a/erts/preloaded/ebin/prim_file.beam b/erts/preloaded/ebin/prim_file.beam Binary files differindex 2d8a355c82..84a7b734e5 100644 --- a/erts/preloaded/ebin/prim_file.beam +++ b/erts/preloaded/ebin/prim_file.beam diff --git a/erts/preloaded/ebin/prim_inet.beam b/erts/preloaded/ebin/prim_inet.beam Binary files differindex e2559ac034..066fd0e172 100644 --- a/erts/preloaded/ebin/prim_inet.beam +++ b/erts/preloaded/ebin/prim_inet.beam diff --git a/erts/preloaded/ebin/prim_zip.beam b/erts/preloaded/ebin/prim_zip.beam Binary files differindex 12cbb5bf7a..327c55ce4d 100644 --- a/erts/preloaded/ebin/prim_zip.beam +++ b/erts/preloaded/ebin/prim_zip.beam diff --git a/erts/preloaded/ebin/zlib.beam b/erts/preloaded/ebin/zlib.beam Binary files differindex f5217eadf6..687942fbd5 100644 --- a/erts/preloaded/ebin/zlib.beam +++ b/erts/preloaded/ebin/zlib.beam diff --git a/erts/preloaded/src/add_abstract_code b/erts/preloaded/src/add_abstract_code index 943987872e..f53791cdc2 100644 --- a/erts/preloaded/src/add_abstract_code +++ b/erts/preloaded/src/add_abstract_code @@ -28,12 +28,12 @@ main([BeamFile,AbstrFile]) -> {ok,_,Chunks0} = beam_lib:all_chunks(BeamFile), {ok,Abstr} = file:consult(AbstrFile), - Chunks1 = lists:keyreplace("Abst", 1, Chunks0, - {"Abst",term_to_binary({raw_abstract_v1,Abstr})}), - {"CInf",CInf0} = lists:keyfind("CInf", 1, Chunks1), - CInf = fix_options(CInf0), - Chunks = lists:keyreplace("CInf", 1, Chunks1, {"CInf",CInf}), - {ok,Module} = beam_lib:build_module(Chunks), + {"CInf",CInf0} = lists:keyfind("CInf", 1, Chunks0), + {CInf, COpts} = fix_options(CInf0), + Chunks1 = lists:keyreplace("CInf", 1, Chunks0, {"CInf",CInf}), + Chunks2 = lists:keyreplace("Dbgi", 1, Chunks1, + {"Dbgi",term_to_binary({debug_info_v1,erl_abstract_code,{Abstr, COpts}})}), + {ok,Module} = beam_lib:build_module(Chunks2), ok = file:write_file(BeamFile, Module), init:stop(). @@ -42,4 +42,4 @@ fix_options(CInf0) -> {options,Opts0} = lists:keyfind(options, 1, CInf1), Opts = Opts0 -- [from_asm], CInf = lists:keyreplace(options, 1, CInf1, {options,Opts}), - term_to_binary(CInf). + {term_to_binary(CInf), Opts}. diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index 888d2beee0..8499d40c9a 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -129,7 +129,7 @@ -export([list_to_atom/1, list_to_binary/1]). -export([list_to_bitstring/1, list_to_existing_atom/1, list_to_float/1]). -export([list_to_integer/1, list_to_integer/2]). --export([list_to_pid/1, list_to_ref/1, list_to_tuple/1, loaded/0]). +-export([list_to_pid/1, list_to_port/1, list_to_ref/1, list_to_tuple/1, loaded/0]). -export([localtime/0, make_ref/0]). -export([map_size/1, match_spec_test/3, md5/1, md5_final/1]). -export([md5_init/0, md5_update/2, module_loaded/1, monitor/2]). @@ -1159,9 +1159,15 @@ list_to_integer(_String,_Base) -> String :: string(). list_to_pid(_String) -> erlang:nif_error(undefined). + +%% list_to_port/1 +-spec list_to_port(String) -> port() when + String :: string(). +list_to_port(_String) -> + erlang:nif_error(undefined). %% list_to_ref/1 --spec erlang:list_to_ref(String) -> reference() when +-spec list_to_ref(String) -> reference() when String :: string(). list_to_ref(_String) -> erlang:nif_error(undefined). @@ -1337,7 +1343,7 @@ pid_to_list(_Pid) -> erlang:nif_error(undefined). %% port_to_list/1 --spec erlang:port_to_list(Port) -> string() when +-spec port_to_list(Port) -> string() when Port :: port(). port_to_list(_Port) -> erlang:nif_error(undefined). @@ -1547,7 +1553,7 @@ read_timer(_TimerRef, _Options) -> erlang:nif_error(undefined). %% ref_to_list/1 --spec erlang:ref_to_list(Ref) -> string() when +-spec ref_to_list(Ref) -> string() when Ref :: reference(). ref_to_list(_Ref) -> erlang:nif_error(undefined). diff --git a/erts/preloaded/src/init.erl b/erts/preloaded/src/init.erl index c12a5b159a..1ccf8d599f 100644 --- a/erts/preloaded/src/init.erl +++ b/erts/preloaded/src/init.erl @@ -689,16 +689,8 @@ do_unload([M|Mods]) -> catch erlang:purge_module(M), do_unload(Mods); do_unload([]) -> - purge_all_hipe_refs(), ok. -purge_all_hipe_refs() -> - case erlang:system_info(hipe_architecture) of - undefined -> ok; - _ -> hipe_bifs:remove_refs_from(all) - end. - - sub([H|T],L) -> sub(T,del(H,L)); sub([],L) -> L. diff --git a/erts/preloaded/src/prim_inet.erl b/erts/preloaded/src/prim_inet.erl index 61f727e8a4..19f8b0d873 100644 --- a/erts/preloaded/src/prim_inet.erl +++ b/erts/preloaded/src/prim_inet.erl @@ -1234,6 +1234,7 @@ enc_opt(netns) -> ?INET_LOPT_NETNS; enc_opt(show_econnreset) -> ?INET_LOPT_TCP_SHOW_ECONNRESET; enc_opt(line_delimiter) -> ?INET_LOPT_LINE_DELIM; enc_opt(raw) -> ?INET_OPT_RAW; +enc_opt(bind_to_device) -> ?INET_OPT_BIND_TO_DEVICE; % Names of SCTP opts: enc_opt(sctp_rtoinfo) -> ?SCTP_OPT_RTOINFO; enc_opt(sctp_associnfo) -> ?SCTP_OPT_ASSOCINFO; @@ -1294,6 +1295,7 @@ dec_opt(?INET_LOPT_NETNS) -> netns; dec_opt(?INET_LOPT_TCP_SHOW_ECONNRESET) -> show_econnreset; dec_opt(?INET_LOPT_LINE_DELIM) -> line_delimiter; dec_opt(?INET_OPT_RAW) -> raw; +dec_opt(?INET_OPT_BIND_TO_DEVICE) -> bind_to_device; dec_opt(I) when is_integer(I) -> undefined. @@ -1395,6 +1397,7 @@ type_opt_1(packet_size) -> uint; type_opt_1(read_packets) -> uint; type_opt_1(netns) -> binary; type_opt_1(show_econnreset) -> bool; +type_opt_1(bind_to_device) -> binary; %% %% SCTP options (to be set). If the type is a record type, the corresponding %% record signature is returned, otherwise, an "elementary" type tag |