diff options
Diffstat (limited to 'erts/emulator')
152 files changed, 13315 insertions, 10485 deletions
diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index c6511a4b49..3117b24bcc 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -63,7 +63,28 @@ ARFLAGS=rc OMIT_OMIT_FP=no TYPE_LIBS= -DIRTY_SCHEDULER_SUPPORT=@DIRTY_SCHEDULER_SUPPORT@ +PROFILE_COMPILER=@PROFILE_COMPILER@ +PROFILE_MARKER= +ifeq ($(PROFILE),generate) +PROFILE_MARKER=_pg +else +ifeq ($(PROFILE),use) +PROFILE_MARKER=_pu +endif +endif + +ifeq ($(PROFILE_COMPILER), gcc) +PROFILE_CORRECTION=@PROFILE_CORRECTION@ +PROFILE_GENERATE=-fprofile-generate +PROFILE_USE=-fprofile-use $(PROFILE_CORRECTION) +PROFILE_USE_DEPS=$(OBJDIR)/%_pu.gcda +endif +ifeq ($(PROFILE_COMPILER), clang) +PROFILE_GENERATE=-fprofile-instr-generate +PROFILE_USE=-fprofile-instr-use=$(OBJDIR)/default.profdata +PROFILE_USE_DEPS=$(OBJDIR)/default.profdata +endif + DIRTY_SCHEDULER_TEST=@DIRTY_SCHEDULER_TEST@ ifeq ($(TYPE),debug) @@ -194,10 +215,6 @@ override FLAVOR=smp FLAVOR_MARKER=.smp ENABLE_ALLOC_TYPE_VARS += nofrag -ifeq ($(DIRTY_SCHEDULER_SUPPORT),yes) -THR_DEFS += -DERTS_DIRTY_SCHEDULERS -DS_SUPPORT=yes - ifeq ($(DIRTY_SCHEDULER_TEST),yes) DS_TEST=yes THR_DEFS += -DERTS_DIRTY_SCHEDULERS_TEST @@ -205,11 +222,6 @@ else # DIRTY_SCHEDULER_TEST DS_TEST=no endif # DIRTY_SCHEDULER_TEST -else # DIRTY_SCHEDULER_SUPPORT -DS_SUPPORT=no -DS_TEST=no -endif # DIRTY_SCHEDULER_SUPPORT - TF_MARKER=$(TYPEMARKER)$(FLAVOR_MARKER) ifeq ($(TYPE)-@HAVE_VALGRIND@,valgrind-no) @@ -228,18 +240,9 @@ ARCH=@ARCH@ ultrasparcCFLAGS=-Wa,-xarch=v8plusa ARCHCFLAGS=$($(ARCH)CFLAGS) -ifdef HIPE_ENABLED -ifeq ($(OPSYS),linux) -ppcBEAMLDFLAGS=-Wl,-m,elf32ppc -ppc64BEAMLDFLAGS=-Wl,-m,elf64ppc,-T,hipe/elf64ppc.x -endif -ifeq ($(OPSYS),darwin) -amd64BEAMLDFLAGS=-pagezero_size 0x10000000 -endif -HIPEBEAMLDFLAGS=$($(ARCH)BEAMLDFLAGS) -endif +HIPEBEAMLDFLAGS=@HIPEBEAMLDFLAGS@ -ERTS_ENABLE_KERNEL_POLL=@ERTS_ENABLE_KERNEL_POLL@ +ERTS_BUILD_FALLBACK_POLL=@ERTS_BUILD_FALLBACK_POLL@ # # @@ -418,9 +421,20 @@ ifeq ($(TARGET), win32) EMULATOR_EXECUTABLE = beam$(TF_MARKER).dll else EMULATOR_EXECUTABLE = beam$(TF_MARKER) +PROFILE_EXECUTABLE = beam.prof$(TF_MARKER) endif CS_EXECUTABLE = erl_child_setup$(TYPEMARKER) +ifeq ($(PROFILE), generate) +EMULATOR_EXECUTABLE = $(PROFILE_EXECUTABLE) +ifeq ($(PROFILE_COMPILER), gcc) +PROFILE_LDFLAGS = -fprofile-generate +endif +ifeq ($(PROFILE_COMPILER), clang) +PROFILE_LDFLAGS = -fprofile-instr-generate +endif +endif + # ---------------------------------------------------------------------- ifeq ($(ERLANG_OSTYPE), unix) @@ -476,7 +490,6 @@ ifeq ($(TARGET),win32) RELEASE_INCLUDES += sys/$(ERLANG_OSTYPE)/erl_win_dyn_driver.h endif - .PHONY: release_spec ifdef VOID_EMULATOR release_spec: @@ -545,8 +558,8 @@ OPCODE_TABLES += hipe/hipe_ops.tab hipe/hipe_instrs.tab endif $(TTF_DIR)/beam_cold.h \ +$(TTF_DIR)/beam_warm.h \ $(TTF_DIR)/beam_hot.h \ -$(TTF_DIR)/beam_instrs.h \ $(TTF_DIR)/beam_opcodes.c \ $(TTF_DIR)/beam_opcodes.h \ $(TTF_DIR)/beam_pred_funcs.h \ @@ -555,6 +568,7 @@ $(TTF_DIR)/beam_tr_funcs.h \ $(TTF_DIR)/OPCODES-GENERATED: $(OPCODE_TABLES) utils/beam_makeops $(gen_verbose)LANG=C $(PERL) utils/beam_makeops \ -wordsize @EXTERNAL_WORD_SIZE@ \ + -code-model @CODE_MODEL@ \ -outdir $(TTF_DIR) \ -DUSE_VM_PROBES=$(if $(USE_VM_PROBES),1,0) \ -DNO_FPE_SIGNALS=$(if $filter(unreliable,$(FPE)),1,0) \ @@ -594,7 +608,7 @@ $(HIPE_NBIF_FILES) \ : $(TTF_DIR)/TABLES-GENERATED $(TTF_DIR)/TABLES-GENERATED: $(ATOMS) $(DIRTY_BIFS) $(BIFS) utils/make_tables $(gen_verbose)LANG=C $(PERL) utils/make_tables -src $(TTF_DIR) -include $(TTF_DIR)\ - -ds $(DS_SUPPORT) -dst $(DS_TEST) -hipe $(HIPE) $(ATOMS) $(DIRTY_BIFS) $(BIFS) && echo $? >$(TTF_DIR)/TABLES-GENERATED + -dst $(DS_TEST) -hipe $(HIPE) $(ATOMS) $(DIRTY_BIFS) $(BIFS) && echo $? >$(TTF_DIR)/TABLES-GENERATED GENERATE += $(TTF_DIR)/TABLES-GENERATED $(TTF_DIR)/erl_alloc_types.h: beam/erl_alloc.types utils/make_alloc_types @@ -688,17 +702,66 @@ $(OBJDIR)/beams.$(RES_EXT): $(TARGET)/beams.rc endif -ifneq ($(filter tile-%,$(TARGET)),) -$(OBJDIR)/beam_emu.o: beam/beam_emu.c - $(V_CC) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) \ - $(INCLUDES) -c $< -o $@ -else +# We disable the implicit rule of .S -> .o so that the verbose asm +# generate is not used for compiling erts. This is only a problem on +# old solaris make +%.o : %.S + # Usually the same as the default rule, but certain platforms (e.g. win32) mix # different compilers $(OBJDIR)/beam_emu.o: beam/beam_emu.c $(V_EMU_CC) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@ + +$(OBJDIR)/beam_emu.S: beam/beam_emu.c + $(V_EMU_CC) -S -fverbose-asm $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@ + +$(OBJDIR)/%_pg.o: beam/%.c + $(V_CC) $(PROFILE_GENERATE) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@ +$(OBJDIR)/%_pu.o: beam/%.c $(PROFILE_USE_DEPS) + $(V_CC) $(PROFILE_USE) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@ + +$(OBJDIR)/%_pu.S: beam/%.c $(PROFILE_USE_DEPS) + $(V_CC) -S -fverbose-asm $(PROFILE_USE) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@ + +$(OBJDIR)/PROFILE: $(BINDIR)/$(PROFILE_EXECUTABLE) + $(V_at)echo " PROFILE ${PROFILE_EXECUTABLE}" + $(V_at)rm -f $(OBJDIR)/erl*.profraw + $(V_at)set -e; LLVM_PROFILE_FILE="$(OBJDIR)/erlc-%m.profraw" \ + ERL_FLAGS="-emu_type prof${TYPEMARKER} +S 1" $(ERLC) -DPGO \ + -o $(OBJDIR) test/estone_SUITE.erl > $(OBJDIR)/PROFILE_LOG + $(V_at)set -e; LLVM_PROFILE_FILE="$(OBJDIR)/erl-%m.profraw" \ + ERL_FLAGS="-emu_type prof${TYPEMARKER} +S 1" $(ERL) -pa $(OBJDIR) \ + -noshell -s estone_SUITE pgo -s init stop >> $(OBJDIR)/PROFILE_LOG + $(V_at)touch $@ + +.SECONDARY: $(patsubst %.o,%_pu.gcda,$(PROFILE_OBJS)) + +$(OBJDIR)/%_pu.gcda: $(OBJDIR)/PROFILE + $(V_at)mv $(OBJDIR)/$*_pg.gcda $@ + $(V_at)touch $@ + +$(OBJDIR)/default.profdata: $(OBJDIR)/PROFILE + $(V_LLVM_PROFDATA) merge -output $@ $(OBJDIR)/*.profraw + +ifeq ($(ERTS_BUILD_FALLBACK_POLL),yes) +# Have to treat erl_poll differently as the same .c file is used +# twice for kernel poll builds. +$(OBJDIR)/erl_poll.o: sys/common/erl_poll.c + $(V_CC) -DERTS_KERNEL_POLL_VERSION $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@ + +# Do a copy in order to make debuggers less confused +$(TTF_DIR)/erl_poll.flbk.c: sys/common/erl_poll.c + $(V_at) cp $< $@ + @touch $@ + +$(OBJDIR)/erl_poll.flbk.o: $(TTF_DIR)/erl_poll.flbk.c + $(V_CC) -DERTS_NO_KERNEL_POLL_VERSION $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@ endif + +# ---------------------------------------------------------------------- +# General targets +# $(OBJDIR)/%.o: beam/%.c $(V_CC) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@ @@ -732,12 +795,6 @@ $(BINDIR)/$(CS_EXECUTABLE): $(TTF_DIR)/GENERATED $(PRELOAD_SRC) $(CS_OBJ) $(ERTS $(ld_verbose)$(CS_PURIFY) $(LD) $(CS_LDFLAGS) -o $(BINDIR)/$(CS_EXECUTABLE) \ $(CS_CFLAGS) $(COMMON_INCLUDES) $(CS_OBJ) $(CS_LIBS) -$(OBJDIR)/%.kp.o: sys/common/%.c - $(V_CC) -DERTS_KERNEL_POLL_VERSION $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@ - -$(OBJDIR)/%.nkp.o: sys/common/%.c - $(V_CC) -DERTS_NO_KERNEL_POLL_VERSION $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@ - ifeq ($(GCC),yes) $(OBJDIR)/erl_goodfit_alloc.o: beam/erl_goodfit_alloc.c @@ -759,15 +816,23 @@ $(ERL_TOP)/lib/%.beam: INIT_OBJS = $(OBJDIR)/erl_main.o $(PRELOAD_OBJ) +# -fprofile-correction is needed in order to use PGO on erl_process +# as multiple threads execute in that file. +ifeq ($(PROFILE_CORRECTION),) +PROFILE_OBJS = $(OBJDIR)/beam_emu.o +RUN_OBJS = $(OBJDIR)/erl_process.o +else +PROFILE_OBJS = $(OBJDIR)/beam_emu.o $(OBJDIR)/erl_process.o +endif + EMU_OBJS = \ - $(OBJDIR)/beam_emu.o $(OBJDIR)/beam_opcodes.o \ + $(OBJDIR)/beam_opcodes.o \ $(OBJDIR)/beam_load.o $(OBJDIR)/beam_bif_load.o \ $(OBJDIR)/beam_debug.o $(OBJDIR)/beam_bp.o \ - $(OBJDIR)/beam_catches.o \ - $(OBJDIR)/code_ix.o \ + $(OBJDIR)/beam_catches.o $(OBJDIR)/code_ix.o \ $(OBJDIR)/beam_ranges.o -RUN_OBJS = \ +RUN_OBJS += \ $(OBJDIR)/erl_alloc.o $(OBJDIR)/erl_mtrace.o \ $(OBJDIR)/erl_alloc_util.o $(OBJDIR)/erl_goodfit_alloc.o \ $(OBJDIR)/erl_bestfit_alloc.o $(OBJDIR)/erl_afit_alloc.o \ @@ -783,7 +848,7 @@ RUN_OBJS = \ $(OBJDIR)/utils.o $(OBJDIR)/bif.o \ $(OBJDIR)/io.o $(OBJDIR)/erl_printf_term.o\ $(OBJDIR)/erl_debug.o $(OBJDIR)/erl_md5.o \ - $(OBJDIR)/erl_message.o $(OBJDIR)/erl_process.o \ + $(OBJDIR)/erl_message.o \ $(OBJDIR)/erl_process_dict.o $(OBJDIR)/erl_process_lock.o \ $(OBJDIR)/erl_port_task.o $(OBJDIR)/erl_arith.o \ $(OBJDIR)/time.o $(OBJDIR)/erl_time_sup.o \ @@ -810,17 +875,19 @@ RUN_OBJS = \ $(OBJDIR)/erl_bif_binary.o $(OBJDIR)/erl_ao_firstfit_alloc.o \ $(OBJDIR)/erl_thr_queue.o $(OBJDIR)/erl_sched_spec_pre_alloc.o \ $(OBJDIR)/erl_ptab.o $(OBJDIR)/erl_map.o \ - $(OBJDIR)/erl_msacc.o $(OBJDIR)/erl_lock_flags.o + $(OBJDIR)/erl_msacc.o $(OBJDIR)/erl_lock_flags.o \ + $(OBJDIR)/erl_io_queue.o LTTNG_OBJS = $(OBJDIR)/erlang_lttng.o -NIF_OBJS = $(OBJDIR)/erl_tracer_nif.o +NIF_OBJS = \ + $(OBJDIR)/erl_tracer_nif.o \ + $(OBJDIR)/zlib_nif.o ifeq ($(TARGET),win32) DRV_OBJS = \ $(OBJDIR)/registry_drv.o \ $(OBJDIR)/efile_drv.o \ $(OBJDIR)/inet_drv.o \ - $(OBJDIR)/zlib_drv.o \ $(OBJDIR)/ram_file_drv.o \ $(OBJDIR)/ttsl_drv.o OS_OBJS = \ @@ -850,7 +917,6 @@ OS_OBJS = \ DRV_OBJS = \ $(OBJDIR)/efile_drv.o \ $(OBJDIR)/inet_drv.o \ - $(OBJDIR)/zlib_drv.o \ $(OBJDIR)/ram_file_drv.o \ $(OBJDIR)/ttsl_drv.o endif @@ -879,23 +945,19 @@ $(STATIC_NIF_LIBS) $(STATIC_DRIVER_LIBS): echo "=== Leaving lib after making static libs" endif -ifeq ($(ERTS_ENABLE_KERNEL_POLL),yes) -OS_OBJS += $(OBJDIR)/erl_poll.kp.o \ - $(OBJDIR)/erl_check_io.kp.o \ - $(OBJDIR)/erl_poll.nkp.o \ - $(OBJDIR)/erl_check_io.nkp.o -else OS_OBJS += $(OBJDIR)/erl_poll.o \ - $(OBJDIR)/erl_check_io.o -endif - -OS_OBJS += $(OBJDIR)/erl_mseg.o \ + $(OBJDIR)/erl_check_io.o \ + $(OBJDIR)/erl_mseg.o \ $(OBJDIR)/erl_mmap.o \ $(OBJDIR)/erl_$(ERLANG_OSTYPE)_sys_ddll.o \ $(OBJDIR)/erl_mtrace_sys_wrap.o \ $(OBJDIR)/erl_sys_common_misc.o \ $(OBJDIR)/erl_os_monotonic_time_extender.o +ifeq ($(ERTS_BUILD_FALLBACK_POLL),yes) +OS_OBJS += $(OBJDIR)/erl_poll.flbk.o +endif + HIPE_ARCH64_OBJS=$(OBJDIR)/hipe_bif64.o HIPE_x86_OS_OBJS=$(HIPE_x86_$(OPSYS)_OBJS) @@ -924,21 +986,23 @@ ifdef HIPE_ENABLED EXTRA_BASE_OBJS += $(HIPE_OBJS) endif -BASE_OBJS = $(EMU_OBJS) $(RUN_OBJS) $(OS_OBJS) $(EXTRA_BASE_OBJS) $(LTTNG_OBJS) +BASE_OBJS = $(EMU_OBJS) $(RUN_OBJS) $(OS_OBJS) $(EXTRA_BASE_OBJS) \ + $(LTTNG_OBJS) $(DRV_OBJS) $(NIF_OBJS) + +PROF_OBJS = $(patsubst %.o,%$(PROFILE_MARKER).o,$(PROFILE_OBJS)) $(BASE_OBJS) -before_DTrace_OBJS = $(BASE_OBJS) $(DRV_OBJS) $(NIF_OBJS) +OBJS = $(PROF_OBJS) -DTRACE_OBJS = ifdef DTRACE_ENABLED_2STEP -DTRACE_OBJS = $(OBJDIR)/erlang_dtrace.o -$(OBJDIR)/erlang_dtrace.o: $(before_DTrace_OBJS) $(TARGET)/erlang_dtrace.h +# The $(PROFILE_MARKER) is placed in the object file name in order to +# make sure we re-compile with the new object files for the profiled emulator +OBJS += $(OBJDIR)/erlang$(PROFILE_MARKER)_dtrace.o +$(OBJDIR)/erlang$(PROFILE_MARKER)_dtrace.o: $(PROF_OBJS) $(TARGET)/erlang_dtrace.h dtrace -G -C -Ibeam \ -s beam/erlang_dtrace.d \ - -o $@ $(before_DTrace_OBJS) + -o $@ $(PROF_OBJS) endif -OBJS = $(before_DTrace_OBJS) $(DTRACE_OBJS) - $(INIT_OBJS): $(TTF_DIR)/GENERATED $(OBJS): $(TTF_DIR)/GENERATED @@ -1030,8 +1094,8 @@ $(BINDIR)/$(EMULATOR_EXECUTABLE): $(INIT_OBJS) $(OBJS) $(DEPLIBS) else $(BINDIR)/$(EMULATOR_EXECUTABLE): $(INIT_OBJS) $(OBJS) $(DEPLIBS) - $(ld_verbose)$(PURIFY) $(LD) -o $(BINDIR)/$(EMULATOR_EXECUTABLE) \ - $(HIPEBEAMLDFLAGS) $(LDFLAGS) $(DEXPORT) $(INIT_OBJS) $(OBJS) \ + $(ld_verbose)$(PURIFY) $(LD) -o $@ \ + $(HIPEBEAMLDFLAGS) $(PROFILE_LDFLAGS) $(LDFLAGS) $(DEXPORT) $(INIT_OBJS) $(OBJS) \ $(STATIC_NIF_LIBS) $(STATIC_DRIVER_LIBS) $(LIBS) endif @@ -1050,8 +1114,7 @@ SED_REPL_O=s|^\([^:]*:\)|$$(OBJDIR)/\1|g SED_REPL_O_ZLIB=s|^\([^:]*:\)|$$(ZLIB_OBJDIR)/\1|g SED_REPL_TTF_DIR=s|$(TTF_DIR)/|$$(TTF_DIR)/|g SED_REPL_ERL_TOP=s|\([ ]\)$(ERL_TOP)/|\1$$(ERL_TOP)/|g;s|^$(ERL_TOP)/|$$(ERL_TOP)/|g -SED_REPL_POLL=s|$$(OBJDIR)/erl_poll.o|$$(OBJDIR)/erl_poll.kp.o $$(OBJDIR)/erl_poll.nkp.o|g -SED_REPL_CHK_IO=s|$$(OBJDIR)/erl_check_io.o|$$(OBJDIR)/erl_check_io.kp.o $$(OBJDIR)/erl_check_io.nkp.o|g +SED_REPL_POLL=s|$$(OBJDIR)/erl_poll.o|$$(OBJDIR)/erl_poll.o $$(OBJDIR)/erl_poll.flbk.o|g SED_REPL_TTF_COMP_FLAGS=s|\([^/]\)erl_compile_flags\.h|\1$$(TTF_DIR)/erl_compile_flags\.h|g ifeq ($(TARGET),win32) @@ -1061,8 +1124,8 @@ else SED_PREFIX= endif -ifeq ($(ERTS_ENABLE_KERNEL_POLL),yes) -SED_SUFFIX=;$(SED_REPL_POLL);$(SED_REPL_CHK_IO) +ifeq ($(ERTS_BUILD_FALLBACK_POLL),yes) +SED_SUFFIX=;$(SED_REPL_POLL) else SED_SUFFIX= endif @@ -1116,6 +1179,10 @@ DEP_FLAGS=-MM $(MG_FLAG) $(CFLAGS) $(INCLUDES) -Inifs/common -Idrivers/common -I SYS_SRC=$(ALL_SYS_SRC) endif +.PHONY: check_emu_registers +check_emu_registers: $(OBJDIR)/beam_emu$(PROFILE_MARKER).S + utils/beam_emu_vars -vars 'c_p E HTOP FCALLS I reg' $^ + .PHONY: $(TARGET)/gen_git_version.mk $(TARGET)/gen_git_version.mk: # We touch beam/erl_bif.info.c if we regenerated the git version to force a diff --git a/erts/emulator/beam/arith_instrs.tab b/erts/emulator/beam/arith_instrs.tab index 91fe21e161..b828e86788 100644 --- a/erts/emulator/beam/arith_instrs.tab +++ b/erts/emulator/beam/arith_instrs.tab @@ -28,7 +28,7 @@ OUTLINED_ARITH_2(Fail, Live, Name, BIF, Op1, Op2, Dst) { result = erts_gc_$Name (c_p, reg, live); HEAVY_SWAPIN; ERTS_HOLE_CHECK(c_p); - if (is_value(result)) { + if (ERTS_LIKELY(is_value(result))) { $REFRESH_GEN_DEST(); $Dst = result; $NEXT0(); @@ -49,10 +49,9 @@ plus.fetch(Op1, Op2) { } plus.execute(Fail, Live, Dst) { - if (is_both_small(PlusOp1, PlusOp2)) { + if (ERTS_LIKELY(is_both_small(PlusOp1, PlusOp2))) { Sint i = signed_val(PlusOp1) + signed_val(PlusOp2); - ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); - if (MY_IS_SSMALL(i)) { + if (ERTS_LIKELY(IS_SSMALL(i))) { $Dst = make_small(i); $NEXT0(); } @@ -72,10 +71,9 @@ minus.fetch(Op1, Op2) { } minus.execute(Fail, Live, Dst) { - if (is_both_small(MinusOp1, MinusOp2)) { + if (ERTS_LIKELY(is_both_small(MinusOp1, MinusOp2))) { Sint i = signed_val(MinusOp1) - signed_val(MinusOp2); - ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); - if (MY_IS_SSMALL(i)) { + if (ERTS_LIKELY(IS_SSMALL(i))) { $Dst = make_small(i); $NEXT0(); } @@ -87,9 +85,6 @@ i_increment := increment.fetch.execute; increment.head() { Eterm increment_reg_val; - Eterm increment_val; - Uint live; - Eterm result; } increment.fetch(Src) { @@ -97,11 +92,13 @@ increment.fetch(Src) { } increment.execute(IncrementVal, Live, Dst) { - increment_val = $IncrementVal; - if (is_small(increment_reg_val)) { + Eterm increment_val = $IncrementVal; + Uint live; + Eterm result; + + if (ERTS_LIKELY(is_small(increment_reg_val))) { Sint i = signed_val(increment_reg_val) + increment_val; - ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); - if (MY_IS_SSMALL(i)) { + if (ERTS_LIKELY(IS_SSMALL(i))) { $Dst = make_small(i); $NEXT0(); } @@ -113,7 +110,7 @@ increment.execute(IncrementVal, Live, Dst) { result = erts_gc_mixed_plus(c_p, reg, live); HEAVY_SWAPIN; ERTS_HOLE_CHECK(c_p); - if (is_value(result)) { + if (ERTS_LIKELY(is_value(result))) { $REFRESH_GEN_DEST(); $Dst = result; $NEXT0(); @@ -137,12 +134,12 @@ i_m_div(Fail, Live, Op1, Op2, Dst) { i_int_div(Fail, Live, Op1, Op2, Dst) { Eterm op1 = $Op1; Eterm op2 = $Op2; - if (op2 == SMALL_ZERO) { + if (ERTS_UNLIKELY(op2 == SMALL_ZERO)) { c_p->freason = BADARITH; $BIF_ERROR_ARITY_2($Fail, BIF_intdiv_2, op1, op2); - } else if (is_both_small(op1, op2)) { + } else if (ERTS_LIKELY(is_both_small(op1, op2))) { Sint ires = signed_val(op1) / signed_val(op2); - if (MY_IS_SSMALL(ires)) { + if (ERTS_LIKELY(IS_SSMALL(ires))) { $Dst = make_small(ires); $NEXT0(); } @@ -162,15 +159,15 @@ rem.fetch(Src1, Src2) { } rem.execute(Fail, Live, Dst) { - if (RemOp2 == SMALL_ZERO) { - c_p->freason = BADARITH; - $BIF_ERROR_ARITY_2($Fail, BIF_rem_2, RemOp1, RemOp2); - } else if (is_both_small(RemOp1, RemOp2)) { - $Dst = make_small(signed_val(RemOp1) % signed_val(RemOp2)); - $NEXT0(); - } else { - $OUTLINED_ARITH_2($Fail, $Live, int_rem, BIF_rem_2, RemOp1, RemOp2, $Dst); - } + if (ERTS_UNLIKELY(RemOp2 == SMALL_ZERO)) { + c_p->freason = BADARITH; + $BIF_ERROR_ARITY_2($Fail, BIF_rem_2, RemOp1, RemOp2); + } else if (ERTS_LIKELY(is_both_small(RemOp1, RemOp2))) { + $Dst = make_small(signed_val(RemOp1) % signed_val(RemOp2)); + $NEXT0(); + } else { + $OUTLINED_ARITH_2($Fail, $Live, int_rem, BIF_rem_2, RemOp1, RemOp2, $Dst); + } } i_band := band.fetch.execute; @@ -185,7 +182,7 @@ band.fetch(Src1, Src2) { } band.execute(Fail, Live, Dst) { - if (is_both_small(BandOp1, BandOp2)) { + if (ERTS_LIKELY(is_both_small(BandOp1, BandOp2))) { /* * No need to untag -- TAG & TAG == TAG. */ @@ -196,7 +193,7 @@ band.execute(Fail, Live, Dst) { } i_bor(Fail, Live, Src1, Src2, Dst) { - if (is_both_small($Src1, $Src2)) { + if (ERTS_LIKELY(is_both_small($Src1, $Src2))) { /* * No need to untag -- TAG | TAG == TAG. */ @@ -207,7 +204,7 @@ i_bor(Fail, Live, Src1, Src2, Dst) { } i_bxor(Fail, Live, Src1, Src2, Dst) { - if (is_both_small($Src1, $Src2)) { + if (ERTS_LIKELY(is_both_small($Src1, $Src2))) { /* * TAG ^ TAG == 0. * @@ -226,17 +223,13 @@ i_bsr := shift.setup_bsr.execute; shift.head() { Eterm Op1, Op2; Sint shift_left_count; - Sint ires; - Eterm* bigp; - Eterm tmp_big[2]; - Uint BIF; } shift.setup_bsr(Src1, Src2) { Op1 = $Src1; Op2 = $Src2; - BIF = BIF_bsr_2; - if (is_small(Op2)) { + shift_left_count = 0; + if (ERTS_LIKELY(is_small(Op2))) { shift_left_count = -signed_val(Op2); } else if (is_big(Op2)) { /* @@ -245,16 +238,14 @@ shift.setup_bsr(Src1, Src2) { */ shift_left_count = make_small(bignum_header_is_neg(*big_val(Op2)) ? MAX_SMALL : MIN_SMALL); - } else { - shift_left_count = 0; } } shift.setup_bsl(Src1, Src2) { Op1 = $Src1; Op2 = $Src2; - BIF = BIF_bsl_2; - if (is_small(Op2)) { + shift_left_count = 0; + if (ERTS_LIKELY(is_small(Op2))) { shift_left_count = signed_val(Op2); } else if (is_big(Op2)) { if (bignum_header_is_neg(*big_val(Op2))) { @@ -271,66 +262,65 @@ shift.setup_bsl(Src1, Src2) { */ shift_left_count = MAX_SMALL; } - } else { - shift_left_count = 0; } } shift.execute(Fail, Live, Dst) { - if (is_small(Op1)) { - ires = signed_val(Op1); - if (shift_left_count == 0 || ires == 0) { - if (is_not_integer(Op2)) { - c_p->freason = BADARITH; - $BIF_ERROR_ARITY_2($Fail, BIF, Op1, Op2); + Uint big_words_needed; + + if (ERTS_LIKELY(is_small(Op1))) { + Sint int_res = signed_val(Op1); + if (ERTS_UNLIKELY(shift_left_count == 0 || int_res == 0)) { + if (ERTS_UNLIKELY(is_not_integer(Op2))) { + goto shift_error; } - if (ires == 0) { + if (int_res == 0) { $Dst = Op1; $NEXT0(); } } else if (shift_left_count < 0) { /* Right shift */ + Eterm bsr_res; shift_left_count = -shift_left_count; if (shift_left_count >= SMALL_BITS-1) { - $Dst = (ires < 0) ? SMALL_MINUS_ONE : SMALL_ZERO; + bsr_res = (int_res < 0) ? SMALL_MINUS_ONE : SMALL_ZERO; } else { - $Dst = make_small(ires >> shift_left_count); + bsr_res = make_small(int_res >> shift_left_count); } + $Dst = bsr_res; $NEXT0(); } else if (shift_left_count < SMALL_BITS-1) { /* Left shift */ - if ((ires > 0 && - ((~(Uint)0 << ((SMALL_BITS-1)-shift_left_count)) & - ires) == 0) || - ((~(Uint)0 << ((SMALL_BITS-1)-shift_left_count)) & - ~ires) == 0) { - $Dst = make_small(ires << shift_left_count); + if ((int_res > 0 && + ((~(Uint)0 << ((SMALL_BITS-1)-shift_left_count)) & int_res) == 0) || + ((~(Uint)0 << ((SMALL_BITS-1)-shift_left_count)) & ~int_res) == 0) { + $Dst = make_small(int_res << shift_left_count); $NEXT0(); } } - ires = 1; /* big_size(small_to_big(Op1)) */ + big_words_needed = 1; /* big_size(small_to_big(Op1)) */ goto big_shift; } else if (is_big(Op1)) { if (shift_left_count == 0) { if (is_not_integer(Op2)) { - c_p->freason = BADARITH; - $BIF_ERROR_ARITY_2($Fail, BIF, Op1, Op2); + goto shift_error; } $Dst = Op1; $NEXT0(); } - ires = big_size(Op1); + big_words_needed = big_size(Op1); big_shift: if (shift_left_count > 0) { /* Left shift. */ - ires += (shift_left_count / D_EXP); + big_words_needed += (shift_left_count / D_EXP); } else { /* Right shift. */ - if (ires <= (-shift_left_count / D_EXP)) { - ires = 3; /* ??? */ + if (big_words_needed <= (-shift_left_count / D_EXP)) { + big_words_needed = 3; /* ??? */ } else { - ires -= (-shift_left_count / D_EXP); + big_words_needed -= (-shift_left_count / D_EXP); } } { - ires = BIG_NEED_SIZE(ires+1); + Eterm tmp_big[2]; + Sint big_need_size = BIG_NEED_SIZE(big_words_needed+1); /* * Slightly conservative check the size to avoid @@ -338,20 +328,19 @@ shift.execute(Fail, Live, Dst) { * clearly would overflow the arity in the header * word. */ - if (ires-8 > BIG_ARITY_MAX) { + if (big_need_size-8 > BIG_ARITY_MAX) { $SYSTEM_LIMIT($Fail); } - $GC_TEST_PRESERVE(ires+1, $Live, Op1); + $GC_TEST_PRESERVE(big_need_size+1, $Live, Op1); if (is_small(Op1)) { Op1 = small_to_big(signed_val(Op1), tmp_big); } - bigp = HTOP; - Op1 = big_lshift(Op1, shift_left_count, bigp); + Op1 = big_lshift(Op1, shift_left_count, HTOP); if (is_big(Op1)) { HTOP += bignum_header_arity(*HTOP) + 1; } HEAP_SPACE_VERIFIED(0); - if (is_nil(Op1)) { + if (ERTS_UNLIKELY(is_nil(Op1))) { /* * This result must have been only slighty larger * than allowed since it wasn't caught by the @@ -369,13 +358,27 @@ shift.execute(Fail, Live, Dst) { /* * One or more non-integer arguments. */ + shift_error: c_p->freason = BADARITH; - $BIF_ERROR_ARITY_2($Fail, BIF, Op1, Op2); + if ($Fail) { + $FAIL($Fail); + } else { + reg[0] = Op1; + reg[1] = Op2; + SWAPOUT; + if (IsOpCode(I[0], i_bsl_ssjtd)) { + I = handle_error(c_p, I, reg, &bif_export[BIF_bsl_2]->info.mfa); + } else { + ASSERT(IsOpCode(I[0], i_bsr_ssjtd)); + I = handle_error(c_p, I, reg, &bif_export[BIF_bsr_2]->info.mfa); + } + goto post_error_handling; + } } i_int_bnot(Fail, Src, Live, Dst) { Eterm bnot_val = $Src; - if (is_small(bnot_val)) { + if (ERTS_LIKELY(is_small(bnot_val))) { bnot_val = make_small(~signed_val(bnot_val)); } else { Uint live = $Live; @@ -384,7 +387,7 @@ i_int_bnot(Fail, Src, Live, Dst) { bnot_val = erts_gc_bnot(c_p, reg, live); HEAVY_SWAPIN; ERTS_HOLE_CHECK(c_p); - if (is_nil(bnot_val)) { + if (ERTS_UNLIKELY(is_nil(bnot_val))) { $BIF_ERROR_ARITY_1($Fail, BIF_bnot_1, reg[live]); } $REFRESH_GEN_DEST(); diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index 1b691386eb..bbe1cb3e11 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -136,7 +136,7 @@ atom_hash(Atom* obj) while(len--) { v = *p++; /* latin1 clutch for r16 */ - if ((v & 0xFE) == 0xC2 && (*p & 0xC0) == 0x80) { + if (len && (v & 0xFE) == 0xC2 && (*p & 0xC0) == 0x80) { v = (v << 6) | (*p & 0x3F); p++; len--; } diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index a44d23b181..fc55b687d4 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -217,6 +217,8 @@ atom discard atom display_items atom dist atom dist_cmd +atom dist_ctrl_put_data +atom dist_data atom Div='/' atom div atom dlink diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index b78f617560..00822d1415 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -65,9 +65,7 @@ static struct { Process *erts_code_purger = NULL; -#ifdef ERTS_DIRTY_SCHEDULERS Process *erts_dirty_process_code_checker; -#endif erts_atomic_t erts_copy_literal_area__; #define ERTS_SET_COPY_LITERAL_AREA(LA) \ erts_atomic_set_nob(&erts_copy_literal_area__, \ @@ -261,7 +259,7 @@ struct m { Binary* code; Eterm module; Module* modp; - Uint exception; + Eterm exception; }; static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int, int); @@ -280,7 +278,7 @@ exception_list(Process* p, Eterm tag, struct m* mp, Sint exceptions) Eterm res = NIL; while (exceptions > 0) { - if (mp->exception) { + if (is_value(mp->exception)) { res = CONS(hp, mp->module, res); hp += 2; exceptions--; @@ -381,9 +379,9 @@ finish_loading_1(BIF_ALIST_1) exceptions = 0; for (i = 0; i < n; i++) { - p[i].exception = 0; + p[i].exception = THE_NON_VALUE; if (p[i].modp->seen) { - p[i].exception = 1; + p[i].exception = am_duplicated; exceptions++; } p[i].modp->seen = 1; @@ -417,9 +415,9 @@ finish_loading_1(BIF_ALIST_1) exceptions = 0; for (i = 0; i < n; i++) { - p[i].exception = 0; + p[i].exception = THE_NON_VALUE; if (p[i].modp->curr.code_hdr && p[i].modp->old.code_hdr) { - p[i].exception = 1; + p[i].exception = am_not_purged; exceptions++; } } @@ -440,7 +438,7 @@ finish_loading_1(BIF_ALIST_1) retval = erts_finish_loading(p[i].code, BIF_P, 0, &mod); ASSERT(retval == NIL || retval == am_on_load); if (retval == am_on_load) { - p[i].exception = 1; + p[i].exception = am_on_load; exceptions++; } } @@ -471,7 +469,8 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, erts_commit_staging_code_ix(); for (i=0; i < nmods; i++) { - if (mods[i].modp->curr.code_hdr) { + if (mods[i].modp->curr.code_hdr + && mods[i].exception != am_on_load) { set_default_trace_pattern(mods[i].module); } #ifdef HIPE @@ -604,9 +603,6 @@ badarg: BIF_RETTYPE erts_internal_check_dirty_process_code_2(BIF_ALIST_2) { -#if !defined(ERTS_DIRTY_SCHEDULERS) - BIF_ERROR(BIF_P, EXC_NOTSUP); -#else Process *rp; int reds = 0; Eterm res; @@ -636,7 +632,6 @@ BIF_RETTYPE erts_internal_check_dirty_process_code_2(BIF_ALIST_2) ASSERT(is_value(res)); BIF_RET2(res, reds); -#endif } BIF_RETTYPE delete_module_1(BIF_ALIST_1) @@ -692,6 +687,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) Eterm retval; mod.module = BIF_ARG_1; mod.modp = modp; + mod.exception = THE_NON_VALUE; retval = staging_epilogue(BIF_P, success, res, is_blocking, &mod, 1, 0); return retval; } @@ -827,11 +823,11 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) ep->beam[1] = 0; } else { if (ep->addressv[code_ix] == ep->beam && - ep->beam[0] == (BeamInstr) em_apply_bif) { + BeamIsOpCode(ep->beam[0], op_apply_bif)) { continue; } - ep->addressv[code_ix] = ep->beam; - ep->beam[0] = (BeamInstr) em_call_error_handler; + ep->addressv[code_ix] = ep->beam; + ep->beam[0] = BeamOpCodeAddr(op_call_error_handler); } } modp->curr.code_hdr->on_load_function_ptr = NULL; @@ -853,7 +849,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { continue; } - if (ep->beam[0] == (BeamInstr) em_apply_bif) { + if (BeamIsOpCode(ep->beam[0], op_apply_bif)) { continue; } ep->beam[1] = 0; @@ -1053,10 +1049,8 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed return_ok: -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p))) c_p->flags &= ~F_DIRTY_CLA; -#endif return am_ok; @@ -1071,10 +1065,8 @@ literal_gc: *redsp += erts_garbage_collect_literals(c_p, (Eterm *) literals, lit_bsize, oh, fcalls); -#ifdef ERTS_DIRTY_SCHEDULERS if (c_p->flags & F_DIRTY_CLA) return THE_NON_VALUE; -#endif return am_ok; } @@ -1774,22 +1766,23 @@ delete_code(Module* modp) Export *ep = export_list(i, code_ix); if (ep != NULL && (ep->info.mfa.module == module)) { if (ep->addressv[code_ix] == ep->beam) { - if (ep->beam[0] == (BeamInstr) em_apply_bif) { + if (BeamIsOpCode(ep->beam[0], op_apply_bif)) { continue; } - else if (ep->beam[0] == - (BeamInstr) BeamOp(op_i_generic_breakpoint)) { + else if (BeamIsOpCode(ep->beam[0], op_i_generic_breakpoint)) { ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); ASSERT(modp->curr.num_traced_exports > 0); DBG_TRACE_MFA_P(&ep->info.mfa, "export trace cleared, code_ix=%d", code_ix); erts_clear_export_break(modp, &ep->info); } - else ASSERT(ep->beam[0] == (BeamInstr) em_call_error_handler - || !erts_initialized); - } + else { + ASSERT(BeamIsOpCode(ep->beam[0], op_call_error_handler) || + !erts_initialized); + } + } ep->addressv[code_ix] = ep->beam; - ep->beam[0] = (BeamInstr) em_call_error_handler; + ep->beam[0] = BeamOpCodeAddr(op_call_error_handler); ep->beam[1] = 0; DBG_TRACE_MFA_P(&ep->info.mfa, "export invalidation, code_ix=%d", code_ix); diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 79a75f6698..fe1e15701b 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -75,9 +75,7 @@ extern BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ erts_atomic32_t erts_active_bp_index; erts_atomic32_t erts_staging_bp_index; -#ifdef ERTS_DIRTY_SCHEDULERS erts_mtx_t erts_dirty_bp_ix_mtx; -#endif /* * Inlined helpers @@ -94,22 +92,18 @@ acquire_bp_sched_ix(Process *c_p) { ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); ASSERT(esdp); -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { erts_mtx_lock(&erts_dirty_bp_ix_mtx); return (Uint32) erts_no_schedulers; } -#endif return (Uint32) esdp->no - 1; } static ERTS_INLINE void release_bp_sched_ix(Uint32 ix) { -#ifdef ERTS_DIRTY_SCHEDULERS if (ix == (Uint32) erts_no_schedulers) erts_mtx_unlock(&erts_dirty_bp_ix_mtx); -#endif } @@ -164,10 +158,8 @@ void erts_bp_init(void) { erts_atomic32_init_nob(&erts_active_bp_index, 0); erts_atomic32_init_nob(&erts_staging_bp_index, 1); -#ifdef ERTS_DIRTY_SCHEDULERS erts_mtx_init(&erts_dirty_bp_ix_mtx, "dirty_break_point_index", NIL, ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG); -#endif } @@ -211,7 +203,7 @@ erts_bp_match_functions(BpFunctions* f, ErtsCodeMFA *mfa, int specified) for (fi = 0; fi < num_functions; fi++) { ci = code_hdr->functions[fi]; - ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI)); if (erts_is_function_native(ci)) { continue; } @@ -273,11 +265,11 @@ erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified) pc = ep->beam; if (ep->addressv[code_ix] == pc) { - if ((*pc == (BeamInstr) em_apply_bif || - *pc == (BeamInstr) em_call_error_handler)) { - continue; + if (BeamIsOpCode(*pc, op_apply_bif) || + BeamIsOpCode(*pc, op_call_error_handler)) { + continue; } - ASSERT(*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)); + ASSERT(BeamIsOpCode(*pc, op_i_generic_breakpoint)); } else if (erts_is_function_native(erts_code_to_codeinfo(ep->addressv[code_ix]))) { continue; } @@ -374,8 +366,8 @@ consolidate_bp_data(Module* modp, ErtsCodeInfo *ci, int local) } ASSERT(modp->curr.num_breakpoints >= 0); ASSERT(modp->curr.num_traced_exports >= 0); - ASSERT(*erts_codeinfo_to_code(ci) != - (BeamInstr) BeamOp(op_i_generic_breakpoint)); + ASSERT(! BeamIsOpCode(*erts_codeinfo_to_code(ci), + op_i_generic_breakpoint)); } ci->u.gen_bp = NULL; Free(g); @@ -423,13 +415,15 @@ erts_install_breakpoints(BpFunctions* f) { Uint i; Uint n = f->matched; - BeamInstr br = (BeamInstr) BeamOp(op_i_generic_breakpoint); + BeamInstr br = BeamOpCodeAddr(op_i_generic_breakpoint); for (i = 0; i < n; i++) { ErtsCodeInfo* ci = f->matching[i].ci; - BeamInstr *pc = erts_codeinfo_to_code(ci); GenericBp* g = ci->u.gen_bp; - if (*pc != br && g) { + BeamInstr volatile *pc = erts_codeinfo_to_code(ci); + BeamInstr instr = *pc; + + if (!BeamIsOpCode(instr, op_i_generic_breakpoint) && g) { Module* modp = f->matching[i].mod; /* @@ -443,11 +437,16 @@ erts_install_breakpoints(BpFunctions* f) /* * The following write is not protected by any lock. We * assume that the hardware guarantees that a write of an - * aligned word-size (or half-word) writes is atomic - * (i.e. that other processes executing this code will not - * see a half pointer). + * aligned word-size writes is atomic (i.e. that other + * processes executing this code will not see a half + * pointer). + * + * The contents of *pc is marked 'volatile' to ensure that + * the compiler will do a single full-word write, and not + * try any fancy optimizations to write a half word. */ - *pc = br; + instr = BeamSetCodeAddr(instr, br); + *pc = instr; modp->curr.num_breakpoints++; } } @@ -468,7 +467,7 @@ static void uninstall_breakpoint(ErtsCodeInfo *ci) { BeamInstr *pc = erts_codeinfo_to_code(ci); - if (*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { + if (BeamIsOpCode(*pc, op_i_generic_breakpoint)) { GenericBp* g = ci->u.gen_bp; if (g->data[erts_active_bp_ix()].flags == 0) { /* @@ -642,6 +641,49 @@ erts_clear_export_break(Module* modp, ErtsCodeInfo *ci) ASSERT(ci->u.gen_bp == NULL); } +/* + * If c_p->cp is a trace return instruction, we set cp + * to be the place where we again start to execute code. + * + * cp is used by match spec {caller} to get the calling + * function, and if we don't do this fixup it will be + * 'undefined'. This has the odd side effect of {caller} + * not really being which function is the caller, but + * rather which function we are about to return to. + */ +static void fixup_cp_before_trace(Process *c_p, int *return_to_trace) +{ + Eterm *cpp, *E = c_p->stop; + BeamInstr w = *c_p->cp; + if (BeamIsOpCode(w, op_return_trace)) { + cpp = &E[2]; + } else if (BeamIsOpCode(w, op_i_return_to_trace)) { + *return_to_trace = 1; + cpp = &E[0]; + } else if (BeamIsOpCode(w, op_i_return_time_trace)) { + cpp = &E[0]; + } else { + cpp = NULL; + } + if (cpp) { + for (;;) { + BeamInstr w = *cp_val(*cpp); + if (BeamIsOpCode(w, op_return_trace)) { + cpp += 3; + } else if (BeamIsOpCode(w, op_i_return_to_trace)) { + *return_to_trace = 1; + cpp += 1; + } else if (BeamIsOpCode(w, op_i_return_time_trace)) { + cpp += 2; + } else { + break; + } + } + c_p->cp = (BeamInstr *) cp_val(*cpp); + ASSERT(is_CP(*cpp)); + } +} + BeamInstr erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg) { @@ -650,7 +692,7 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg) Uint bp_flags; ErtsBpIndex ix = erts_active_bp_ix(); - ASSERT(info->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + ASSERT(BeamIsOpCode(info->op, op_i_func_info_IaaI)); g = info->u.gen_bp; bp = &g->data[ix]; @@ -703,9 +745,9 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg) Eterm w; erts_trace_time_call(c_p, info, bp->time); w = (BeamInstr) *c_p->cp; - if (! (w == (BeamInstr) BeamOp(op_i_return_time_trace) || - w == (BeamInstr) BeamOp(op_return_trace) || - w == (BeamInstr) BeamOp(op_i_return_to_trace)) ) { + if (! (BeamIsOpCode(w, op_i_return_time_trace) || + BeamIsOpCode(w, op_return_trace) || + BeamIsOpCode(w, op_i_return_to_trace)) ) { Eterm* E = c_p->stop; ASSERT(c_p->htop <= E && E <= c_p->hend); if (E - 2 < c_p->htop) { @@ -725,7 +767,7 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg) } if (bp_flags & ERTS_BPF_DEBUG) { - return (BeamInstr) BeamOp(op_i_debug_breakpoint); + return BeamOpCodeAddr(op_i_debug_breakpoint); } else { return g->orig_instr; } @@ -752,6 +794,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) GenericBp* g; GenericBpData* bp = NULL; Uint bp_flags = 0; + int return_to_trace = 0; ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); @@ -767,6 +810,8 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) */ if (!applying) { p->cp = I; + } else { + fixup_cp_before_trace(p, &return_to_trace); } if (bp_flags & (ERTS_BPF_LOCAL_TRACE|ERTS_BPF_GLOBAL_TRACE) && IS_TRACED_FL(p, F_TRACE_CALLS)) { @@ -945,49 +990,18 @@ static ErtsTracer do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg, int local, Binary* ms, ErtsTracer tracer) { - Eterm* cpp; int return_to_trace = 0; - BeamInstr w; BeamInstr *cp_save = c_p->cp; Uint32 flags; Uint need = 0; Eterm* E = c_p->stop; - w = *c_p->cp; - if (w == (BeamInstr) BeamOp(op_return_trace)) { - cpp = &E[2]; - } else if (w == (BeamInstr) BeamOp(op_i_return_to_trace)) { - return_to_trace = 1; - cpp = &E[0]; - } else if (w == (BeamInstr) BeamOp(op_i_return_time_trace)) { - cpp = &E[0]; - } else { - cpp = NULL; - } - if (cpp) { - for (;;) { - BeamInstr w = *cp_val(*cpp); - if (w == (BeamInstr) BeamOp(op_return_trace)) { - cpp += 3; - } else if (w == (BeamInstr) BeamOp(op_i_return_to_trace)) { - return_to_trace = 1; - cpp += 1; - } else if (w == (BeamInstr) BeamOp(op_i_return_time_trace)) { - cpp += 2; - } else { - break; - } - } - cp_save = c_p->cp; - c_p->cp = (BeamInstr *) cp_val(*cpp); - ASSERT(is_CP(*cpp)); - } - ERTS_UNREQ_PROC_MAIN_LOCK(c_p); + fixup_cp_before_trace(c_p, &return_to_trace); + flags = erts_call_trace(c_p, info, ms, reg, local, &tracer); - ERTS_REQ_PROC_MAIN_LOCK(c_p); - if (cpp) { - c_p->cp = cp_save; - } + + /* restore cp after potential fixup */ + c_p->cp = cp_save; ASSERT(!ERTS_PROC_IS_EXITING(c_p)); if ((flags & MATCH_SET_RETURN_TO_TRACE) && !return_to_trace) { @@ -1301,7 +1315,7 @@ erts_find_local_func(ErtsCodeMFA *mfa) { n = (BeamInstr) code_hdr->num_functions; for (i = 0; i < n; ++i) { ci = code_hdr->functions[i]; - ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == ci->op); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI)); ASSERT(mfa->module == ci->mfa.module || is_nil(ci->mfa.module)); if (mfa->function == ci->mfa.function && mfa->arity == ci->mfa.arity) { @@ -1585,11 +1599,7 @@ set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags, ASSERT((bp->flags & ERTS_BPF_TIME_TRACE) == 0); bdt = Alloc(sizeof(BpDataTime)); erts_refc_init(&bdt->refc, 1); -#ifdef ERTS_DIRTY_SCHEDULERS bdt->n = erts_no_schedulers + 1; -#else - bdt->n = erts_no_schedulers; -#endif bdt->hash = Alloc(sizeof(bp_time_hash_t)*(bdt->n)); for (i = 0; i < bdt->n; i++) { bp_hash_init(&(bdt->hash[i]), 32); @@ -1720,7 +1730,7 @@ check_break(ErtsCodeInfo *ci, Uint break_flags) { GenericBp* g = ci->u.gen_bp; - ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI)); if (erts_is_function_native(ci)) { return 0; } diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 1e1f6a7534..a64765822b 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -79,9 +79,7 @@ typedef struct generic_bp { #define ERTS_BP_CALL_TIME_SCHEDULE_OUT (1) #define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2) -#ifdef ERTS_DIRTY_SCHEDULERS extern erts_mtx_t erts_dirty_bp_ix_mtx; -#endif enum erts_break_op{ ERTS_BREAK_NOP = 0, /* Must be false */ diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index afe87288ce..70078c8c59 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -53,6 +53,8 @@ void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg); static int print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr); static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif); +static BeamInstr* f_to_addr(BeamInstr* base, int op, BeamInstr* ap); +static BeamInstr* f_to_addr_packed(BeamInstr* base, int op, Sint32* ap); BIF_RETTYPE erts_debug_same_2(BIF_ALIST_2) @@ -197,9 +199,9 @@ void debug_dump_code(BeamInstr *I, int num) erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr); instr = (BeamInstr) code_ptr[0]; for (i = 0; i < NUM_SPECIFIC_OPS; i++) { - if (instr == (BeamInstr) BeamOp(i) && opc[i].name[0] != '\0') { + if (BeamIsOpCode(instr, i) && opc[i].name[0] != '\0') { code_ptr += print_op(ERTS_PRINT_DSBUF, (void *) dsbufp, - i, opc[i].sz-1, code_ptr+1) + 1; + i, opc[i].sz-1, code_ptr) + 1; break; } } @@ -317,9 +319,9 @@ erts_debug_disassemble_1(BIF_ALIST_1) erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr); instr = (BeamInstr) code_ptr[0]; for (i = 0; i < NUM_SPECIFIC_OPS; i++) { - if (instr == (BeamInstr) BeamOp(i) && opc[i].name[0] != '\0') { + if (BeamIsOpCode(instr, i) && opc[i].name[0] != '\0') { code_ptr += print_op(ERTS_PRINT_DSBUF, (void *) dsbufp, - i, opc[i].sz-1, code_ptr+1) + 1; + i, opc[i].sz-1, code_ptr) + 1; break; } } @@ -403,8 +405,11 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) * Avoid copying because instructions containing bignum operands * are bigger than actually declared. */ - ap = (BeamInstr *) addr; + addr++; + ap = addr; } else { + BeamInstr instr_word = addr++[0]; + /* * Copy all arguments to a local buffer for the unpacking. */ @@ -424,26 +429,27 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) while (start_prog < prog) { prog--; switch (*prog) { + case 'f': case 'g': + case 'q': *ap++ = *--sp; break; - case 'i': /* Initialize packing accumulator. */ - *ap++ = packed; - break; - case 's': - *ap++ = packed & 0x3ff; - packed >>= 10; +#ifdef ARCH_64 + case '1': /* Tightest shift */ + *ap++ = (packed & BEAM_TIGHTEST_MASK) << 3; + packed >>= BEAM_TIGHTEST_SHIFT; break; - case '0': /* Tight shift */ +#endif + case '2': /* Tight shift */ *ap++ = packed & BEAM_TIGHT_MASK; packed >>= BEAM_TIGHT_SHIFT; break; - case '6': /* Shift 16 steps */ + case '3': /* Loose shift */ *ap++ = packed & BEAM_LOOSE_MASK; packed >>= BEAM_LOOSE_SHIFT; break; #ifdef ARCH_64 - case 'w': /* Shift 32 steps */ + case '4': /* Shift 32 steps */ *ap++ = packed & BEAM_WIDE_MASK; packed >>= BEAM_WIDE_SHIFT; break; @@ -454,8 +460,18 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) case 'P': packed = *--sp; break; +#if defined(ARCH_64) && defined(CODE_MODEL_SMALL) + case '#': /* -1 */ + case '$': /* -2 */ + case '%': /* -3 */ + case '&': /* -4 */ + case '\'': /* -5 */ + case '(': /* -6 */ + packed = (packed << BEAM_WIDE_SHIFT) | BeamExtraData(instr_word); + break; +#endif default: - ASSERT(0); + erts_exit(ERTS_ERROR_EXIT, "beam_debug: invalid packing op: %c\n", *prog); } } ap = args; @@ -489,6 +505,14 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) case 'n': /* Nil */ erts_print(to, to_arg, "[]"); break; + case 'S': /* Register */ + { + Uint reg_type = (*ap & 1) ? 'y' : 'x'; + Uint n = ap[0] / sizeof(Eterm); + erts_print(to, to_arg, "%c(%d)", reg_type, n); + ap++; + break; + } case 's': /* Any source (tagged constant or register) */ tag = loader_tag(*ap); if (tag == LOADER_X_REG) { @@ -522,12 +546,13 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) } ap++; break; - case 'I': /* Untagged integer. */ - case 't': + case 't': /* Untagged integers */ + case 'I': + case 'W': switch (op) { - case op_i_gc_bif1_jIsId: - case op_i_gc_bif2_jIIssd: - case op_i_gc_bif3_jIIssd: + case op_i_gc_bif1_jWstd: + case op_i_gc_bif2_jWtssd: + case op_i_gc_bif3_jWtssd: { const ErtsGcBif* p; BifFunction gcf = (BifFunction) *ap; @@ -549,9 +574,10 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) break; case 'f': /* Destination label */ { - ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap); - if (!cmfa || erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) { - erts_print(to, to_arg, "f(" HEXF ")", *ap); + BeamInstr* target = f_to_addr(addr, op, ap); + ErtsCodeMFA* cmfa = find_function_from_pc(target); + if (!cmfa || erts_codemfa_to_code(cmfa) != target) { + erts_print(to, to_arg, "f(" HEXF ")", target); } else { erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module, cmfa->function, cmfa->arity); @@ -561,18 +587,18 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) break; case 'p': /* Pointer (to label) */ { - ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap); - if (!cmfa || erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) { - erts_print(to, to_arg, "p(" HEXF ")", *ap); - } else { - erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module, - cmfa->function, cmfa->arity); - } + BeamInstr* target = f_to_addr(addr, op, ap); + erts_print(to, to_arg, "p(" HEXF ")", target); ap++; } break; case 'j': /* Pointer (to label) */ - erts_print(to, to_arg, "j(" HEXF ")", *ap); + if (*ap == 0) { + erts_print(to, to_arg, "j(0)"); + } else { + BeamInstr* target = f_to_addr(addr, op, ap); + erts_print(to, to_arg, "j(" HEXF ")", target); + } ap++; break; case 'e': /* Export entry */ @@ -615,12 +641,22 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) unpacked = ap; ap = addr + size; + + /* + * In the code below, never use ap[-1], ap[-2], ... + * (will not work if the arguments have been packed). + * + * Instead use unpacked[-1], unpacked[-2], ... + */ switch (op) { case op_i_select_val_lins_xfI: case op_i_select_val_lins_yfI: + case op_i_select_val_bins_xfI: + case op_i_select_val_bins_yfI: { - int n = ap[-1]; + int n = unpacked[-1]; int ix = n; + Sint32* jump_tab = (Sint32 *)(ap + n); while (ix--) { erts_print(to, to_arg, "%T ", (Eterm) ap[0]); @@ -629,30 +665,19 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) } ix = n; while (ix--) { - erts_print(to, to_arg, "f(" HEXF ") ", (Eterm) ap[0]); - ap++; - size++; - } - } - break; - case op_i_select_val_bins_xfI: - case op_i_select_val_bins_yfI: - { - int n = ap[-1]; - - while (n > 0) { - erts_print(to, to_arg, "%T f(" HEXF ") ", (Eterm) ap[0], ap[1]); - ap += 2; - size += 2; - n--; + BeamInstr* target = f_to_addr_packed(addr, op, jump_tab); + erts_print(to, to_arg, "f(" HEXF ") ", target); + jump_tab++; } + size += (n+1) / 2; } break; case op_i_select_tuple_arity_xfI: case op_i_select_tuple_arity_yfI: { - int n = ap[-1]; + int n = unpacked[-1]; int ix = n - 1; /* without sentinel */ + Sint32* jump_tab = (Sint32 *)(ap + n); while (ix--) { Uint arity = arityval(ap[0]); @@ -666,39 +691,62 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) size++; ix = n; while (ix--) { - erts_print(to, to_arg, "f(" HEXF ") ", ap[0]); - ap++; - size++; + BeamInstr* target = f_to_addr_packed(addr, op, jump_tab); + erts_print(to, to_arg, "f(" HEXF ") ", target); + jump_tab++; + } + size += (n+1) / 2; + } + break; + case op_i_select_val2_xfcc: + case op_i_select_val2_yfcc: + case op_i_select_tuple_arity2_xfAA: + case op_i_select_tuple_arity2_yfAA: + { + Sint32* jump_tab = (Sint32 *) ap; + BeamInstr* target; + int i; + + for (i = 0; i < 2; i++) { + target = f_to_addr_packed(addr, op, jump_tab++); + erts_print(to, to_arg, "f(" HEXF ") ", target); } + size += 1; } break; - case op_i_jump_on_val_xfII: - case op_i_jump_on_val_yfII: + case op_i_jump_on_val_xfIW: + case op_i_jump_on_val_yfIW: { - int n; - for (n = ap[-2]; n > 0; n--) { - erts_print(to, to_arg, "f(" HEXF ") ", ap[0]); - ap++; - size++; + int n = unpacked[-2]; + Sint32* jump_tab = (Sint32 *) ap; + + size += (n+1) / 2; + while (n-- > 0) { + BeamInstr* target = f_to_addr_packed(addr, op, jump_tab); + erts_print(to, to_arg, "f(" HEXF ") ", target); + jump_tab++; } } break; case op_i_jump_on_val_zero_xfI: case op_i_jump_on_val_zero_yfI: { - int n; - for (n = ap[-1]; n > 0; n--) { - erts_print(to, to_arg, "f(" HEXF ") ", ap[0]); - ap++; - size++; + int n = unpacked[-1]; + Sint32* jump_tab = (Sint32 *) ap; + + size += (n+1) / 2; + while (n-- > 0) { + BeamInstr* target = f_to_addr_packed(addr, op, jump_tab); + erts_print(to, to_arg, "f(" HEXF ") ", target); + jump_tab++; } } break; case op_i_put_tuple_xI: case op_i_put_tuple_yI: - case op_new_map_dII: - case op_update_map_assoc_sdII: - case op_update_map_exact_jsdII: + case op_new_map_dtI: + case op_update_map_assoc_sdtI: + case op_update_map_exact_jsdtI: { int n = unpacked[-1]; @@ -708,7 +756,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) erts_print(to, to_arg, " x(%d)", loader_x_reg_index(ap[0])); break; case LOADER_Y_REG: - erts_print(to, to_arg, " x(%d)", loader_y_reg_index(ap[0])); + erts_print(to, to_arg, " y(%d)", loader_y_reg_index(ap[0]) - CP_SIZE); break; default: erts_print(to, to_arg, " %T", (Eterm) ap[0]); @@ -718,7 +766,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) } } break; - case op_i_new_small_map_lit_dIq: + case op_i_new_small_map_lit_dtq: { Eterm *tp = tuple_val(unpacked[-1]); int n = arityval(*tp); @@ -729,7 +777,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) erts_print(to, to_arg, " x(%d)", loader_x_reg_index(ap[0])); break; case LOADER_Y_REG: - erts_print(to, to_arg, " x(%d)", loader_y_reg_index(ap[0])); + erts_print(to, to_arg, " y(%d)", loader_y_reg_index(ap[0]) - CP_SIZE); break; default: erts_print(to, to_arg, " %T", (Eterm) ap[0]); @@ -752,7 +800,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) erts_print(to, to_arg, " x(%d)", loader_x_reg_index(ap[0])); break; case LOADER_Y_REG: - erts_print(to, to_arg, " y(%d)", loader_y_reg_index(ap[0])); + erts_print(to, to_arg, " y(%d)", loader_y_reg_index(ap[0]) - CP_SIZE); break; default: erts_print(to, to_arg, " %T", (Eterm) ap[0]); @@ -787,6 +835,17 @@ static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif) } } +static BeamInstr* f_to_addr(BeamInstr* base, int op, BeamInstr* ap) +{ + return base - 1 + opc[op].adjust + (Sint32) *ap; +} + +static BeamInstr* f_to_addr_packed(BeamInstr* base, int op, Sint32* ap) +{ + return base - 1 + opc[op].adjust + *ap; +} + + /* * Dirty BIF testing. * @@ -795,10 +854,8 @@ static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif) * test suite. */ -#ifdef ERTS_DIRTY_SCHEDULERS static int ms_wait(Process *c_p, Eterm etimeout, int busy); static int dirty_send_message(Process *c_p, Eterm to, Eterm tag); -#endif static BIF_RETTYPE dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I); /* @@ -827,7 +884,6 @@ erts_debug_dirty_io_2(BIF_ALIST_2) BIF_RETTYPE erts_debug_dirty_3(BIF_ALIST_3) { -#ifdef ERTS_DIRTY_SCHEDULERS Eterm argv[2]; switch (BIF_ARG_1) { case am_normal: @@ -857,9 +913,6 @@ erts_debug_dirty_3(BIF_ALIST_3) default: BIF_ERROR(BIF_P, EXC_BADARG); } -#else - BIF_ERROR(BIF_P, EXC_UNDEF); -#endif } @@ -867,7 +920,6 @@ static BIF_RETTYPE dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I) { BIF_RETTYPE ret; -#ifdef ERTS_DIRTY_SCHEDULERS if (am_scheduler == arg1) { ErtsSchedulerData *esdp; if (arg2 != am_type) @@ -1053,13 +1105,9 @@ dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I) badarg: ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); } -#else - ERTS_BIF_PREP_ERROR(ret, c_p, EXC_UNDEF); -#endif return ret; } -#ifdef ERTS_DIRTY_SCHEDULERS static int dirty_send_message(Process *c_p, Eterm to, Eterm tag) @@ -1146,7 +1194,6 @@ ms_wait(Process *c_p, Eterm etimeout, int busy) return 1; } -#endif /* ERTS_DIRTY_SCHEDULERS */ # define ERTS_STACK_LIMIT ((char *) ethr_get_stacklimit()) diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index e5935f5f02..aa94fbf536 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -50,15 +50,16 @@ #if defined(NO_JUMP_TABLE) # define OpCase(OpCode) case op_##OpCode # define CountCase(OpCode) case op_count_##OpCode -# define OpCode(OpCode) ((Uint*)op_##OpCode) -# define Goto(Rel) {Go = (int)(UWord)(Rel); goto emulator_loop;} -# define LabelAddr(Addr) &&##Addr +# define IsOpCode(InstrWord, OpCode) (BeamCodeAddr(InstrWord) == (BeamInstr)op_##OpCode) +# define Goto(Rel) {Go = BeamCodeAddr(Rel); goto emulator_loop;} +# define GotoPF(Rel) Goto(Rel) #else # define OpCase(OpCode) lb_##OpCode # define CountCase(OpCode) lb_count_##OpCode -# define Goto(Rel) goto *((void *)Rel) -# define LabelAddr(Label) &&Label -# define OpCode(OpCode) (&&lb_##OpCode) +# define IsOpCode(InstrWord, OpCode) (BeamCodeAddr(InstrWord) == (BeamInstr)&&lb_##OpCode) +# define Goto(Rel) goto *((void *)BeamCodeAddr(Rel)) +# define GotoPF(Rel) goto *((void *)Rel) +# define LabelAddr(Label) &&Label #endif #ifdef ERTS_ENABLE_LOCK_CHECK @@ -107,9 +108,7 @@ do { \ # define CHECK_ARGS(T) #endif -#ifndef MAX -#define MAX(x, y) (((x) > (y)) ? (x) : (y)) -#endif +#define CHECK_ALIGNED(Dst) ASSERT((((Uint)&Dst) & (sizeof(Uint)-1)) == 0) #define GET_BIF_MODULE(p) (p->info.mfa.module) #define GET_BIF_FUNCTION(p) (p->info.mfa.function) @@ -134,11 +133,11 @@ do { \ /* We don't check the range if an ordinary switch is used */ #ifdef NO_JUMP_TABLE -#define VALID_INSTR(IP) ((UWord)(IP) < (NUMBER_OF_OPCODES*2+10)) +# define VALID_INSTR(IP) (BeamCodeAddr(IP) < (NUMBER_OF_OPCODES*2+10)) #else -#define VALID_INSTR(IP) \ - ((SWord)LabelAddr(emulator_loop) <= (SWord)(IP) && \ - (SWord)(IP) < (SWord)LabelAddr(end_emulator_loop)) +# define VALID_INSTR(IP) \ + ((BeamInstr)LabelAddr(emulator_loop) <= BeamCodeAddr(IP) && \ + BeamCodeAddr(IP) < (BeamInstr)LabelAddr(end_emulator_loop)) #endif /* NO_JUMP_TABLE */ #define SET_CP(p, ip) \ @@ -153,10 +152,7 @@ do { \ * Register target (X or Y register). */ -#define REG_TARGET_PTR(Target) (((Target) & 1) ? &yb(Target-1) : &xb(Target)) -#define REG_TARGET(Target) (*REG_TARGET_PTR(Target)) - -#define ISCATCHEND(instr) ((Eterm *) *(instr) == OpCode(catch_end_y)) +#define REG_TARGET_PTR(Target) (((Target) & 1) ? &yb((Target)-1) : &xb(Target)) /* * Special Beam instructions. @@ -166,11 +162,6 @@ BeamInstr beam_apply[2]; BeamInstr beam_exit[1]; BeamInstr beam_continue_exit[1]; -BeamInstr* em_call_error_handler; -BeamInstr* em_apply_bif; -BeamInstr* em_call_nif; -BeamInstr* em_call_bif_e; - /* NOTE These should be the only variables containing trace instructions. ** Sometimes tests are form the instruction value, and sometimes @@ -241,15 +232,22 @@ void** beam_ops; PROCESS_MAIN_CHK_LOCKS((P)); \ ERTS_UNREQ_PROC_MAIN_LOCK((P)) +#define db(N) (N) +#define fb(N) ((Sint)(Sint32)(N)) +#define jb(N) ((Sint)(Sint32)(N)) #define tb(N) (N) -#define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N))) -#define yb(N) (*(Eterm *) (((unsigned char *)E) + (N))) +#define xb(N) (*ADD_BYTE_OFFSET(reg, N)) +#define yb(N) (*ADD_BYTE_OFFSET(E, N)) +#define Sb(N) (*REG_TARGET_PTR(N)) #define lb(N) (*(double *) (((unsigned char *)&(freg[0].fd)) + (N))) #define Qb(N) (N) #define Ib(N) (N) + #define x(N) reg[N] #define y(N) E[N] #define r(N) x(N) +#define Q(N) (N*sizeof(Eterm *)) +#define l(N) (freg[N].fd) /* * Check that we haven't used the reductions and jump to function pointed to by @@ -258,8 +256,8 @@ void** beam_ops; #define DispatchMacro() \ do { \ - BeamInstr* dis_next; \ - dis_next = (BeamInstr *) *I; \ + BeamInstr dis_next; \ + dis_next = *I; \ CHECK_ARGS(I); \ if (FCALLS > 0 || FCALLS > neg_o_reds) { \ FCALLS--; \ @@ -267,12 +265,12 @@ void** beam_ops; } else { \ goto context_switch; \ } \ - } while (0) + } while (0) \ #define DispatchMacroFun() \ do { \ - BeamInstr* dis_next; \ - dis_next = (BeamInstr *) *I; \ + BeamInstr dis_next; \ + dis_next = *I; \ CHECK_ARGS(I); \ if (FCALLS > 0 || FCALLS > neg_o_reds) { \ FCALLS--; \ @@ -282,23 +280,23 @@ void** beam_ops; } \ } while (0) -#define DispatchMacrox() \ - do { \ - if (FCALLS > 0) { \ - Eterm* dis_next; \ - SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \ - dis_next = (Eterm *) *I; \ - FCALLS--; \ - CHECK_ARGS(I); \ - Goto(dis_next); \ - } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p) \ - && FCALLS > neg_o_reds) { \ - goto save_calls1; \ - } else { \ - SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \ - CHECK_ARGS(I); \ - goto context_switch; \ - } \ +#define DispatchMacrox() \ + do { \ + if (FCALLS > 0) { \ + BeamInstr dis_next; \ + SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \ + dis_next = *I; \ + FCALLS--; \ + CHECK_ARGS(I); \ + Goto(dis_next); \ + } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p) \ + && FCALLS > neg_o_reds) { \ + goto save_calls1; \ + } else { \ + SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \ + CHECK_ARGS(I); \ + goto context_switch; \ + } \ } while (0) #ifdef DEBUG @@ -318,10 +316,6 @@ void** beam_ops; #endif #define Arg(N) I[(N)+1] -#define Next(N) \ - I += (N) + 1; \ - ASSERT(VALID_INSTR(*I)); \ - Goto(*I) #define GetR(pos, tr) \ do { \ @@ -389,6 +383,7 @@ do { \ * The following functions are called directly by process_main(). * Don't inline them. */ +static void init_emulator_finish(void) NOINLINE; static ErtsCodeMFA *ubif2mfa(void* uf) NOINLINE; static ErtsCodeMFA *gcbif2mfa(void* gcf) NOINLINE; static BeamInstr* handle_error(Process* c_p, BeamInstr* pc, @@ -397,21 +392,21 @@ static BeamInstr* call_error_handler(Process* p, ErtsCodeMFA* mfa, Eterm* reg, Eterm func) NOINLINE; static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity, BeamInstr *I, Uint offs) NOINLINE; -static BeamInstr* apply(Process* p, Eterm module, Eterm function, - Eterm args, Eterm* reg, - BeamInstr *I, Uint offs) NOINLINE; +static BeamInstr* apply(Process* p, Eterm* reg, + BeamInstr *I, Uint offs) NOINLINE; static BeamInstr* call_fun(Process* p, int arity, Eterm* reg, Eterm args) NOINLINE; static BeamInstr* apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg) NOINLINE; static Eterm new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) NOINLINE; -static Eterm new_map(Process* p, Eterm* reg, BeamInstr* I) NOINLINE; -static Eterm new_small_map_lit(Process* p, Eterm* reg, Uint* n_exp, BeamInstr* I) NOINLINE; -static Eterm update_map_assoc(Process* p, Eterm* reg, - Eterm map, BeamInstr* I) NOINLINE; -static Eterm update_map_exact(Process* p, Eterm* reg, - Eterm map, BeamInstr* I) NOINLINE; +static Eterm new_map(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* ptr) NOINLINE; +static Eterm new_small_map_lit(Process* p, Eterm* reg, Eterm keys_literal, + Uint live, BeamInstr* ptr) NOINLINE; +static Eterm update_map_assoc(Process* p, Eterm* reg, Uint live, + Uint n, BeamInstr* new_p) NOINLINE; +static Eterm update_map_exact(Process* p, Eterm* reg, Uint live, + Uint n, Eterm* new_p) NOINLINE; static Eterm get_map_element(Eterm map, Eterm key); static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx); @@ -443,6 +438,12 @@ init_emulator(void) # define REG_stop asm("%l3") # define REG_I asm("%l4") # define REG_fcalls asm("%l5") +#elif defined(__GNUC__) && defined(__amd64__) && !defined(DEBUG) +# define REG_xregs asm("%r12") +# define REG_htop +# define REG_stop asm("%r13") +# define REG_I asm("%rbx") +# define REG_fcalls asm("%r14") #else # define REG_xregs # define REG_htop @@ -628,7 +629,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) #ifndef NO_JUMP_TABLE static void* opcodes[] = { DEFINE_OPCODES }; #else - int Go; + register BeamInstr Go; #endif #endif @@ -648,7 +649,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) * Note: c_p->arity must be set to reflect the number of useful terms in * c_p->arg_reg before calling the scheduler. */ - if (!init_done) { + if (ERTS_UNLIKELY(!init_done)) { /* This should only be reached during the init phase when only the main * process is running. I.e. there is no race for init_done. */ @@ -704,7 +705,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) { int reds; Eterm* argp; - BeamInstr *next; + BeamInstr next; int i; argp = c_p->arg_reg; @@ -736,7 +737,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) ERTS_DBG_CHK_REDS(c_p, FCALLS); - next = (BeamInstr *) *I; + next = *I; SWAPIN; ASSERT(VALID_INSTR(next)); @@ -774,7 +775,6 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) #endif #include "beam_hot.h" -#include "beam_instrs.h" #ifdef DEBUG /* @@ -875,6 +875,8 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) goto do_schedule1; } +#include "beam_warm.h" + OpCase(normal_exit): { SWAPOUT; c_p->freason = EXC_NORMAL; @@ -963,9 +965,6 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) init_emulator: { - int i; - Export* ep; - #ifndef NO_JUMP_TABLE #ifdef ERTS_OPCODE_COUNTER_SUPPORT #ifdef DEBUG @@ -977,35 +976,8 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) beam_ops = opcodes; #endif /* ERTS_OPCODE_COUNTER_SUPPORT */ #endif /* NO_JUMP_TABLE */ - - em_call_error_handler = OpCode(call_error_handler); - em_apply_bif = OpCode(apply_bif); - em_call_nif = OpCode(call_nif); - em_call_bif_e = OpCode(call_bif_e); - - beam_apply[0] = (BeamInstr) OpCode(i_apply); - beam_apply[1] = (BeamInstr) OpCode(normal_exit); - beam_exit[0] = (BeamInstr) OpCode(error_action_code); - beam_continue_exit[0] = (BeamInstr) OpCode(continue_exit); - beam_return_to_trace[0] = (BeamInstr) OpCode(i_return_to_trace); - beam_return_trace[0] = (BeamInstr) OpCode(return_trace); - beam_exception_trace[0] = (BeamInstr) OpCode(return_trace); /* UGLY */ - beam_return_time_trace[0] = (BeamInstr) OpCode(i_return_time_trace); - - /* - * Enter all BIFs into the export table. - */ - for (i = 0; i < BIF_SIZE; i++) { - ep = erts_export_put(bif_table[i].module, - bif_table[i].name, - bif_table[i].arity); - bif_export[i] = ep; - ep->beam[0] = (BeamInstr) OpCode(apply_bif); - ep->beam[1] = (BeamInstr) bif_table[i].f; - /* XXX: set func info for bifs */ - ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI); - } + init_emulator_finish(); return; } #ifdef NO_JUMP_TABLE @@ -1017,26 +989,71 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) save_calls1: { - Eterm* dis_next; + BeamInstr dis_next; save_calls(c_p, (Export *) Arg(0)); SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); - dis_next = (Eterm *) *I; + dis_next = *I; FCALLS--; Goto(dis_next); } } /* + * One-time initialization of emulator. Does not need to be + * in process_main(). + */ +static void +init_emulator_finish(void) +{ + int i; + Export* ep; + +#if defined(ARCH_64) && defined(CODE_MODEL_SMALL) + for (i = 0; i < NUMBER_OF_OPCODES; i++) { + BeamInstr instr = BeamOpCodeAddr(i); + if (instr >= (1ull << 32)) { + erts_exit(ERTS_ERROR_EXIT, + "This run-time was supposed be compiled with all code below 2Gb,\n" + "but the instruction '%s' is located at %016lx.\n", + opc[i].name, instr); + } + } +#endif + + beam_apply[0] = BeamOpCodeAddr(op_i_apply); + beam_apply[1] = BeamOpCodeAddr(op_normal_exit); + beam_exit[0] = BeamOpCodeAddr(op_error_action_code); + beam_continue_exit[0] = BeamOpCodeAddr(op_continue_exit); + beam_return_to_trace[0] = BeamOpCodeAddr(op_i_return_to_trace); + beam_return_trace[0] = BeamOpCodeAddr(op_return_trace); + beam_exception_trace[0] = BeamOpCodeAddr(op_return_trace); /* UGLY */ + beam_return_time_trace[0] = BeamOpCodeAddr(op_i_return_time_trace); + + /* + * Enter all BIFs into the export table. + */ + for (i = 0; i < BIF_SIZE; i++) { + ep = erts_export_put(bif_table[i].module, + bif_table[i].name, + bif_table[i].arity); + bif_export[i] = ep; + ep->beam[0] = BeamOpCodeAddr(op_apply_bif); + ep->beam[1] = (BeamInstr) bif_table[i].f; + /* XXX: set func info for bifs */ + ep->info.op = BeamOpCodeAddr(op_i_func_info_IaaI); + } +} + +/* * erts_dirty_process_main() is what dirty schedulers execute. Since they handle * only NIF calls they do not need to be able to execute all BEAM * instructions. */ void erts_dirty_process_main(ErtsSchedulerData *esdp) { -#ifdef ERTS_DIRTY_SCHEDULERS Process* c_p = NULL; ErtsMonotonicTime start_time; #ifdef DEBUG @@ -1251,12 +1268,12 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) ERTS_UNREQ_PROC_MAIN_LOCK(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - if (em_apply_bif == (BeamInstr *) *I) { + if (BeamIsOpCode(*I, op_apply_bif)) { exiting = erts_call_dirty_bif(esdp, c_p, I, reg); } else { - ASSERT(em_call_nif == (BeamInstr *) *I); - exiting = erts_call_dirty_nif(esdp, c_p, I, reg); + ASSERT(BeamIsOpCode(*I, op_call_nif)); + exiting = erts_call_dirty_nif(esdp, c_p, I, reg); } ASSERT(!(c_p->flags & F_HIBERNATE_SCHED)); @@ -1275,7 +1292,6 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) I = c_p->i; goto context_switch; } -#endif /* ERTS_DIRTY_SCHEDULERS */ } static ErtsCodeMFA * @@ -2092,8 +2108,8 @@ apply_bif_error_adjustment(Process *p, Export *ep, * and apply_last_IP. */ if (I - && ep->beam[0] == (BeamInstr) em_apply_bif - && (ep == bif_export[BIF_error_1] + && BeamIsOpCode(ep->beam[0], op_apply_bif) + && (ep == bif_export[BIF_error_1] || ep == bif_export[BIF_error_2] || ep == bif_export[BIF_exit_1] || ep == bif_export[BIF_throw_1])) { @@ -2179,13 +2195,14 @@ apply_bif_error_adjustment(Process *p, Export *ep, } static BeamInstr* -apply( -Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg, -BeamInstr *I, Uint stack_offset) +apply(Process* p, Eterm* reg, BeamInstr *I, Uint stack_offset) { int arity; Export* ep; Eterm tmp; + Eterm module = reg[0]; + Eterm function = reg[1]; + Eterm args = reg[2]; /* * Check the arguments which should be of the form apply(Module, @@ -2302,8 +2319,9 @@ fixed_apply(Process* p, Eterm* reg, Uint arity, if (is_not_atom(module)) goto error; /* Handle apply of apply/3... */ - if (module == am_erlang && function == am_apply && arity == 3) - return apply(p, reg[0], reg[1], reg[2], reg, I, stack_offset); + if (module == am_erlang && function == am_apply && arity == 3) { + return apply(p, reg, I, stack_offset); + } /* * Get the index into the export table, or failing that the export @@ -2324,10 +2342,13 @@ fixed_apply(Process* p, Eterm* reg, Uint arity, } int -erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg) +erts_hibernate(Process* c_p, Eterm* reg) { int arity; Eterm tmp; + Eterm module = reg[0]; + Eterm function = reg[1]; + Eterm args = reg[2]; if (is_not_atom(module) || is_not_atom(function)) { /* @@ -2734,24 +2755,20 @@ do { \ static Eterm -new_map(Process* p, Eterm* reg, BeamInstr* I) +new_map(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* ptr) { - Uint n = Arg(3); Uint i; Uint need = n + 1 /* hdr */ + 1 /*size*/ + 1 /* ptr */ + 1 /* arity */; Eterm keys; Eterm *mhp,*thp; Eterm *E; - BeamInstr *ptr; flatmap_t *mp; ErtsHeapFactory factory; - ptr = &Arg(4); - if (n > 2*MAP_SMALL_MAP_LIMIT) { Eterm res; if (HeapWordsLeft(p) < n) { - erts_garbage_collect(p, n, reg, Arg(2)); + erts_garbage_collect(p, n, reg, live); } mhp = p->htop; @@ -2772,7 +2789,7 @@ new_map(Process* p, Eterm* reg, BeamInstr* I) } if (HeapWordsLeft(p) < need) { - erts_garbage_collect(p, need, reg, Arg(2)); + erts_garbage_collect(p, need, reg, live); } thp = p->htop; @@ -2795,24 +2812,20 @@ new_map(Process* p, Eterm* reg, BeamInstr* I) } static Eterm -new_small_map_lit(Process* p, Eterm* reg, Uint* n_exp, BeamInstr* I) +new_small_map_lit(Process* p, Eterm* reg, Eterm keys_literal, Uint live, BeamInstr* ptr) { - Eterm* keys = tuple_val(Arg(3)); + Eterm* keys = tuple_val(keys_literal); Uint n = arityval(*keys); Uint need = n + 1 /* hdr */ + 1 /*size*/ + 1 /* ptr */ + 1 /* arity */; Uint i; - BeamInstr *ptr; flatmap_t *mp; Eterm *mhp; Eterm *E; - *n_exp = n; - ptr = &Arg(4); - ASSERT(n <= MAP_SMALL_MAP_LIMIT); if (HeapWordsLeft(p) < need) { - erts_garbage_collect(p, need, reg, Arg(2)); + erts_garbage_collect(p, need, reg, live); } mhp = p->htop; @@ -2821,7 +2834,7 @@ new_small_map_lit(Process* p, Eterm* reg, Uint* n_exp, BeamInstr* I) mp = (flatmap_t *)mhp; mhp += MAP_HEADER_FLATMAP_SZ; mp->thing_word = MAP_HEADER_FLATMAP; mp->size = n; - mp->keys = Arg(3); + mp->keys = keys_literal; for (i = 0; i < n; i++) { GET_TERM(*ptr++, *mhp++); @@ -2833,9 +2846,8 @@ new_small_map_lit(Process* p, Eterm* reg, Uint* n_exp, BeamInstr* I) } static Eterm -update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) +update_map_assoc(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* new_p) { - Uint n; Uint num_old; Uint num_updates; Uint need; @@ -2845,12 +2857,12 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) Eterm* E; Eterm* old_keys; Eterm* old_vals; - BeamInstr* new_p; Eterm new_key; Eterm* kp; + Eterm map; - new_p = &Arg(4); - num_updates = Arg(3) / 2; + num_updates = n / 2; + map = reg[live]; if (is_not_flatmap(map)) { Uint32 hx; @@ -2880,7 +2892,7 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) */ if (num_old == 0) { - return new_map(p, reg, I); + return new_map(p, reg, live, n, new_p); } /* @@ -2890,8 +2902,6 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) need = 2*(num_old+num_updates) + 1 + MAP_HEADER_FLATMAP_SZ; if (HeapWordsLeft(p) < need) { - Uint live = Arg(2); - reg[live] = map; erts_garbage_collect(p, need, reg, live+1); map = reg[live]; old_mp = (flatmap_t *)flatmap_val(map); @@ -3038,9 +3048,8 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) */ static Eterm -update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) +update_map_exact(Process* p, Eterm* reg, Uint live, Uint n, Eterm* new_p) { - Uint n; Uint i; Uint num_old; Uint need; @@ -3050,12 +3059,12 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) Eterm* E; Eterm* old_keys; Eterm* old_vals; - BeamInstr* new_p; Eterm new_key; + Eterm map; - new_p = &Arg(5); - n = Arg(4) / 2; /* Number of values to be updated */ + n /= 2; /* Number of values to be updated */ ASSERT(n > 0); + map = reg[live]; if (is_not_flatmap(map)) { Uint32 hx; @@ -3109,8 +3118,6 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) need = num_old + MAP_HEADER_FLATMAP_SZ; if (HeapWordsLeft(p) < need) { - Uint live = Arg(3); - reg[live] = map; erts_garbage_collect(p, need, reg, live+1); map = reg[live]; old_mp = (flatmap_t *)flatmap_val(map); @@ -3208,8 +3215,8 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity) if ((ep = export_get(&e)) == NULL) { return 0; } - return ep->addressv[erts_active_code_ix()] == ep->beam - && (ep->beam[0] == (BeamInstr) em_apply_bif); + return ep->addressv[erts_active_code_ix()] == ep->beam && + BeamIsOpCode(ep->beam[0], op_apply_bif); } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index dcd312f54f..00dd28b26c 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -55,12 +55,6 @@ ErlDrvBinary* erts_gzinflate_buffer(char*, int); #define DEFINED 1 #define EXPORTED 2 -#ifdef NO_JUMP_TABLE -# define BeamOpCode(Op) ((BeamInstr)(Op)) -#else -# define BeamOpCode(Op) ((BeamInstr)beam_ops[Op]) -#endif - #if defined(WORDS_BIGENDIAN) # define NATIVE_ENDIAN(F) \ if ((F).val & BSF_NATIVE) { \ @@ -81,15 +75,28 @@ ErlDrvBinary* erts_gzinflate_buffer(char*, int); #define TE_FAIL (-1) #define TE_SHORT_WINDOW (-2) +/* + * Type for a reference to a label that must be patched. + */ + typedef struct { - Uint value; /* Value of label (NULL if not known yet). */ - Sint patches; /* Index (into code buffer) to first - * location which must be patched with - * the value of this label. - */ + Uint pos; /* Position of label reference to patch. */ + Uint offset; /* Offset from patch location. */ + int packed; /* 0 (not packed), 1 (lsw), 2 (msw) */ +} LabelPatch; + +/* + * Type for a label. + */ + +typedef struct { + Uint value; /* Value of label (0 if not known yet). */ Uint looprec_targeted; /* Non-zero if this label is the target of a loop_rec * instruction. */ + LabelPatch* patches; /* Array of label patches. */ + Uint num_patches; /* Number of patches in array. */ + Uint num_allocated; /* Number of allocated patches. */ } Label; /* @@ -226,7 +233,7 @@ typedef struct { typedef struct literal_patch LiteralPatch; struct literal_patch { - int pos; /* Position in code */ + Uint pos; /* Position in code */ LiteralPatch* next; }; @@ -507,6 +514,7 @@ static int read_lambda_table(LoaderState* stp); static int read_literal_table(LoaderState* stp); static int read_line_table(LoaderState* stp); static int read_code_header(LoaderState* stp); +static void init_label(Label* lp); static int load_code(LoaderState* stp); static GenOp* gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index, GenOpArg Tuple, GenOpArg Dst); @@ -835,10 +843,9 @@ erts_finish_loading(Binary* magic, Process* c_p, continue; } if (ep->addressv[code_ix] == ep->beam) { - if (ep->beam[0] == (BeamInstr) em_apply_bif) { + if (BeamIsOpCode(ep->beam[0], op_apply_bif)) { continue; - } else if (ep->beam[0] == - (BeamInstr) BeamOp(op_i_generic_breakpoint)) { + } else if (BeamIsOpCode(ep->beam[0], op_i_generic_breakpoint)) { ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); ASSERT(mod_tab_p->curr.num_traced_exports > 0); erts_clear_export_break(mod_tab_p, &ep->info); @@ -1051,6 +1058,10 @@ loader_state_dtor(Binary* magic) stp->codev = 0; } if (stp->labels != 0) { + Uint num; + for (num = 0; num < stp->num_labels; num++) { + erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->labels[num].patches); + } erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->labels); stp->labels = 0; } @@ -1464,7 +1475,7 @@ load_import_table(LoaderState* stp) * the BIF function. */ if ((e = erts_active_export_entry(mod, func, arity)) != NULL) { - if (e->beam[0] == (BeamInstr) em_apply_bif) { + if (BeamIsOpCode(e->beam[0], op_apply_bif)) { stp->import[i].bf = (BifFunction) e->beam[1]; if (func == am_load_nif && mod == am_erlang && arity == 2) { stp->may_load_nif = 1; @@ -1534,7 +1545,7 @@ read_export_table(LoaderState* stp) * any other functions that walk through all local functions. */ - if (stp->labels[n].patches >= 0) { + if (stp->labels[n].num_patches > 0) { LoadError3(stp, "there are local calls to the stub for " "the BIF %T:%T/%d", stp->module, func, arity); @@ -1558,7 +1569,7 @@ is_bif(Eterm mod, Eterm func, unsigned arity) if (e == NULL) { return 0; } - if (e->beam[0] != (BeamInstr) em_apply_bif) { + if (! BeamIsOpCode(e->beam[0], op_apply_bif)) { return 0; } if (mod == am_erlang && func == am_apply && arity == 3) { @@ -1880,9 +1891,7 @@ read_code_header(LoaderState* stp) stp->labels = (Label *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, stp->num_labels * sizeof(Label)); for (i = 0; i < stp->num_labels; i++) { - stp->labels[i].value = 0; - stp->labels[i].patches = -1; - stp->labels[i].looprec_targeted = 0; + init_label(&stp->labels[i]); } stp->catches = 0; @@ -1911,12 +1920,43 @@ read_code_header(LoaderState* stp) #define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm)))) +static void init_label(Label* lp) +{ + lp->value = 0; + lp->looprec_targeted = 0; + lp->num_patches = 0; + lp->num_allocated = 4; + lp->patches = erts_alloc(ERTS_ALC_T_PREPARED_CODE, + lp->num_allocated * sizeof(LabelPatch)); +} + +static void +register_label_patch(LoaderState* stp, Uint label, Uint ci, Uint offset) +{ + Label* lp; + + ASSERT(label < stp->num_labels); + lp = &stp->labels[label]; + if (lp->num_allocated <= lp->num_patches) { + lp->num_allocated *= 2; + lp->patches = erts_realloc(ERTS_ALC_T_PREPARED_CODE, + (void *) lp->patches, + lp->num_allocated * sizeof(LabelPatch)); + } + lp->patches[lp->num_patches].pos = ci; + lp->patches[lp->num_patches].offset = offset; + lp->patches[lp->num_patches].packed = 0; + lp->num_patches++; + stp->codev[ci] = label; +} + static int load_code(LoaderState* stp) { int i; - int ci; - int last_func_start = 0; /* Needed by nif loading and line instructions */ + Uint ci; + Uint last_instr_start; /* Needed for relative jumps */ + Uint last_func_start = 0; /* Needed by nif loading and line instructions */ char* sign; int arg; /* Number of current argument. */ int num_specific; /* Number of specific ops for current. */ @@ -1929,6 +1969,9 @@ load_code(LoaderState* stp) GenOp** last_op_next = NULL; int arity; int retval = 1; +#if defined(BEAM_WIDE_SHIFT) + int num_trailing_f; /* Number of extra 'f' arguments in a list */ +#endif /* * The size of the loaded func_info instruction is needed @@ -2034,30 +2077,10 @@ load_code(LoaderState* stp) case 0: /* Floating point number. * Not generated by the compiler in R16B and later. + * (The literal pool is used instead.) */ - { - Eterm* hp; -#if !defined(ARCH_64) - Uint high, low; -# endif - last_op->a[arg].val = new_literal(stp, &hp, - FLOAT_SIZE_OBJECT); - hp[0] = HEADER_FLONUM; - last_op->a[arg].type = TAG_q; -#if defined(ARCH_64) - GetInt(stp, 8, hp[1]); -# else - GetInt(stp, 4, high); - GetInt(stp, 4, low); - if (must_swap_floats) { - Uint t = high; - high = low; - low = t; - } - hp[1] = high; - hp[2] = low; -# endif - } + LoadError0(stp, "please re-compile this module with an " + ERLANG_OTP_RELEASE " compiler"); break; case 1: /* List. */ if (arg+1 != arity) { @@ -2292,7 +2315,8 @@ load_code(LoaderState* stp) stp->specific_op = specific; CodeNeed(opc[stp->specific_op].sz+16); /* Extra margin for packing */ - code[ci++] = BeamOpCode(stp->specific_op); + last_instr_start = ci + opc[stp->specific_op].adjust; + code[ci++] = BeamOpCodeAddr(stp->specific_op); } /* @@ -2374,7 +2398,8 @@ load_code(LoaderState* stp) break; } break; - case 'd': /* Destination (x(0), x(N), y(N) */ + case 'd': /* Destination (x(N), y(N) */ + case 'S': /* Source (x(N), y(N)) */ switch (tag) { case TAG_x: code[ci++] = tmp_op->a[arg].val * sizeof(Eterm); @@ -2388,11 +2413,29 @@ load_code(LoaderState* stp) break; } break; - case 'I': /* Untagged integer (or pointer). */ - VerifyTag(stp, tag, TAG_u); - code[ci++] = tmp_op->a[arg].val; - break; - case 't': /* Small untagged integer -- can be packed. */ + case 't': /* Small untagged integer (16 bits) -- can be packed. */ + case 'I': /* Untagged integer (32 bits) -- can be packed. */ + case 'W': /* Untagged integer or pointer (machine word). */ +#ifdef DEBUG + switch (*sign) { + case 't': + if (tmp_op->a[arg].val >> 16 != 0) { + load_printf(__LINE__, stp, "value %lu of type 't' does not fit in 16 bits", + tmp_op->a[arg].val); + ASSERT(0); + } + break; +#ifdef ARCH_64 + case 'I': + if (tmp_op->a[arg].val >> 32 != 0) { + load_printf(__LINE__, stp, "value %lu of type 'I' does not fit in 32 bits", + tmp_op->a[arg].val); + ASSERT(0); + } + break; +#endif + } +#endif VerifyTag(stp, tag, TAG_u); code[ci++] = tmp_op->a[arg].val; break; @@ -2402,16 +2445,14 @@ load_code(LoaderState* stp) break; case 'f': /* Destination label */ VerifyTag(stp, tag_to_letter[tag], *sign); - code[ci] = stp->labels[tmp_op->a[arg].val].patches; - stp->labels[tmp_op->a[arg].val].patches = ci; + register_label_patch(stp, tmp_op->a[arg].val, ci, -last_instr_start); ci++; break; case 'j': /* 'f' or 'p' */ if (tag == TAG_p) { code[ci] = 0; } else if (tag == TAG_f) { - code[ci] = stp->labels[tmp_op->a[arg].val].patches; - stp->labels[tmp_op->a[arg].val].patches = ci; + register_label_patch(stp, tmp_op->a[arg].val, ci, -last_instr_start); } else { LoadError3(stp, "bad tag %d; expected %d or %d", tag, TAG_f, TAG_p); @@ -2431,7 +2472,6 @@ load_code(LoaderState* stp) LoadError1(stp, "label %d defined more than once", last_label); } stp->labels[last_label].value = ci; - ASSERT(stp->labels[last_label].patches < ci); break; case 'e': /* Export entry */ VerifyTag(stp, tag, TAG_u); @@ -2477,39 +2517,168 @@ load_code(LoaderState* stp) * The packing engine. */ if (opc[stp->specific_op].pack[0]) { - char* prog; /* Program for packing engine. */ - BeamInstr stack[8]; /* Stack. */ - BeamInstr* sp = stack; /* Points to next free position. */ - BeamInstr packed = 0; /* Accumulator for packed operations. */ + char* prog; /* Program for packing engine. */ + struct pack_stack { + BeamInstr instr; + Uint* patch_pos; + } stack[8]; /* Stack. */ + struct pack_stack* sp = stack; /* Points to next free position. */ + BeamInstr packed = 0; /* Accumulator for packed operations. */ + LabelPatch* packed_label = 0; for (prog = opc[stp->specific_op].pack; *prog; prog++) { switch (*prog) { - case 'g': /* Get instruction; push on stack. */ - *sp++ = code[--ci]; - break; - case 'i': /* Initialize packing accumulator. */ - packed = code[--ci]; + case 'g': /* Get operand and push on stack. */ + ci--; + sp->instr = code[ci]; + sp->patch_pos = 0; + sp++; + break; + case 'f': /* Get possible 'f' operand and push on stack. */ + { + Uint w = code[--ci]; + sp->instr = w; + sp->patch_pos = 0; + + if (w != 0) { + LabelPatch* lbl_p; + int num_patches; + int patch; + + ASSERT(w < stp->num_labels); + lbl_p = stp->labels[w].patches; + num_patches = stp->labels[w].num_patches; + for (patch = num_patches - 1; patch >= 0; patch--) { + if (lbl_p[patch].pos == ci) { + sp->patch_pos = &lbl_p[patch].pos; + break; + } + } + ASSERT(sp->patch_pos); + } + sp++; + } + break; + case 'q': /* Get possible 'q' operand and push on stack. */ + { + LiteralPatch* lp; + + ci--; + sp->instr = code[ci]; + sp->patch_pos = 0; + + for (lp = stp->literal_patches; + lp && lp->pos > ci-MAX_OPARGS; + lp = lp->next) { + if (lp->pos == ci) { + sp->patch_pos = &lp->pos; + break; + } + } + sp++; + } + break; +#ifdef ARCH_64 + case '1': /* Tightest shift (always 10 bits) */ + ci--; + ASSERT((code[ci] & ~0x1FF8ull) == 0); /* Fits in 10 bits */ + packed = (packed << BEAM_TIGHTEST_SHIFT); + packed |= code[ci] >> 3; + if (packed_label) { + packed_label->packed++; + } break; - case '0': /* Tight shift */ +#endif + case '2': /* Tight shift (10 or 16 bits) */ packed = (packed << BEAM_TIGHT_SHIFT) | code[--ci]; + if (packed_label) { + packed_label->packed++; + } break; - case '6': /* Shift 16 steps */ + case '3': /* Loose shift (16 bits) */ packed = (packed << BEAM_LOOSE_SHIFT) | code[--ci]; + if (packed_label) { + packed_label->packed++; + } break; #ifdef ARCH_64 - case 'w': /* Shift 32 steps */ - packed = (packed << BEAM_WIDE_SHIFT) | code[--ci]; - break; + case '4': /* Wide shift (32 bits) */ + { + Uint w = code[--ci]; + + if (packed_label) { + packed_label->packed++; + } + + /* + * 'w' can handle both labels ('f' and 'j'), as well + * as 'I'. Test whether this is a label. + */ + + if (w < stp->num_labels) { + /* + * Probably a label. Look for patch pointing to this + * position. + */ + LabelPatch* lp = stp->labels[w].patches; + int num_patches = stp->labels[w].num_patches; + int patch; + for (patch = num_patches - 1; patch >= 0; patch--) { + if (lp[patch].pos == ci) { + lp[patch].packed = 1; + packed_label = &lp[patch]; + break; + } + } + } + packed = (packed << BEAM_WIDE_SHIFT) | + (code[ci] & BEAM_WIDE_MASK); + } + break; #endif case 'p': /* Put instruction (from stack). */ - code[ci++] = *--sp; + --sp; + code[ci] = sp->instr; + if (sp->patch_pos) { + *sp->patch_pos = ci; + } + ci++; break; - case 'P': /* Put packed operands. */ - *sp++ = packed; + case 'P': /* Put packed operands (on the stack). */ + sp->instr = packed; + sp->patch_pos = 0; + if (packed_label) { + sp->patch_pos = &packed_label->pos; + packed_label = 0; + } + sp++; packed = 0; break; +#if defined(ARCH_64) && defined(CODE_MODEL_SMALL) + case '#': /* -1 */ + case '$': /* -2 */ + case '%': /* -3 */ + case '&': /* -4 */ + case '\'': /* -5 */ + case '(': /* -6 */ + /* Pack accumulator contents into instruction word. */ + { + Sint pos = ci - (*prog - '#' + 1); + /* Are the high 32 bits of the instruction word zero? */ + ASSERT((code[pos] & ~((1ull << BEAM_WIDE_SHIFT)-1)) == 0); + code[pos] |= packed << BEAM_WIDE_SHIFT; + if (packed_label) { + ASSERT(packed_label->packed == 1); + packed_label->pos = pos; + packed_label->packed = 2; + packed_label = 0; + } + packed >>= BEAM_WIDE_SHIFT; + } + break; +#endif default: - ASSERT(0); + erts_exit(ERTS_ERROR_EXIT, "beam_load: invalid packing op: %c\n", *prog); } } ASSERT(sp == stack); /* Incorrect program? */ @@ -2519,7 +2688,17 @@ load_code(LoaderState* stp) * Load any list arguments using the primitive tags. */ +#if defined(BEAM_WIDE_SHIFT) + num_trailing_f = 0; +#endif for ( ; arg < tmp_op->arity; arg++) { +#if defined(BEAM_WIDE_SHIFT) + if (tmp_op->a[arg].type == TAG_f) { + num_trailing_f++; + } else { + num_trailing_f = 0; + } +#endif switch (tmp_op->a[arg].type) { case TAG_i: CodeNeed(1); @@ -2533,8 +2712,7 @@ load_code(LoaderState* stp) break; case TAG_f: CodeNeed(1); - code[ci] = stp->labels[tmp_op->a[arg].val].patches; - stp->labels[tmp_op->a[arg].val].patches = ci; + register_label_patch(stp, tmp_op->a[arg].val, ci, -last_instr_start); ci++; break; case TAG_x: @@ -2560,6 +2738,61 @@ load_code(LoaderState* stp) } } + /* + * If all the extra arguments were 'f' operands, + * and the wordsize is 64 bits, pack two 'f' operands + * into each word. + */ + +#if defined(BEAM_WIDE_SHIFT) + if (num_trailing_f >= 1) { + Uint src_index = ci - num_trailing_f; + Uint src_limit = ci; + Uint dst_limit = src_index + (num_trailing_f+1)/2; + + ci = src_index; + while (ci < dst_limit) { + Uint w[2]; + BeamInstr packed = 0; + int wi; + + w[0] = code[src_index]; + if (src_index+1 < src_limit) { + w[1] = code[src_index+1]; + } else { + w[1] = 0; + } + for (wi = 0; wi < 2; wi++) { + Uint lbl = w[wi]; + LabelPatch* lp = stp->labels[lbl].patches; + int num_patches = stp->labels[lbl].num_patches; + +#if defined(WORDS_BIGENDIAN) + packed <<= BEAM_WIDE_SHIFT; + packed |= lbl & BEAM_WIDE_MASK; +#else + packed >>= BEAM_WIDE_SHIFT; + packed |= lbl << BEAM_WIDE_SHIFT; +#endif + while (num_patches-- > 0) { + if (lp->pos == src_index + wi) { + lp->pos = ci; +#if defined(WORDS_BIGENDIAN) + lp->packed = 2 - wi; +#else + lp->packed = wi + 1; +#endif + break; + } + lp++; + } + } + code[ci++] = packed; + src_index += 2; + } + } +#endif + /* * Handle a few special cases. */ @@ -2606,17 +2839,16 @@ load_code(LoaderState* stp) the size of the ops.tab i_func_info instruction is not the same as FUNC_INFO_SZ */ ASSERT(stp->labels[last_label].value == ci - FUNC_INFO_SZ); - stp->hdr->functions[function_number] = (ErtsCodeInfo*) stp->labels[last_label].patches; offset = function_number; - stp->labels[last_label].patches = offset; + register_label_patch(stp, last_label, offset, 0); function_number++; if (stp->arity > MAX_ARG) { LoadError1(stp, "too many arguments: %d", stp->arity); } #ifdef DEBUG - ASSERT(stp->labels[0].patches < 0); /* Should not be referenced. */ + ASSERT(stp->labels[0].num_patches == 0); /* Should not be referenced. */ for (i = 1; i < stp->num_labels; i++) { - ASSERT(stp->labels[i].patches < ci); + ASSERT(stp->labels[i].num_patches <= stp->labels[i].num_allocated); } #endif } @@ -2627,8 +2859,8 @@ load_code(LoaderState* stp) /* Remember offset for the on_load function. */ stp->on_load = ci; break; - case op_bs_put_string_II: - case op_i_bs_match_string_xfII: + case op_bs_put_string_WW: + case op_i_bs_match_string_xfWW: new_string_patch(stp, ci-1); break; @@ -2884,6 +3116,7 @@ gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index, op->next = NULL; if (Index.type == TAG_i && Index.val > 0 && + Index.val <= ERTS_MAX_TUPLE_SIZE && (Tuple.type == TAG_x || Tuple.type == TAG_y)) { op->op = genop_i_fast_element_4; op->a[0] = Tuple; @@ -3420,7 +3653,7 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time) Sint timeout; NEW_GENOP(stp, op); - op->op = genop_wait_timeout_unlocked_2; + op->op = genop_wait_timeout_unlocked_int_2; op->next = NULL; op->arity = 2; op->a[0].type = TAG_u; @@ -3467,12 +3700,12 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time) Sint timeout; NEW_GENOP(stp, op); - op->op = genop_wait_timeout_locked_2; + op->op = genop_wait_timeout_locked_int_2; op->next = NULL; op->arity = 2; - op->a[0] = Fail; - op->a[1].type = TAG_u; - + op->a[0].type = TAG_u; + op->a[1] = Fail; + if (Time.type == TAG_i && (timeout = Time.val) >= 0 && #if defined(ARCH_64) (timeout >> 32) == 0 @@ -3480,7 +3713,7 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time) 1 #endif ) { - op->a[1].val = timeout; + op->a[0].val = timeout; #if !defined(ARCH_64) } else if (Time.type == TAG_q) { Eterm big; @@ -3494,7 +3727,7 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time) } else { Uint u; (void) term_to_Uint(big, &u); - op->a[1].val = (BeamInstr) u; + op->a[0].val = (BeamInstr) u; } #endif } else { @@ -3540,7 +3773,7 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail, if (size == 2) { NEW_GENOP(stp, op); op->next = NULL; - op->op = genop_i_select_tuple_arity2_6; + op->op = genop_i_select_tuple_arity2_4; GENOP_ARITY(op, arity - 1); op->a[0] = S; op->a[1] = Fail; @@ -3830,14 +4063,13 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, int i, j, align = 0; if (size == 2) { - /* * Use a special-cased instruction if there are only two values. */ NEW_GENOP(stp, op); op->next = NULL; - op->op = genop_i_select_val2_6; + op->op = genop_i_select_val2_4; GENOP_ARITY(op, arity - 1); op->a[0] = S; op->a[1] = Fail; @@ -3847,47 +4079,19 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, op->a[5] = Rest[3]; return op; - - } else if (size > 10) { - - /* binary search instruction */ - - NEW_GENOP(stp, op); - op->next = NULL; - op->op = genop_i_select_val_bins_3; - GENOP_ARITY(op, arity); - op->a[0] = S; - op->a[1] = Fail; - op->a[2].type = TAG_u; - op->a[2].val = size; - for (i = 3; i < arity; i++) { - op->a[i] = Rest[i-3]; - } - - /* - * Sort the values to make them useful for a binary search. - */ - - qsort(op->a+3, size, 2*sizeof(GenOpArg), - (int (*)(const void *, const void *)) genopargcompare); -#ifdef DEBUG - for (i = 3; i < arity-2; i += 2) { - ASSERT(op->a[i].val < op->a[i+2].val); - } -#endif - return op; } - /* linear search instruction */ - - align = 1; + if (size <= 10) { + /* Use linear search. Reserve place for a sentinel. */ + align = 1; + } arity += 2*align; size += align; NEW_GENOP(stp, op); op->next = NULL; - op->op = genop_i_select_val_lins_3; + op->op = (align == 0) ? genop_i_select_val_bins_3 : genop_i_select_val_lins_3; GENOP_ARITY(op, arity); op->a[0] = S; op->a[1] = Fail; @@ -3901,7 +4105,7 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, } /* - * Sort the values to make them useful for a sentinel search + * Sort the values to make them useful for a binary or sentinel search. */ qsort(tmp, size - align, 2*sizeof(GenOpArg), @@ -3916,11 +4120,12 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, erts_free(ERTS_ALC_T_LOADER_TMP, (void *) tmp); - /* add sentinel */ - - op->a[j].type = TAG_u; - op->a[j].val = ~((BeamInstr)0); - op->a[j+size] = Fail; + if (align) { + /* Add sentinel for linear search. */ + op->a[j].type = TAG_u; + op->a[j].val = ~((BeamInstr)0); + op->a[j+size] = Fail; + } #ifdef DEBUG for (i = 0; i < size - 1; i++) { @@ -4804,21 +5009,57 @@ freeze_code(LoaderState* stp) */ for (i = 0; i < stp->num_labels; i++) { - Sint this_patch; - Sint next_patch; + Uint patch; Uint value = stp->labels[i].value; - - if (value == 0 && stp->labels[i].patches >= 0) { + + if (value == 0 && stp->labels[i].num_patches != 0) { LoadError1(stp, "label %d not resolved", i); } ASSERT(value < stp->ci); - this_patch = stp->labels[i].patches; - while (this_patch >= 0) { - ASSERT(this_patch < stp->ci); - next_patch = codev[this_patch]; - ASSERT(next_patch < stp->ci); - codev[this_patch] = (BeamInstr) (codev + value); - this_patch = next_patch; + for (patch = 0; patch < stp->labels[i].num_patches; patch++) { + LabelPatch* lp = &stp->labels[i].patches[patch]; + Uint pos = lp->pos; + ASSERT(pos < stp->ci); + if (pos < stp->num_functions) { + /* + * This is the array of pointers to the beginning of + * each function. The pointers must remain absolute. + */ + codev[pos] = (BeamInstr) (codev + value); + } else { +#ifdef DEBUG + Uint w; +#endif + Sint32 rel = lp->offset + value; + switch (lp->packed) { + case 0: /* Not packed */ + ASSERT(codev[pos] == i); + codev[pos] = rel; + break; +#ifdef BEAM_WIDE_MASK + case 1: /* Least significant word. */ +#ifdef DEBUG + w = codev[pos] & BEAM_WIDE_MASK; + /* Correct label in least significant word? */ + ASSERT(w == i); +#endif + codev[pos] = (codev[pos] & ~BEAM_WIDE_MASK) | + (rel & BEAM_WIDE_MASK); + break; + case 2: /* Most significant word */ +#ifdef DEBUG + w = (codev[pos] >> BEAM_WIDE_SHIFT) & BEAM_WIDE_MASK; + /* Correct label in most significant word? */ + ASSERT(w == i); +#endif + codev[pos] = ((Uint)rel << BEAM_WIDE_SHIFT) | + (codev[pos] & BEAM_WIDE_MASK); + break; +#endif + default: + ASSERT(0); + } + } } } CHKBLK(ERTS_ALC_T_CODE,code_hdr); @@ -4861,8 +5102,11 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) catches = BEAM_CATCHES_NIL; while (index != 0) { BeamInstr next = codev[index]; - codev[index] = BeamOpCode(op_catch_yf); - catches = beam_catches_cons((BeamInstr *)codev[index+2], catches); + BeamInstr* abs_addr; + codev[index] = BeamOpCodeAddr(op_catch_yf); + /* We must make the address of the label absolute again. */ + abs_addr = (BeamInstr *)codev + index + codev[index+2]; + catches = beam_catches_cons(abs_addr, catches); codev[index+2] = make_catch(catches); index = next; } @@ -5329,12 +5573,15 @@ get_tag_and_value(LoaderState* stp, Uint len_code, { Uint count; Sint val; - byte default_buf[128]; - byte* bigbuf = default_buf; + byte default_byte_buf[128]; + byte* byte_buf = default_byte_buf; + Eterm default_big_buf[128/sizeof(Eterm)]; + Eterm* big_buf = default_big_buf; + Eterm tmp_big; byte* s; int i; int neg = 0; - Uint arity; + Uint words_needed; Eterm* hp; /* @@ -5411,8 +5658,11 @@ get_tag_and_value(LoaderState* stp, Uint len_code, *result = val; return TAG_i; } else { - *result = new_literal(stp, &hp, BIG_UINT_HEAP_SIZE); - (void) small_to_big(val, hp); + tmp_big = small_to_big(val, big_buf); + if (!find_literal(stp, tmp_big, result)) { + *result = new_literal(stp, &hp, BIG_UINT_HEAP_SIZE); + sys_memcpy(hp, big_buf, BIG_UINT_HEAP_SIZE*sizeof(Eterm)); + } return TAG_q; } } @@ -5422,8 +5672,8 @@ get_tag_and_value(LoaderState* stp, Uint len_code, * (including margin). */ - if (count+8 > sizeof(default_buf)) { - bigbuf = erts_alloc(ERTS_ALC_T_LOADER_TMP, count+8); + if (count+8 > sizeof(default_byte_buf)) { + byte_buf = erts_alloc(ERTS_ALC_T_LOADER_TMP, count+8); } /* @@ -5432,20 +5682,20 @@ get_tag_and_value(LoaderState* stp, Uint len_code, GetString(stp, s, count); for (i = 0; i < count; i++) { - bigbuf[count-i-1] = *s++; + byte_buf[count-i-1] = *s++; } /* * Check if the number is negative, and negate it if so. */ - if ((bigbuf[count-1] & 0x80) != 0) { + if ((byte_buf[count-1] & 0x80) != 0) { unsigned carry = 1; neg = 1; for (i = 0; i < count; i++) { - bigbuf[i] = ~bigbuf[i] + carry; - carry = (bigbuf[i] == 0 && carry == 1); + byte_buf[i] = ~byte_buf[i] + carry; + carry = (byte_buf[i] == 0 && carry == 1); } ASSERT(carry == 0); } @@ -5454,33 +5704,52 @@ get_tag_and_value(LoaderState* stp, Uint len_code, * Align to word boundary. */ - if (bigbuf[count-1] == 0) { + if (byte_buf[count-1] == 0) { count--; } - if (bigbuf[count-1] == 0) { + if (byte_buf[count-1] == 0) { LoadError0(stp, "bignum not normalized"); } while (count % sizeof(Eterm) != 0) { - bigbuf[count++] = 0; + byte_buf[count++] = 0; } /* - * Allocate heap space for the bignum and copy it. + * Convert to a bignum. */ - arity = count/sizeof(Eterm); - *result = new_literal(stp, &hp, arity+1); - if (is_nil(bytes_to_big(bigbuf, count, neg, hp))) - goto load_error; + words_needed = count/sizeof(Eterm) + 1; + if (words_needed*sizeof(Eterm) > sizeof(default_big_buf)) { + big_buf = erts_alloc(ERTS_ALC_T_LOADER_TMP, words_needed*sizeof(Eterm)); + } + tmp_big = bytes_to_big(byte_buf, count, neg, big_buf); + if (is_nil(tmp_big)) { + goto load_error; + } - if (bigbuf != default_buf) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) bigbuf); + /* + * Create a literal if there is no previous literal with the same value. + */ + + if (!find_literal(stp, tmp_big, result)) { + *result = new_literal(stp, &hp, words_needed); + sys_memcpy(hp, big_buf, words_needed*sizeof(Eterm)); + } + + if (byte_buf != default_byte_buf) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) byte_buf); + } + if (big_buf != default_big_buf) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) big_buf); } return TAG_q; load_error: - if (bigbuf != default_buf) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) bigbuf); + if (byte_buf != default_byte_buf) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) byte_buf); + } + if (big_buf != default_big_buf) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) big_buf); } return -1; } @@ -5525,8 +5794,7 @@ new_label(LoaderState* stp) stp->labels = (Label *) erts_realloc(ERTS_ALC_T_PREPARED_CODE, (void *) stp->labels, stp->num_labels * sizeof(Label)); - stp->labels[num].value = 0; - stp->labels[num].patches = -1; + init_label(&stp->labels[num]); return num; } @@ -5789,9 +6057,9 @@ int erts_is_function_native(ErtsCodeInfo *ci) { #ifdef HIPE - ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - return erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call) - || erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call_closure); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI)); + return BeamIsOpCode(erts_codeinfo_to_code(ci)[0], op_hipe_trap_call) || + BeamIsOpCode(erts_codeinfo_to_code(ci)[0], op_hipe_trap_call_closure); #else return 0; #endif @@ -5860,7 +6128,7 @@ exported_from_module(Process* p, /* Process whose heap to use. */ Eterm tuple; if (ep->addressv[code_ix] == ep->beam && - ep->beam[0] == (BeamInstr) em_call_error_handler) { + BeamIsOpCode(ep->beam[0], op_call_error_handler)) { /* There is a call to the function, but it does not exist. */ continue; } @@ -6131,7 +6399,7 @@ make_stub(ErtsCodeInfo* info, Eterm mod, Eterm func, Uint arity, Uint native, Be { DBG_TRACE_MFA(mod,func,arity,"make beam stub at %p", erts_codeinfo_to_code(info)); ASSERT(WORDS_PER_FUNCTION == 6); - info->op = (BeamInstr) BeamOp(op_i_func_info_IaaI); + info->op = BeamOpCodeAddr(op_i_func_info_IaaI); info->u.ncallee = (void (*)(void)) native; info->mfa.module = mod; info->mfa.function = func; @@ -6236,7 +6504,7 @@ stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci) for (i = 0, lp = stp->lambdas; i < n; i++, lp++) { ErlFunEntry* fe = stp->lambdas[i].fe; if (lp->function == ci->mfa.function && lp->arity == ci->mfa.arity) { - *erts_codeinfo_to_code(ci) = (Eterm) BeamOpCode(op_hipe_trap_call_closure); + *erts_codeinfo_to_code(ci) = BeamOpCodeAddr(op_hipe_trap_call_closure); fe->address = erts_codeinfo_to_code(ci); } } @@ -6560,7 +6828,7 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) * as the body until we know what kind of trap we should put there. */ code_hdr->functions[i] = (ErtsCodeInfo*)fp; - op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */ + op = BeamOpCodeAddr(op_hipe_trap_call); /* Might be changed later. */ fp = make_stub((ErtsCodeInfo*)fp, hipe_stp->module, func, arity, (Uint)native_address, op); } @@ -6570,7 +6838,7 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) */ code_hdr->functions[i] = (ErtsCodeInfo*)fp; - *fp++ = (BeamInstr) BeamOp(op_int_code_end); + *fp++ = BeamOpCodeAddr(op_int_code_end); /* * Copy attributes and compilation information. diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index c088bdb751..156c3c45e2 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -37,11 +37,6 @@ typedef struct gen_op_entry { extern const GenOpEntry gen_opc[]; -extern BeamInstr beam_debug_apply[]; -extern BeamInstr* em_call_error_handler; -extern BeamInstr* em_apply_bif; -extern BeamInstr* em_call_nif; - struct ErtsLiteralArea_; /* @@ -111,11 +106,7 @@ typedef struct beam_code_header { }BeamCodeHeader; -#ifdef ERTS_DIRTY_SCHEDULERS # define BEAM_NIF_MIN_FUNC_SZ 4 -#else -# define BEAM_NIF_MIN_FUNC_SZ 3 -#endif void erts_release_literal_area(struct ErtsLiteralArea_* literal_area); int erts_is_module_native(BeamCodeHeader* code); diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 4b45e98685..ad555bb195 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -437,7 +437,6 @@ BIF_RETTYPE demonitor(Process *c_p, Eterm ref, Eterm *multip) ErtsMonitor *mon = NULL; /* The monitor entry to delete */ Eterm to = NIL; /* Monitor link traget */ DistEntry *dep = NULL; /* Target's distribution entry */ - int deref_de = 0; BIF_RETTYPE res = am_false; int unlock_link = 1; @@ -467,8 +466,6 @@ BIF_RETTYPE demonitor(Process *c_p, Eterm ref, Eterm *multip) ASSERT(is_node_name_atom(to)); dep = erts_sysname_to_connected_dist_entry(to); ASSERT(dep != erts_this_dist_entry); - if (dep) - deref_de = 1; } else if (is_port(to)) { if (port_dist_entry(to) != erts_this_dist_entry) { goto badarg; @@ -486,11 +483,6 @@ BIF_RETTYPE demonitor(Process *c_p, Eterm ref, Eterm *multip) unlock_link = 0; } else { /* Local monitor */ - if (deref_de) { - deref_de = 0; - erts_deref_dist_entry(dep); - } - dep = NULL; demonitor_local_process(c_p, ref, to, &res); } break; @@ -505,11 +497,6 @@ done: if (unlock_link) erts_proc_unlock(c_p, ERTS_PROC_LOCK_LINK); - if (deref_de) { - ASSERT(dep); - erts_deref_dist_entry(dep); - } - ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); BIF_RET(res); } @@ -844,7 +831,6 @@ BIF_RETTYPE monitor_2(BIF_ALIST_2) Eterm target = BIF_ARG_2; BIF_RETTYPE ret; DistEntry *dep = NULL; - int deref_de = 0; /* Only process monitors are implemented */ switch (BIF_ARG_1) { @@ -904,21 +890,14 @@ local_port: } dep = erts_sysname_to_connected_dist_entry(remote_node); if (dep == erts_this_dist_entry) { - deref_de = 1; ret = local_name_monitor(BIF_P, BIF_ARG_1, name); } else { - if (dep) - deref_de = 1; ret = remote_monitor(BIF_P, BIF_ARG_1, BIF_ARG_2, dep, name, 1); } } else { badarg: ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); } - if (deref_de) { - deref_de = 0; - erts_deref_dist_entry(dep); - } return ret; } @@ -1271,7 +1250,11 @@ BIF_RETTYPE hibernate_3(BIF_ALIST_3) */ Eterm reg[3]; - if (erts_hibernate(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, reg)) { + reg[0] = BIF_ARG_1; + reg[1] = BIF_ARG_2; + reg[2] = BIF_ARG_3; + + if (erts_hibernate(BIF_P, reg)) { /* * If hibernate succeeded, TRAP. The process will be wait in a * hibernated state if its state is inactive (!ERTS_PSFLG_ACTIVE); @@ -2000,6 +1983,7 @@ static Sint remote_send(Process *p, DistEntry *dep, ASSERT(is_atom(to) || is_external_pid(to)); + ctx->dep = dep; code = erts_dsig_prepare(&ctx->dsd, dep, p, ERTS_DSP_NO_LOCK, !ctx->suspend); switch (code) { case ERTS_DSIG_PREP_NOT_ALIVE: @@ -2201,7 +2185,6 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx) if (dep == erts_this_dist_entry) { Eterm id; - erts_deref_dist_entry(dep); if (IS_TRACED_FL(p, F_TRACE_SEND)) trace_send(p, to, msg); if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) @@ -2224,11 +2207,9 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx) } ret = remote_send(p, dep, tp[1], to, msg, ctx); - if (ret != SEND_YIELD_CONTINUE) { - if (dep) { - erts_deref_dist_entry(dep); - } - } else { + if (ret == SEND_YIELD_CONTINUE) { + if (dep) + erts_ref_dist_entry(dep); ctx->dep_to_deref = dep; } return ret; @@ -2963,17 +2944,17 @@ BIF_RETTYPE list_to_atom_1(BIF_ALIST_1) { Eterm res; byte *buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_SZ_LIMIT); - Sint i = erts_unicode_list_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS); - + Sint written; + int i = erts_unicode_list_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS, + &written); if (i < 0) { erts_free(ERTS_ALC_T_TMP, (void *) buf); - i = erts_list_length(BIF_ARG_1); - if (i > MAX_ATOM_CHARACTERS) { + if (i == -2) { BIF_ERROR(BIF_P, SYSTEM_LIMIT); } BIF_ERROR(BIF_P, BADARG); } - res = erts_atom_put(buf, i, ERTS_ATOM_ENC_UTF8, 1); + res = erts_atom_put(buf, written, ERTS_ATOM_ENC_UTF8, 1); ASSERT(is_atom(res)); erts_free(ERTS_ALC_T_TMP, (void *) buf); BIF_RET(res); @@ -2984,8 +2965,9 @@ BIF_RETTYPE list_to_atom_1(BIF_ALIST_1) BIF_RETTYPE list_to_existing_atom_1(BIF_ALIST_1) { byte *buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_SZ_LIMIT); - Sint i = erts_unicode_list_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS); - + Sint written; + int i = erts_unicode_list_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS, + &written); if (i < 0) { error: erts_free(ERTS_ALC_T_TMP, (void *) buf); @@ -2993,7 +2975,7 @@ BIF_RETTYPE list_to_existing_atom_1(BIF_ALIST_1) } else { Eterm a; - if (erts_atom_get((char *) buf, i, &a, ERTS_ATOM_ENC_UTF8)) { + if (erts_atom_get((char *) buf, written, &a, ERTS_ATOM_ENC_UTF8)) { erts_free(ERTS_ALC_T_TMP, (void *) buf); BIF_RET(a); } else { @@ -3888,15 +3870,18 @@ BIF_RETTYPE display_string_1(BIF_ALIST_1) { Process* p = BIF_P; Eterm string = BIF_ARG_1; - Sint len = is_string(string); - char *str; + Sint len = erts_unicode_list_to_buf_len(string); + Sint written; + byte *str; + int res; - if (len <= 0) { + if (len < 0) { BIF_ERROR(p, BADARG); } - str = (char *) erts_alloc(ERTS_ALC_T_TMP, sizeof(char)*(len + 1)); - if (intlist_to_buf(string, str, len) != len) - erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); + str = (byte *) erts_alloc(ERTS_ALC_T_TMP, sizeof(char)*(len + 1)); + res = erts_unicode_list_to_buf(string, str, len, &written); + if (res != 0 || written != len) + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error (%d)\n", __FILE__, __LINE__, res); str[len] = '\0'; erts_fprintf(stderr, "%s", str); erts_free(ERTS_ALC_T_TMP, (void *) str); @@ -3912,9 +3897,6 @@ BIF_RETTYPE display_nl_0(BIF_ALIST_0) /**********************************************************************/ -#define HALT_MSG_SIZE 200 -static char halt_msg[HALT_MSG_SIZE+1]; - /* stop the system with exit code and flags */ BIF_RETTYPE halt_2(BIF_ALIST_2) { @@ -3964,16 +3946,17 @@ BIF_RETTYPE halt_2(BIF_ALIST_2) erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_exit(ERTS_ABORT_EXIT, ""); } - else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) { - Sint i; + else if (is_list(BIF_ARG_1) || BIF_ARG_1 == NIL) { +# define HALT_MSG_SIZE 200 + static byte halt_msg[4*HALT_MSG_SIZE+1]; + Sint written; - if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE)) == -1) { + if (erts_unicode_list_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE, + &written) == -1 ) { goto error; } - if (i == -2) /* truncated string */ - i = HALT_MSG_SIZE; - ASSERT(i >= 0 && i <= HALT_MSG_SIZE); - halt_msg[i] = '\0'; + ASSERT(written >= 0 && written < sizeof(halt_msg)); + halt_msg[written] = '\0'; VERBOSE(DEBUG_SYSTEM, ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); @@ -4162,7 +4145,6 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1) goto bad; if(dep == erts_this_dist_entry) { - erts_deref_dist_entry(dep); BIF_RET(make_internal_pid(make_pid_data(c, b))); } else { @@ -4182,13 +4164,10 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1) etp->data.ui[0] = make_pid_data(c, b); MSO(BIF_P).first = (struct erl_off_heap_header*) etp; - erts_deref_dist_entry(dep); BIF_RET(make_external_pid(etp)); } bad: - if (dep) - erts_deref_dist_entry(dep); if (buf) erts_free(ERTS_ALC_T_TMP, (void *) buf); BIF_ERROR(BIF_P, BADARG); @@ -4233,7 +4212,6 @@ BIF_RETTYPE list_to_port_1(BIF_ALIST_1) goto bad; if(dep == erts_this_dist_entry) { - erts_deref_dist_entry(dep); BIF_RET(make_internal_port(p)); } else { @@ -4253,13 +4231,10 @@ BIF_RETTYPE list_to_port_1(BIF_ALIST_1) etp->data.ui[0] = p; MSO(BIF_P).first = (struct erl_off_heap_header*) etp; - erts_deref_dist_entry(dep); BIF_RET(make_external_port(etp)); } bad: - if (dep) - erts_deref_dist_entry(dep); BIF_ERROR(BIF_P, BADARG); } @@ -4379,12 +4354,9 @@ BIF_RETTYPE list_to_ref_1(BIF_ALIST_1) res = make_external_ref(etp); } - erts_deref_dist_entry(dep); BIF_RET(res); bad: - if (dep) - erts_deref_dist_entry(dep); BIF_ERROR(BIF_P, BADARG); } @@ -4729,7 +4701,6 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) ref, old ? am_true : am_false); } -#if defined(ERTS_DIRTY_SCHEDULERS) } else if (BIF_ARG_1 == am_dirty_cpu_schedulers_online) { Sint old_no; if (!is_small(BIF_ARG_2)) @@ -4755,7 +4726,6 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); break; } -#endif } else if (BIF_ARG_1 == am_time_offset && ERTS_IS_ATOM_STR("finalize", BIF_ARG_2)) { ErtsTimeOffsetState res; @@ -5061,7 +5031,7 @@ void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, ep->info.mfa.module = m; ep->info.mfa.function = f; ep->info.mfa.arity = a; - ep->beam[0] = (BeamInstr) em_apply_bif; + ep->beam[0] = BeamOpCodeAddr(op_apply_bif); ep->beam[1] = (BeamInstr) bif; } @@ -5126,13 +5096,12 @@ schedule(Process *c_p, Process *dirty_shadow_proc, { ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); (void) erts_nif_export_schedule(c_p, dirty_shadow_proc, - mfa, pc, (BeamInstr) em_apply_bif, + mfa, pc, BeamOpCodeAddr(op_apply_bif), dfunc, ifunc, module, function, argc, argv); } -#ifdef ERTS_DIRTY_SCHEDULERS static BIF_RETTYPE dirty_bif_result(BIF_ALIST_1) { @@ -5175,9 +5144,7 @@ static BIF_RETTYPE dirty_bif_exception(BIF_ALIST_2) BIF_ERROR(BIF_P, freason); } -#endif /* ERTS_DIRTY_SCHEDULERS */ -extern BeamInstr* em_call_bif_e; static BIF_RETTYPE call_bif(Process *c_p, Eterm *reg, BeamInstr *I); BIF_RETTYPE @@ -5193,7 +5160,6 @@ erts_schedule_bif(Process *proc, Process *c_p, *dirty_shadow_proc; ErtsCodeMFA *mfa; -#ifdef ERTS_DIRTY_SCHEDULERS if (proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { dirty_shadow_proc = proc; c_p = proc->next; @@ -5201,7 +5167,6 @@ erts_schedule_bif(Process *proc, erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } else -#endif { dirty_shadow_proc = NULL; c_p = proc; @@ -5217,7 +5182,6 @@ erts_schedule_bif(Process *proc, * ibif - indirect bif */ -#ifdef ERTS_DIRTY_SCHEDULERS erts_aint32_t set, mask; mask = (ERTS_PSFLG_DIRTY_CPU_PROC | ERTS_PSFLG_DIRTY_IO_PROC); @@ -5241,10 +5205,6 @@ erts_schedule_bif(Process *proc, } (void) erts_atomic32_read_bset_nob(&c_p->state, mask, set); -#else - dbif = call_bif; - ibif = bif; -#endif if (i == NULL) { ERTS_INTERNAL_ERROR("Missing instruction pointer"); @@ -5257,13 +5217,13 @@ erts_schedule_bif(Process *proc, mfa = &exp->info.mfa; } #endif - else if (em_call_bif_e == (BeamInstr *) *i) { + else if (BeamIsOpCode(*i, op_call_bif_e)) { /* Pointer to bif export in i+1 */ exp = (Export *) i[1]; pc = i; mfa = &exp->info.mfa; } - else if (em_apply_bif == (BeamInstr *) *i) { + else if (BeamIsOpCode(*i, op_apply_bif)) { /* Pointer to bif in i+1, and mfa in i-3 */ pc = c_p->cp; mfa = erts_code_to_codemfa(i); @@ -5320,7 +5280,6 @@ call_bif(Process *c_p, Eterm *reg, BeamInstr *I) return ret; } -#ifdef ERTS_DIRTY_SCHEDULERS int erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *reg) @@ -5414,7 +5373,6 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * return exiting; } -#endif /* ERTS_DIRTY_SCHEDULERS */ #ifdef HARDDEBUG diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index 9b0870dee2..a2bc883dbe 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -498,10 +498,8 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, Eterm args[], int nargs); -#ifdef ERTS_DIRTY_SCHEDULERS int erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *reg); -#endif BIF_RETTYPE erts_schedule_bif(Process *proc, diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 962b00ae7b..f7b4451890 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -154,6 +154,12 @@ bif erlang:spawn_opt/1 bif erlang:setnode/2 bif erlang:setnode/3 bif erlang:dist_exit/3 +bif erlang:dist_get_stat/1 +bif erlang:dist_ctrl_input_handler/2 +bif erlang:dist_ctrl_put_data/2 +bif erlang:dist_ctrl_get_data/1 +bif erlang:dist_ctrl_get_data_notification/1 + # Static native functions in erts_internal bif erts_internal:port_info/1 @@ -679,3 +685,9 @@ bif math:ceil/1 bif math:fmod/2 bif os:set_signal/2 bif erts_internal:maps_to_list/2 + +# +# New in 20.1 +# + +bif erlang:iolist_to_iovec/1 diff --git a/erts/emulator/beam/bif_instrs.tab b/erts/emulator/beam/bif_instrs.tab index 5aa0523e06..0932b8b985 100644 --- a/erts/emulator/beam/bif_instrs.tab +++ b/erts/emulator/beam/bif_instrs.tab @@ -45,7 +45,7 @@ CALL_GUARD_BIF(BF, TmpReg, Dst) { ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; ERTS_DBG_CHK_REDS(c_p, FCALLS); - if (is_value(result)) { + if (ERTS_LIKELY(is_value(result))) { $Dst = result; $NEXT0(); } @@ -145,13 +145,13 @@ i_gc_bif1(Fail, Bif, Src, Live, Dst) { ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; ERTS_DBG_CHK_REDS(c_p, FCALLS); - if (is_value(result)) { + if (ERTS_LIKELY(is_value(result))) { $REFRESH_GEN_DEST(); $Dst = result; $NEXT0(); } - if ($Fail != 0) { /* Handle error in guard. */ - $NEXT($Fail); + if (ERTS_LIKELY($Fail != 0)) { /* Handle error in guard. */ + $JUMP($Fail); } /* Handle error in body. */ @@ -195,14 +195,14 @@ i_gc_bif2(Fail, Bif, Live, Src1, Src2, Dst) { ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; ERTS_DBG_CHK_REDS(c_p, FCALLS); - if (is_value(result)) { + if (ERTS_LIKELY(is_value(result))) { $REFRESH_GEN_DEST(); $Dst = result; $NEXT0(); } - if ($Fail != 0) { /* Handle error in guard. */ - $NEXT($Fail); + if (ERTS_LIKELY($Fail != 0)) { /* Handle error in guard. */ + $JUMP($Fail); } /* Handle error in body. */ @@ -249,15 +249,15 @@ i_gc_bif3(Fail, Bif, Live, Src2, Src3, Dst) { ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; ERTS_DBG_CHK_REDS(c_p, FCALLS); - if (is_value(result)) { + if (ERTS_LIKELY(is_value(result))) { $REFRESH_GEN_DEST(); $Dst = result; $NEXT0(); } /* Handle error in guard. */ - if ($Fail != 0) { - $NEXT($Fail); + if (ERTS_LIKELY($Fail != 0)) { + $JUMP($Fail); } /* Handle error in body. */ @@ -325,7 +325,7 @@ call_bif(Exp) { ERTS_MSACC_UPDATE_CACHE_X(); } ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); - if (is_value(result)) { + if (ERTS_LIKELY(is_value(result))) { r(0) = result; CHECK_TERM(r(0)); $NEXT0(); @@ -370,7 +370,7 @@ send() { PROCESS_MAIN_CHK_LOCKS(c_p); HTOP = HEAP_TOP(c_p); FCALLS = c_p->fcalls; - if (is_value(result)) { + if (ERTS_LIKELY(is_value(result))) { r(0) = result; CHECK_TERM(r(0)); } else if (c_p->freason == TRAP) { @@ -473,10 +473,10 @@ nif_bif.apply_bif() { /* In case we apply process_info/1,2 or load_nif/1 */ c_p->current = codemfa; - c_p->i = I; /* In case we apply check_process_code/2. */ - c_p->arity = 0; /* To allow garbage collection on ourselves - * (check_process_code/2). - */ + $SET_CP_I_ABS(I); /* In case we apply check_process_code/2. */ + c_p->arity = 0; /* To allow garbage collection on ourselves + * (check_process_code/2). + */ DTRACE_BIF_ENTRY(c_p, codemfa); SWAPOUT; @@ -520,7 +520,7 @@ nif_bif.epilogue() { SWAPIN; /* There might have been a garbage collection. */ FCALLS = c_p->fcalls; ERTS_DBG_CHK_REDS(c_p, FCALLS); - if (is_value(nif_bif_result)) { + if (ERTS_LIKELY(is_value(nif_bif_result))) { r(0) = nif_bif_result; CHECK_TERM(r(0)); SET_I(c_p->cp); diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index 48efce20e7..7556205063 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -70,7 +70,20 @@ typedef Uint dsize_t; /* Vector size type */ /* Check for small */ #define IS_USMALL(sgn,x) ((sgn) ? ((x) <= MAX_SMALL+1) : ((x) <= MAX_SMALL)) -#define IS_SSMALL(x) (((x) >= MIN_SMALL) && ((x) <= MAX_SMALL)) + +/* + * It seems that both clang and gcc will generate sub-optimal code + * for the more obvious way to write the range check: + * + * #define IS_SSMALL(x) (((x) >= MIN_SMALL) && ((x) <= MAX_SMALL)) + * + * Note that IS_SSMALL() may be used in the 32-bit emulator with + * a Uint64 argument. Therefore, we must test the size of the argument + * to ensure that the cast does not discard the high-order 32 bits. + */ +#define _IS_SSMALL32(x) (((Uint32) ((((x)) >> (SMALL_BITS-1)) + 1)) < 2) +#define _IS_SSMALL64(x) (((Uint64) ((((x)) >> (SMALL_BITS-1)) + 1)) < 2) +#define IS_SSMALL(x) (sizeof(x) == sizeof(Uint32) ? _IS_SSMALL32(x) : _IS_SSMALL64(x)) /* The heap size needed for a bignum */ #define BIG_NEED_SIZE(x) ((x) + 1) diff --git a/erts/emulator/beam/bs_instrs.tab b/erts/emulator/beam/bs_instrs.tab index a4d4afe7d4..9f03b19731 100644 --- a/erts/emulator/beam/bs_instrs.tab +++ b/erts/emulator/beam/bs_instrs.tab @@ -364,11 +364,9 @@ i_bs_init_bits_fail := bs_init_bits.fail.verify.execute; i_bs_init_bits_fail_heap := bs_init_bits.fail_heap.verify.execute; bs_init_bits.head() { - Eterm new_binary; Eterm num_bits_term; Uint num_bits; Uint alloc; - Uint num_bytes; } bs_init_bits.plain(NumBits) { @@ -410,7 +408,9 @@ bs_init_bits.verify(Fail) { } bs_init_bits.execute(Live, Dst) { - num_bytes = ((Uint64)num_bits+(Uint64)7) >> 3; + Eterm new_binary; + Uint num_bytes = ((Uint64)num_bits+(Uint64)7) >> 3; + if (num_bits & 7) { alloc += ERL_SUB_BIN_SIZE; } @@ -709,9 +709,6 @@ i_bs_validate_unicode_retract(Fail, Src, Ms) { i_bs_start_match2 := bs_start_match.fetch.execute; bs_start_match.head() { - Uint slots; - Uint live; - Eterm header; Eterm context; } @@ -720,6 +717,9 @@ bs_start_match.fetch(Src) { } bs_start_match.execute(Fail, Live, Slots, Dst) { + Uint slots; + Uint live; + Eterm header; if (!is_boxed(context)) { $FAIL($Fail); } @@ -942,13 +942,10 @@ bs_context_to_binary := ctx_to_bin.fetch.execute; i_bs_get_binary_all_reuse := ctx_to_bin.fetch_bin.execute; ctx_to_bin.head() { - Eterm context; - ErlBinMatchBuffer* mb; - ErlSubBin* sb; - Uint size; - Uint offs; - Uint orig; - Uint hole_size; + Eterm context; + ErlBinMatchBuffer* mb; + Uint size; + Uint offs; } ctx_to_bin.fetch(Src) { @@ -976,8 +973,9 @@ ctx_to_bin.fetch_bin(Src, Fail, Unit) { } ctx_to_bin.execute() { - orig = mb->orig; - sb = (ErlSubBin *) boxed_val(context); + Uint hole_size; + Uint orig = mb->orig; + ErlSubBin* sb = (ErlSubBin *) boxed_val(context); hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE; sb->thing_word = HEADER_SUB_BIN; sb->size = BYTE_OFFSET(size); diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h index 9e3280cd98..42976d2301 100644 --- a/erts/emulator/beam/code_ix.h +++ b/erts/emulator/beam/code_ix.h @@ -176,7 +176,7 @@ int erts_has_code_write_permission(void); ERTS_GLB_INLINE BeamInstr *erts_codeinfo_to_code(ErtsCodeInfo *ci) { - ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI) || !ci->op); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI) || !ci->op); ASSERT_MFA(&ci->mfa); return (BeamInstr*)(ci + 1); } @@ -185,7 +185,7 @@ ERTS_GLB_INLINE ErtsCodeInfo *erts_code_to_codeinfo(BeamInstr *I) { ErtsCodeInfo *ci = ((ErtsCodeInfo *)(((char *)(I)) - sizeof(ErtsCodeInfo))); - ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI) || !ci->op); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI) || !ci->op); ASSERT_MFA(&ci->mfa); return ci; } diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 491c4d378e..bc168fc58d 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -121,7 +121,7 @@ Export* dexit_trap = NULL; Export* dmonitor_p_trap = NULL; /* local variables */ - +static Export *dist_ctrl_put_data_trap; /* forward declarations */ @@ -156,9 +156,7 @@ create_cache(DistEntry *dep) int i; ErtsAtomCache *cp; - ERTS_LC_ASSERT( - is_internal_port(dep->cid) - && erts_lc_is_port_locked(erts_port_lookup_raw(dep->cid))); + ERTS_LC_ASSERT(is_nil(dep->cid)); ASSERT(!dep->cache); dep->cache = cp = (ErtsAtomCache*) erts_alloc(ERTS_ALC_T_DCACHE, @@ -176,11 +174,13 @@ Uint erts_dist_cache_size(void) } static ErtsProcList * -get_suspended_on_de(DistEntry *dep, Uint32 unset_qflgs) +get_suspended_on_de(DistEntry *dep, erts_aint32_t unset_qflgs) { + erts_aint32_t qflgs; ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&dep->qlock)); - dep->qflgs &= ~unset_qflgs; - if (dep->qflgs & ERTS_DE_QFLG_EXIT) { + qflgs = erts_atomic32_read_band_acqb(&dep->qflgs, ~unset_qflgs); + qflgs &= ~unset_qflgs; + if (qflgs & ERTS_DE_QFLG_EXIT) { /* No resume when exit has been scheduled */ return NULL; } @@ -446,7 +446,35 @@ inc_no_nodes(void) #endif erts_atomic_inc_mb(&no_nodes); } - + +static void +kill_dist_ctrl_proc(void *vpid) +{ + Eterm pid = (Eterm) vpid; + ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; + Process *rp = erts_pid2proc(NULL, 0, pid, rp_locks); + if (rp) { + erts_send_exit_signal(NULL, rp->common.id, rp, &rp_locks, + am_kill, NIL, NULL, 0); + if (rp_locks) + erts_proc_unlock(rp, rp_locks); + } +} + +static void +schedule_kill_dist_ctrl_proc(Eterm pid) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + int sched_id = 1; + if (!esdp || ERTS_SCHEDULER_IS_DIRTY(esdp)) + sched_id = 1; + else + sched_id = (int) esdp->no; + erts_schedule_misc_aux_work(sched_id, + kill_dist_ctrl_proc, + (void *) (UWord) pid); +} + /* * proc is currently running or exiting process. */ @@ -456,58 +484,62 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) if (dep == erts_this_dist_entry) { /* Net kernel has died (clean up!!) */ DistEntry *tdep; - int no_dist_port = 0; + int no_dist_ctrl = 0; Eterm nd_reason = (reason == am_no_network ? am_no_network : am_net_kernel_terminated); erts_rwmtx_rlock(&erts_dist_table_rwmtx); for (tdep = erts_hidden_dist_entries; tdep; tdep = tdep->next) - no_dist_port++; + no_dist_ctrl++; for (tdep = erts_visible_dist_entries; tdep; tdep = tdep->next) - no_dist_port++; + no_dist_ctrl++; /* KILL all port controllers */ - if (no_dist_port == 0) + if (no_dist_ctrl == 0) erts_rwmtx_runlock(&erts_dist_table_rwmtx); else { Eterm def_buf[128]; int i = 0; - Eterm *dist_port; + Eterm *dist_ctrl; - if (no_dist_port <= sizeof(def_buf)/sizeof(def_buf[0])) - dist_port = &def_buf[0]; + if (no_dist_ctrl <= sizeof(def_buf)/sizeof(def_buf[0])) + dist_ctrl = &def_buf[0]; else - dist_port = erts_alloc(ERTS_ALC_T_TMP, - sizeof(Eterm)*no_dist_port); + dist_ctrl = erts_alloc(ERTS_ALC_T_TMP, + sizeof(Eterm)*no_dist_ctrl); for (tdep = erts_hidden_dist_entries; tdep; tdep = tdep->next) { - ASSERT(is_internal_port(tdep->cid)); - dist_port[i++] = tdep->cid; + ASSERT(is_internal_port(tdep->cid) || is_internal_pid(tdep->cid)); + dist_ctrl[i++] = tdep->cid; } for (tdep = erts_visible_dist_entries; tdep; tdep = tdep->next) { - ASSERT(is_internal_port(tdep->cid)); - dist_port[i++] = tdep->cid; + ASSERT(is_internal_port(tdep->cid) || is_internal_pid(tdep->cid)); + dist_ctrl[i++] = tdep->cid; } erts_rwmtx_runlock(&erts_dist_table_rwmtx); - for (i = 0; i < no_dist_port; i++) { - Port *prt = erts_port_lookup(dist_port[i], - ERTS_PORT_SFLGS_INVALID_LOOKUP); - if (!prt) - continue; - ASSERT(erts_atomic32_read_nob(&prt->state) - & ERTS_PORT_SFLG_DISTRIBUTION); - - erts_port_exit(NULL, ERTS_PORT_SIG_FLG_FORCE_SCHED, - prt, dist_port[i], nd_reason, NULL); + for (i = 0; i < no_dist_ctrl; i++) { + if (is_internal_pid(dist_ctrl[i])) + schedule_kill_dist_ctrl_proc(dist_ctrl[i]); + else { + Port *prt = erts_port_lookup(dist_ctrl[i], + ERTS_PORT_SFLGS_INVALID_LOOKUP); + if (prt) { + ASSERT(erts_atomic32_read_nob(&prt->state) + & ERTS_PORT_SFLG_DISTRIBUTION); + + erts_port_exit(NULL, ERTS_PORT_SIG_FLG_FORCE_SCHED, + prt, dist_ctrl[i], nd_reason, NULL); + } + } } - if (dist_port != &def_buf[0]) - erts_free(ERTS_ALC_T_TMP, dist_port); + if (dist_ctrl != &def_buf[0]) + erts_free(ERTS_ALC_T_TMP, dist_ctrl); } /* - * When last dist port exits, node will be taken + * When last dist ctrl exits, node will be taken * from alive to not alive. */ ASSERT(is_nil(nodedown.reason) && !nodedown.bp); @@ -524,7 +556,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) &nodedown.bp->off_heap); } } - else { /* Call from distribution port */ + else { /* Call from distribution controller (port/process) */ NetExitsContext nec = {dep}; ErtsLink *nlinks; ErtsLink *node_links; @@ -534,24 +566,23 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) erts_atomic_set_mb(&dep->dist_cmd_scheduled, 1); erts_de_rwlock(dep); - ERTS_LC_ASSERT(is_internal_port(dep->cid) - && erts_lc_is_port_locked(erts_port_lookup_raw(dep->cid))); + if (is_internal_port(dep->cid)) { + ERTS_LC_ASSERT(erts_lc_is_port_locked(erts_port_lookup_raw(dep->cid))); - if (erts_port_task_is_scheduled(&dep->dist_cmd)) - erts_port_task_abort(&dep->dist_cmd); + if (erts_port_task_is_scheduled(&dep->dist_cmd)) + erts_port_task_abort(&dep->dist_cmd); + } if (dep->status & ERTS_DE_SFLG_EXITING) { #ifdef DEBUG - erts_mtx_lock(&dep->qlock); - ASSERT(dep->qflgs & ERTS_DE_QFLG_EXIT); - erts_mtx_unlock(&dep->qlock); + ASSERT(erts_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT); #endif } else { dep->status |= ERTS_DE_SFLG_EXITING; erts_mtx_lock(&dep->qlock); - ASSERT(!(dep->qflgs & ERTS_DE_QFLG_EXIT)); - dep->qflgs |= ERTS_DE_QFLG_EXIT; + ASSERT(!(erts_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT)); + erts_atomic32_read_bor_relb(&dep->qflgs, ERTS_DE_QFLG_EXIT); erts_mtx_unlock(&dep->qlock); } @@ -616,6 +647,9 @@ void init_dist(void) dgroup_leader_trap = trap_function(am_dgroup_leader,2); dexit_trap = trap_function(am_dexit, 2); dmonitor_p_trap = trap_function(am_dmonitor_p, 2); + dist_ctrl_put_data_trap = erts_export_put(am_erts_internal, + am_dist_ctrl_put_data, + 2); } #define ErtsDistOutputBuf2Binary(OB) \ @@ -658,6 +692,8 @@ static void clear_dist_entry(DistEntry *dep) ErtsDistOutputBuf *obuf; erts_de_rwlock(dep); + erts_atomic_set_nob(&dep->input_handler, + (erts_aint_t) NIL); cache = dep->cache; dep->cache = NULL; @@ -671,6 +707,9 @@ static void clear_dist_entry(DistEntry *dep) erts_mtx_lock(&dep->qlock); + erts_atomic64_set_nob(&dep->in, 0); + erts_atomic64_set_nob(&dep->out, 0); + if (!dep->out_queue.last) obuf = dep->finalized_out_queue.first; else { @@ -678,8 +717,15 @@ static void clear_dist_entry(DistEntry *dep) obuf = dep->out_queue.first; } + if (dep->tmp_out_queue.first) { + dep->tmp_out_queue.last->next = obuf; + obuf = dep->tmp_out_queue.first; + } + dep->out_queue.first = NULL; dep->out_queue.last = NULL; + dep->tmp_out_queue.first = NULL; + dep->tmp_out_queue.last = NULL; dep->finalized_out_queue.first = NULL; dep->finalized_out_queue.last = NULL; dep->status = 0; @@ -704,8 +750,9 @@ static void clear_dist_entry(DistEntry *dep) if (obufsize) { erts_mtx_lock(&dep->qlock); - ASSERT(dep->qsize >= obufsize); - dep->qsize -= obufsize; + ASSERT(erts_atomic_read_nob(&dep->qsize) >= obufsize); + erts_atomic_add_nob(&dep->qsize, + (erts_aint_t) -obufsize); erts_mtx_unlock(&dep->qlock); } } @@ -904,11 +951,30 @@ erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx) } #endif - if (token != NIL) - ctl = TUPLE4(&ctx->ctl_heap[0], - make_small(DOP_SEND_TT), am_Empty, remote, token); - else - ctl = TUPLE3(&ctx->ctl_heap[0], make_small(DOP_SEND), am_Empty, remote); + if (token != NIL) { + Eterm el1, el2; + if (ctx->dep->flags & DFLAG_SEND_SENDER) { + el1 = make_small(DOP_SEND_SENDER_TT); + el2 = sender->common.id; + } + else { + el1 = make_small(DOP_SEND_TT); + el2 = am_Empty; + } + ctl = TUPLE4(&ctx->ctl_heap[0], el1, el2, remote, token); + } + else { + Eterm el1, el2; + if (ctx->dep->flags & DFLAG_SEND_SENDER) { + el1 = make_small(DOP_SEND_SENDER); + el2 = sender->common.id; + } + else { + el1 = make_small(DOP_SEND); + el2 = am_Empty; + } + ctl = TUPLE3(&ctx->ctl_heap[0], el1, el2, remote); + } DTRACE6(message_send, sender_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); DTRACE7(message_send_remote, sender_name, node_name, receiver_name, @@ -1145,6 +1211,7 @@ int erts_net_message(Port *prt, ErtsLink *lnk; Uint tuple_arity; int res; + Uint32 connection_id; #ifdef ERTS_DIST_MSG_DBG ErlDrvSizeT orig_len = len; #endif @@ -1153,14 +1220,16 @@ int erts_net_message(Port *prt, ERTS_CHK_NO_PROC_LOCKS; - ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(!prt || erts_lc_is_port_locked(prt)); if (!erts_is_alive) { UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); return 0; } - if (hlen != 0) - goto data_error; + + + ASSERT(hlen == 0); + if (len == 0) { /* HANDLE TICK !!! */ UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); return 0; @@ -1179,30 +1248,31 @@ int erts_net_message(Port *prt, len--; } - if (len == 0) { - PURIFY_MSG("data error"); - goto data_error; - } - - res = erts_prepare_dist_ext(&ede, t, len, dep, dep->cache); + res = erts_prepare_dist_ext(&ede, t, len, dep, dep->cache, &connection_id); - if (res >= 0) - res = ctl_len = erts_decode_dist_ext_size(&ede); - else { + switch (res) { + case ERTS_PREP_DIST_EXT_CLOSED: + return 0; /* Connection not alive; ignore signal... */ + case ERTS_PREP_DIST_EXT_FAILED: #ifdef ERTS_DIST_MSG_DBG erts_fprintf(stderr, "DIST MSG DEBUG: erts_prepare_dist_ext() failed:\n"); bw(buf, orig_len); #endif - ctl_len = 0; - } - - if (res < 0) { + goto data_error; + case ERTS_PREP_DIST_EXT_SUCCESS: + ctl_len = erts_decode_dist_ext_size(&ede); + if (ctl_len < 0) { #ifdef ERTS_DIST_MSG_DBG - erts_fprintf(stderr, "DIST MSG DEBUG: erts_decode_dist_ext_size(CTL) failed:\n"); - bw(buf, orig_len); + erts_fprintf(stderr, "DIST MSG DEBUG: erts_decode_dist_ext_size(CTL) failed:\n"); + bw(buf, orig_len); #endif - PURIFY_MSG("data error"); - goto data_error; + PURIFY_MSG("data error"); + goto data_error; + } + break; + default: + ERTS_INTERNAL_ERROR("Unexpected result from erts_prepare_dist_ext()"); + break; } if (ctl_len > DIST_CTL_DEFAULT_SIZE) { @@ -1233,6 +1303,7 @@ int erts_net_message(Port *prt, } token_size = 0; + token = NIL; switch (type = unsigned_val(tuple[1])) { case DOP_LINK: @@ -1462,38 +1533,52 @@ int erts_net_message(Port *prt, } break; + case DOP_SEND_SENDER_TT: { + Uint xsize; case DOP_SEND_TT: + if (tuple_arity != 4) { goto invalid_message; } - - token_size = size_object(tuple[4]); - /* Fall through ... */ + + token = tuple[4]; + token_size = size_object(token); + xsize = ERTS_HEAP_FRAG_SIZE(token_size); + goto send_common; + + case DOP_SEND_SENDER: case DOP_SEND: + + token = NIL; + xsize = 0; + if (tuple_arity != 3) + goto invalid_message; + + send_common: + /* - * There is intentionally no testing of the cookie (it is always '') - * from R9B and onwards. + * If DOP_SEND_SENDER or DOP_SEND_SENDER_TT element 2 contains + * the sender pid (i.e. DFLAG_SEND_SENDER is set); otherwise, + * the atom '' (empty cookie). */ + ASSERT((type == DOP_SEND_SENDER || type == DOP_SEND_SENDER_TT) + ? (is_pid(tuple[2]) && (dep->flags & DFLAG_SEND_SENDER)) + : tuple[2] == am_Empty); + #ifdef ERTS_DIST_MSG_DBG dist_msg_dbg(&ede, "MSG", buf, orig_len); #endif - if (type != DOP_SEND_TT && tuple_arity != 3) { - goto invalid_message; - } to = tuple[3]; if (is_not_pid(to)) { goto invalid_message; } rp = erts_proc_lookup(to); if (rp) { - Uint xsize = type == DOP_SEND ? 0 : ERTS_HEAP_FRAG_SIZE(token_size); ErtsProcLocks locks = 0; ErtsDistExternal *ede_copy; ede_copy = erts_make_dist_ext_copy(&ede, xsize); - if (type == DOP_SEND) { - token = NIL; - } else { + if (is_not_nil(token)) { ErlHeapFragment *heap_frag; ErlOffHeap *ohp; ASSERT(xsize); @@ -1501,15 +1586,15 @@ int erts_net_message(Port *prt, ERTS_INIT_HEAP_FRAG(heap_frag, token_size, token_size); hp = heap_frag->mem; ohp = &heap_frag->off_heap; - token = tuple[4]; token = copy_struct(token, token_size, &hp, ohp); } - erts_queue_dist_message(rp, locks, ede_copy, token, tuple[2]); + erts_queue_dist_message(rp, locks, ede_copy, token, am_Empty); if (locks) erts_proc_unlock(rp, locks); } break; + } case DOP_MONITOR_P_EXIT: { /* We are monitoring a process on the remote node which dies, we get @@ -1723,7 +1808,7 @@ decode_error: } data_error: UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); - erts_deliver_port_exit(prt, dep->cid, am_killed, 0, 1); + erts_kill_dist_connection(dep, connection_id); ERTS_CHK_NO_PROC_LOCKS; return -1; } @@ -1744,6 +1829,31 @@ static int dsig_send_ctl(ErtsDSigData* dsdp, Eterm ctl, int force_busy) return ret; } +static ERTS_INLINE void +notify_dist_data(Process *c_p, Eterm pid) +{ + Process *rp; + ErtsProcLocks rp_locks; + + ASSERT(erts_get_scheduler_data() + && !ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); + ASSERT(is_internal_pid(pid)); + + if (c_p && c_p->common.id == pid) { + rp = c_p; + rp_locks = ERTS_PROC_LOCK_MAIN; + } + else { + rp = erts_proc_lookup(pid); + rp_locks = 0; + } + + if (rp) { + ErtsMessage *mp = erts_alloc_message(0, NULL); + erts_queue_message(rp, rp_locks, mp, am_dist_data, am_system); + } +} + int erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) { @@ -1859,12 +1969,32 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) free_dist_obuf(ctx->obuf); } else { + Sint qsize; + erts_aint32_t qflgs; ErtsProcList *plp = NULL; + Eterm notify_proc = NIL; + Sint obsz = size_obuf(ctx->obuf); + erts_mtx_lock(&dep->qlock); - dep->qsize += size_obuf(ctx->obuf); - if (dep->qsize >= erts_dist_buf_busy_limit) - dep->qflgs |= ERTS_DE_QFLG_BUSY; - if (!ctx->force_busy && (dep->qflgs & ERTS_DE_QFLG_BUSY)) { + qsize = erts_atomic_add_read_nob(&dep->qsize, (erts_aint_t) obsz); + ASSERT(qsize >= obsz); + qflgs = erts_atomic32_read_nob(&dep->qflgs); + if (!(qflgs & ERTS_DE_QFLG_BUSY) && qsize >= erts_dist_buf_busy_limit) { + erts_atomic32_read_bor_relb(&dep->qflgs, ERTS_DE_QFLG_BUSY); + qflgs |= ERTS_DE_QFLG_BUSY; + } + if (qsize == obsz && (qflgs & ERTS_DE_QFLG_REQ_INFO)) { + /* Previously empty queue and info requested... */ + qflgs = erts_atomic32_read_band_mb(&dep->qflgs, + ~ERTS_DE_QFLG_REQ_INFO); + if (qflgs & ERTS_DE_QFLG_REQ_INFO) { + notify_proc = dep->cid; + ASSERT(is_internal_pid(notify_proc)); + } + /* else: requester will send itself the message... */ + qflgs &= ~ERTS_DE_QFLG_REQ_INFO; + } + if (!ctx->force_busy && (qflgs & ERTS_DE_QFLG_BUSY)) { erts_mtx_unlock(&dep->qlock); plp = erts_proclist_create(ctx->c_p); @@ -1881,7 +2011,8 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) dep->out_queue.last = ctx->obuf; if (!ctx->force_busy) { - if (!(dep->qflgs & ERTS_DE_QFLG_BUSY)) { + qflgs = erts_atomic32_read_nob(&dep->qflgs); + if (!(qflgs & ERTS_DE_QFLG_BUSY)) { if (suspended) resume = 1; /* was busy when we started, but isn't now */ #ifdef USE_VM_PROBES @@ -1906,8 +2037,11 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) } erts_mtx_unlock(&dep->qlock); - erts_schedule_dist_command(NULL, dep); + if (is_internal_port(dep->cid)) + erts_schedule_dist_command(NULL, dep); erts_de_runlock(dep); + if (is_internal_pid(notify_proc)) + notify_dist_data(ctx->c_p, notify_proc); if (resume) { erts_resume(ctx->c_p, ERTS_PROC_LOCK_MAIN); @@ -1961,16 +2095,20 @@ static Uint dist_port_command(Port *prt, ErtsDistOutputBuf *obuf) { int fpe_was_unmasked; - Uint size = obuf->ext_endp - obuf->extp; + ErlDrvSizeT size; + char *bufp; ERTS_CHK_NO_PROC_LOCKS; ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); - if (size > (Uint) INT_MAX) - erts_exit(ERTS_DUMP_EXIT, - "Absurdly large distribution output data buffer " - "(%beu bytes) passed.\n", - size); + if (!obuf) { + size = 0; + bufp = NULL; + } + else { + size = obuf->ext_endp - obuf->extp; + bufp = (char*) obuf->extp; + } #ifdef USE_VM_PROBES if (DTRACE_ENABLED(dist_output)) { @@ -1985,11 +2123,10 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf) remote_str, size); } #endif + prt->caller = NIL; fpe_was_unmasked = erts_block_fpe(); - (*prt->drv_ptr->output)((ErlDrvData) prt->drv_data, - (char*) obuf->extp, - (int) size); + (*prt->drv_ptr->output)((ErlDrvData) prt->drv_data, bufp, size); erts_unblock_fpe(fpe_was_unmasked); return size; } @@ -1998,7 +2135,7 @@ static Uint dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) { int fpe_was_unmasked; - Uint size = obuf->ext_endp - obuf->extp; + ErlDrvSizeT size; SysIOVec iov[2]; ErlDrvBinary* bv[2]; ErlIOVec eiov; @@ -2006,25 +2143,33 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) ERTS_CHK_NO_PROC_LOCKS; ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); - if (size > (Uint) INT_MAX) - erts_exit(ERTS_DUMP_EXIT, - "Absurdly large distribution output data buffer " - "(%beu bytes) passed.\n", - size); - iov[0].iov_base = NULL; iov[0].iov_len = 0; bv[0] = NULL; - iov[1].iov_base = obuf->extp; - iov[1].iov_len = size; - bv[1] = Binary2ErlDrvBinary(ErtsDistOutputBuf2Binary(obuf)); + if (!obuf) { + size = 0; + eiov.vsize = 1; + } + else { + size = obuf->ext_endp - obuf->extp; + eiov.vsize = 2; + + iov[1].iov_base = obuf->extp; + iov[1].iov_len = size; + bv[1] = Binary2ErlDrvBinary(ErtsDistOutputBuf2Binary(obuf)); + } - eiov.vsize = 2; eiov.size = size; eiov.iov = iov; eiov.binv = bv; + if (size > (Uint) INT_MAX) + erts_exit(ERTS_DUMP_EXIT, + "Absurdly large distribution output data buffer " + "(%beu bytes) passed.\n", + size); + ASSERT(prt->drv_ptr->outputv); #ifdef USE_VM_PROBES @@ -2072,7 +2217,7 @@ erts_dist_command(Port *prt, int reds_limit) Sint reds = ERTS_PORT_REDS_DIST_CMD_START; Uint32 status; Uint32 flags; - Sint obufsize = 0; + Sint qsize, obufsize = 0; ErtsDistOutputQueue oq, foq; DistEntry *dep = prt->dist_entry; Uint (*send)(Port *prt, ErtsDistOutputBuf *obuf); @@ -2081,9 +2226,6 @@ erts_dist_command(Port *prt, int reds_limit) ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); - erts_refc_inc(&dep->refc, 1); /* Otherwise dist_entry might be - removed if port command fails */ - erts_atomic_set_mb(&dep->dist_cmd_scheduled, 0); erts_de_rlock(dep); @@ -2094,7 +2236,6 @@ erts_dist_command(Port *prt, int reds_limit) if (status & ERTS_DE_SFLG_EXITING) { erts_deliver_port_exit(prt, prt->common.id, am_killed, 0, 1); - erts_deref_dist_entry(dep); return reds + ERTS_PORT_REDS_DIST_CMD_EXIT; } @@ -2128,20 +2269,20 @@ erts_dist_command(Port *prt, int reds_limit) if (!(sched_flags & ERTS_PTS_FLG_BUSY_PORT) && foq.first) { int preempt = 0; do { - Uint size; - ErtsDistOutputBuf *fob; - - size = (*send)(prt, foq.first); - esdp->io.out += (Uint64) size; + Uint size; + ErtsDistOutputBuf *fob; + size = (*send)(prt, foq.first); + erts_atomic64_inc_nob(&dep->out); + esdp->io.out += (Uint64) size; #ifdef ERTS_RAW_DIST_MSG_DBG - erts_fprintf(stderr, ">> "); - bw(foq.first->extp, size); + erts_fprintf(stderr, ">> "); + bw(foq.first->extp, size); #endif - reds += ERTS_PORT_REDS_DIST_CMD_DATA(size); - fob = foq.first; - obufsize += size_obuf(fob); - foq.first = foq.first->next; - free_dist_obuf(fob); + reds += ERTS_PORT_REDS_DIST_CMD_DATA(size); + fob = foq.first; + obufsize += size_obuf(fob); + foq.first = foq.first->next; + free_dist_obuf(fob); sched_flags = erts_atomic32_read_nob(&prt->sched.flags); preempt = reds > reds_limit || (sched_flags & ERTS_PTS_FLG_EXIT); if (sched_flags & ERTS_PTS_FLG_BUSY_PORT) @@ -2202,31 +2343,33 @@ erts_dist_command(Port *prt, int reds_limit) } } else { + int de_busy; int preempt = 0; while (oq.first && !preempt) { - ErtsDistOutputBuf *fob; - Uint size; - oq.first->extp - = erts_encode_ext_dist_header_finalize(oq.first->extp, - dep->cache, - flags); - reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE; - if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) - *--oq.first->extp = PASS_THROUGH; /* Old node; 'pass through' - needed */ - ASSERT(&oq.first->data[0] <= oq.first->extp - && oq.first->extp < oq.first->ext_endp); - size = (*send)(prt, oq.first); - esdp->io.out += (Uint64) size; + ErtsDistOutputBuf *fob; + Uint size; + oq.first->extp + = erts_encode_ext_dist_header_finalize(oq.first->extp, + dep->cache, + flags); + reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE; + if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) + *--oq.first->extp = PASS_THROUGH; /* Old node; 'pass through' + needed */ + ASSERT(&oq.first->data[0] <= oq.first->extp + && oq.first->extp < oq.first->ext_endp); + size = (*send)(prt, oq.first); + erts_atomic64_inc_nob(&dep->out); + esdp->io.out += (Uint64) size; #ifdef ERTS_RAW_DIST_MSG_DBG - erts_fprintf(stderr, ">> "); - bw(oq.first->extp, size); + erts_fprintf(stderr, ">> "); + bw(oq.first->extp, size); #endif - reds += ERTS_PORT_REDS_DIST_CMD_DATA(size); - fob = oq.first; - obufsize += size_obuf(fob); - oq.first = oq.first->next; - free_dist_obuf(fob); + reds += ERTS_PORT_REDS_DIST_CMD_DATA(size); + fob = oq.first; + obufsize += size_obuf(fob); + oq.first = oq.first->next; + free_dist_obuf(fob); sched_flags = erts_atomic32_read_nob(&prt->sched.flags); preempt = reds > reds_limit || (sched_flags & ERTS_PTS_FLG_EXIT); if ((sched_flags & ERTS_PTS_FLG_BUSY_PORT) && oq.first && !preempt) @@ -2255,12 +2398,13 @@ erts_dist_command(Port *prt, int reds_limit) * processes. */ erts_mtx_lock(&dep->qlock); - ASSERT(dep->qsize >= obufsize); - dep->qsize -= obufsize; + de_busy = !!(erts_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_BUSY); + qsize = (Sint) erts_atomic_add_read_nob(&dep->qsize, + (erts_aint_t) -obufsize); + ASSERT(qsize >= 0); obufsize = 0; if (!(sched_flags & ERTS_PTS_FLG_BUSY_PORT) - && (dep->qflgs & ERTS_DE_QFLG_BUSY) - && dep->qsize < erts_dist_buf_busy_limit) { + && de_busy && qsize < erts_dist_buf_busy_limit) { ErtsProcList *suspendees; int resumed; suspendees = get_suspended_on_de(dep, ERTS_DE_QFLG_BUSY); @@ -2280,8 +2424,13 @@ erts_dist_command(Port *prt, int reds_limit) if (obufsize != 0) { ASSERT(obufsize > 0); erts_mtx_lock(&dep->qlock); - ASSERT(dep->qsize >= obufsize); - dep->qsize -= obufsize; +#ifdef DEBUG + qsize = (Sint) erts_atomic_add_read_nob(&dep->qsize, + (erts_aint_t) -obufsize); + ASSERT(qsize >= 0); +#else + erts_atomic_add_nob(&dep->qsize, (erts_aint_t) -obufsize); +#endif erts_mtx_unlock(&dep->qlock); } @@ -2299,8 +2448,6 @@ erts_dist_command(Port *prt, int reds_limit) if (reds > INT_MAX/2) reds = INT_MAX/2; - erts_deref_dist_entry(dep); - return reds; preempted: @@ -2337,7 +2484,7 @@ erts_dist_command(Port *prt, int reds_limit) #ifdef DEBUG erts_mtx_lock(&dep->qlock); - ASSERT(dep->qsize == obufsize); + ASSERT(erts_atomic_read_nob(&dep->qsize) == obufsize); erts_mtx_unlock(&dep->qlock); #endif } @@ -2348,7 +2495,7 @@ erts_dist_command(Port *prt, int reds_limit) * in out_queue. */ erts_mtx_lock(&dep->qlock); - dep->qsize -= obufsize; + erts_atomic_add_nob(&dep->qsize, -obufsize); obufsize = 0; oq.last->next = dep->out_queue.first; dep->out_queue.first = oq.first; @@ -2362,6 +2509,370 @@ erts_dist_command(Port *prt, int reds_limit) goto done; } +#if 0 + +int +dist_data_finalize(Process *c_p, int reds_limit) +{ + int reds = 5; + DistEntry *dep = ; + ErtsDistOutputQueue oq, foq; + ErtsDistOutputBuf *ob; + int preempt; + + + erts_mtx_lock(&dep->qlock); + flags = dep->flags; + oq.first = dep->out_queue.first; + oq.last = dep->out_queue.last; + dep->out_queue.first = NULL; + dep->out_queue.last = NULL; + erts_mtx_unlock(&dep->qlock); + + if (!oq.first) { + ASSERT(!oq.last); + oq.first = dep->tmp_out_queue.first; + oq.last = dep->tmp_out_queue.last; + } + else { + ErtsDistOutputBuf *f, *l; + ASSERT(oq.last); + if (dep->tmp_out_queue.last) { + dep->tmp_out_queue.last->next = oq.first; + oq.first = dep->tmp_out_queue.first; + } + } + + if (!oq.first) { + /* Nothing to do... */ + ASSERT(!oq.last); + return reds; + } + + foq.first = dep->finalized_out_queue.first; + foq.last = dep->finalized_out_queue.last; + + preempt = 0; + ob = oq.first; + ASSERT(ob); + + do { + ob->extp = erts_encode_ext_dist_header_finalize(ob->extp, + dep->cache, + flags); + if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) + *--ob->extp = PASS_THROUGH; /* Old node; 'pass through' + needed */ + ASSERT(&ob->data[0] <= ob->extp && ob->extp < ob->ext_endp); + reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE; + preempt = reds > reds_limit; + if (preempt) + break; + ob = ob->next; + } while (ob); + /* + * At least one buffer was finalized; if we got preempted, + * ob points to the last buffer that we finalized. + */ + if (foq.last) + foq.last->next = oq.first; + else + foq.first = oq.first; + if (!preempt) { + /* All buffers finalized */ + foq.last = oq.last; + oq.first = oq.last = NULL; + } + else { + /* Not all buffers finalized; split oq. */ + foq.last = ob; + oq.first = ob->next; + if (oq.first) + ob->next = NULL; + else + oq.last = NULL; + } + + dep->finalized_out_queue.first = foq.first; + dep->finalized_out_queue.last = foq.last; + dep->tmp_out_queue.first = oq.first; + dep->tmp_out_queue.last = oq.last; + + return reds; +} + +#endif + +BIF_RETTYPE +dist_ctrl_get_data_notification_1(BIF_ALIST_1) +{ + DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(BIF_P); + erts_aint32_t qflgs; + erts_aint_t qsize; + Eterm receiver = NIL; + + if (!dep) + BIF_ERROR(BIF_P, EXC_NOTSUP); + + if (erts_dhandle_to_dist_entry(BIF_ARG_1) != dep) + BIF_ERROR(BIF_P, BADARG); + + /* + * Caller is the only one that can consume from this queue + * and the only one that can set the req-info flag... + */ + + erts_de_rlock(dep); + + ASSERT(dep->cid == BIF_P->common.id); + + qflgs = erts_atomic32_read_acqb(&dep->qflgs); + + if (!(qflgs & ERTS_DE_QFLG_REQ_INFO)) { + qsize = erts_atomic_read_acqb(&dep->qsize); + ASSERT(qsize >= 0); + if (qsize > 0) + receiver = BIF_P->common.id; /* Notify ourselves... */ + else { /* Empty queue; set req-info flag... */ + qflgs = erts_atomic32_read_bor_mb(&dep->qflgs, + ERTS_DE_QFLG_REQ_INFO); + qsize = erts_atomic_read_acqb(&dep->qsize); + ASSERT(qsize >= 0); + if (qsize > 0) { + qflgs = erts_atomic32_read_band_mb(&dep->qflgs, + ~ERTS_DE_QFLG_REQ_INFO); + if (qflgs & ERTS_DE_QFLG_REQ_INFO) + receiver = BIF_P->common.id; /* Notify ourselves... */ + /* else: someone else will notify us... */ + } + /* else: still empty queue... */ + } + } + /* else: Already requested... */ + + erts_de_runlock(dep); + + if (is_internal_pid(receiver)) + notify_dist_data(BIF_P, receiver); + + BIF_RET(am_ok); +} + +BIF_RETTYPE +dist_ctrl_put_data_2(BIF_ALIST_2) +{ + DistEntry *dep; + ErlDrvSizeT size; + Eterm input_handler; + + if (is_binary(BIF_ARG_2)) + size = binary_size(BIF_ARG_2); + else if (is_nil(BIF_ARG_2)) + size = 0; + else if (is_list(BIF_ARG_2)) + BIF_TRAP2(dist_ctrl_put_data_trap, + BIF_P, BIF_ARG_1, BIF_ARG_2); + else + BIF_ERROR(BIF_P, BADARG); + + dep = erts_dhandle_to_dist_entry(BIF_ARG_1); + if (!dep) + BIF_ERROR(BIF_P, BADARG); + + input_handler = (Eterm) erts_atomic_read_nob(&dep->input_handler); + + if (input_handler != BIF_P->common.id) + BIF_ERROR(BIF_P, EXC_NOTSUP); + + erts_atomic64_inc_nob(&dep->in); + + if (size != 0) { + byte *data, *temp_alloc = NULL; + + data = (byte *) erts_get_aligned_binary_bytes(BIF_ARG_2, &temp_alloc); + if (!data) + BIF_ERROR(BIF_P, BADARG); + + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + + (void) erts_net_message(NULL, dep, NULL, 0, data, size); + /* + * We ignore any decode failures. On fatal failures the + * connection will be taken down by killing the + * distribution channel controller... + */ + + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + + BUMP_REDS(BIF_P, 5); + + erts_free_aligned_binary_bytes(temp_alloc); + + } + + BIF_RET(am_ok); +} + +BIF_RETTYPE +dist_get_stat_1(BIF_ALIST_1) +{ + Sint64 read, write, pend; + Eterm res, *hp, **hpp; + Uint sz, *szp; + DistEntry *dep = erts_dhandle_to_dist_entry(BIF_ARG_1); + + if (!dep) + BIF_ERROR(BIF_P, BADARG); + + erts_de_rlock(dep); + + read = (Sint64) erts_atomic64_read_nob(&dep->in); + write = (Sint64) erts_atomic64_read_nob(&dep->out); + pend = (Sint64) erts_atomic_read_nob(&dep->qsize); + + erts_de_runlock(dep); + + sz = 0; + szp = &sz; + hpp = NULL; + + while (1) { + res = erts_bld_tuple(hpp, szp, 4, + am_ok, + erts_bld_sint64(hpp, szp, read), + erts_bld_sint64(hpp, szp, write), + pend ? am_true : am_false); + if (hpp) + break; + hp = HAlloc(BIF_P, sz); + hpp = &hp; + szp = NULL; + } + + BIF_RET(res); +} + +BIF_RETTYPE +dist_ctrl_input_handler_2(BIF_ALIST_2) +{ + DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(BIF_P); + + if (!dep) + BIF_ERROR(BIF_P, EXC_NOTSUP); + + if (erts_dhandle_to_dist_entry(BIF_ARG_1) != dep) + BIF_ERROR(BIF_P, BADARG); + + if (is_not_internal_pid(BIF_ARG_2)) + BIF_ERROR(BIF_P, BADARG); + + erts_atomic_set_nob(&dep->input_handler, + (erts_aint_t) BIF_ARG_2); + + BIF_RET(am_ok); +} + +BIF_RETTYPE +dist_ctrl_get_data_1(BIF_ALIST_1) +{ + DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(BIF_P); + int reds = 1; + ErtsDistOutputBuf *obuf; + Eterm *hp; + ProcBin *pb; + erts_aint_t qsize; + + if (!dep) + BIF_ERROR(BIF_P, EXC_NOTSUP); + + if (erts_dhandle_to_dist_entry(BIF_ARG_1) != dep) + BIF_ERROR(BIF_P, BADARG); + + erts_de_rlock(dep); + + if (dep->status & ERTS_DE_SFLG_EXITING) + goto return_none; + + ASSERT(dep->cid == BIF_P->common.id); + +#if 0 + if (dep->finalized_out_queue.first) { + obuf = dep->finalized_out_queue.first; + dep->finalized_out_queue.first = obuf->next; + if (!obuf->next) + dep->finalized_out_queue.last = NULL; + } + else +#endif + { + if (!dep->tmp_out_queue.first) { + ASSERT(!dep->tmp_out_queue.last); + qsize = erts_atomic_read_acqb(&dep->qsize); + if (qsize > 0) { + erts_mtx_lock(&dep->qlock); + dep->tmp_out_queue.first = dep->out_queue.first; + dep->tmp_out_queue.last = dep->out_queue.last; + dep->out_queue.first = NULL; + dep->out_queue.last = NULL; + erts_mtx_unlock(&dep->qlock); + } + } + + if (!dep->tmp_out_queue.first) { + ASSERT(!dep->tmp_out_queue.last); + return_none: + erts_de_runlock(dep); + BIF_RET(am_none); + } + else { + obuf = dep->tmp_out_queue.first; + dep->tmp_out_queue.first = obuf->next; + if (!obuf->next) + dep->tmp_out_queue.last = NULL; + } + + obuf->extp = erts_encode_ext_dist_header_finalize(obuf->extp, + dep->cache, + dep->flags); + reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE; + if (!(dep->flags & DFLAG_DIST_HDR_ATOM_CACHE)) + *--obuf->extp = PASS_THROUGH; /* 'pass through' needed */ + ASSERT(&obuf->data[0] <= obuf->extp + && obuf->extp < obuf->ext_endp); + } + + erts_atomic64_inc_nob(&dep->out); + + erts_de_runlock(dep); + + hp = HAlloc(BIF_P, PROC_BIN_SIZE); + pb = (ProcBin *) (char *) hp; + pb->thing_word = HEADER_PROC_BIN; + pb->size = obuf->ext_endp - obuf->extp; + pb->next = MSO(BIF_P).first; + MSO(BIF_P).first = (struct erl_off_heap_header*) pb; + pb->val = ErtsDistOutputBuf2Binary(obuf); + pb->bytes = (byte*) obuf->extp; + pb->flags = 0; + + qsize = erts_atomic_add_read_nob(&dep->qsize, -size_obuf(obuf)); + ASSERT(qsize >= 0); + + if (qsize < erts_dist_buf_busy_limit/2 + && (erts_atomic32_read_acqb(&dep->qflgs) & ERTS_DE_QFLG_BUSY)) { + ErtsProcList *resume_procs = NULL; + erts_mtx_lock(&dep->qlock); + resume_procs = get_suspended_on_de(dep, ERTS_DE_QFLG_BUSY); + erts_mtx_unlock(&dep->qlock); + if (resume_procs) { + int resumed = erts_resume_processes(resume_procs); + reds += resumed*ERTS_PORT_REDS_DIST_CMD_RESUMED; + } + } + + BIF_RET2(make_binary(pb), reds); +} + void erts_dist_port_not_busy(Port *prt) { @@ -2385,18 +2896,20 @@ void erts_kill_dist_connection(DistEntry *dep, Uint32 connection_id) { erts_de_rwlock(dep); - if (is_internal_port(dep->cid) - && connection_id == dep->connection_id + if (connection_id == dep->connection_id && !(dep->status & ERTS_DE_SFLG_EXITING)) { dep->status |= ERTS_DE_SFLG_EXITING; erts_mtx_lock(&dep->qlock); - ASSERT(!(dep->qflgs & ERTS_DE_QFLG_EXIT)); - dep->qflgs |= ERTS_DE_QFLG_EXIT; + ASSERT(!(erts_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT)); + erts_atomic32_read_bor_nob(&dep->qflgs, ERTS_DE_QFLG_EXIT); erts_mtx_unlock(&dep->qlock); - erts_schedule_dist_command(NULL, dep); + if (is_internal_port(dep->cid)) + erts_schedule_dist_command(NULL, dep); + else if (is_internal_pid(dep->cid)) + schedule_kill_dist_ctrl_proc(dep->cid); } erts_de_rwunlock(dep); } @@ -2513,9 +3026,6 @@ info_dist_entry(fmtfn_t to, void *arg, DistEntry *dep, int visible, int connecte } erts_print(to, arg, "Name: %T", dep->sysname); -#ifdef DEBUG - erts_print(to, arg, " (refc=%d)", erts_refc_read(&dep->refc, 0)); -#endif erts_print(to, arg, "\n"); if (!connected && is_nil(dep->cid)) { if (dep->nlinks) { @@ -2635,17 +3145,23 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2) goto error; } - net_kernel = erts_whereis_process(BIF_P, ERTS_PROC_LOCK_MAIN, - am_net_kernel, ERTS_PROC_LOCK_MAIN, 0); - if (!net_kernel) + net_kernel = erts_whereis_process(BIF_P, + ERTS_PROC_LOCK_MAIN, + am_net_kernel, + ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS, + 0); + if (!net_kernel || ERTS_PROC_GET_DIST_ENTRY(net_kernel)) goto error; /* By setting F_DISTRIBUTION on net_kernel, - * do_net_exist will be called when net_kernel is terminated !! */ + * erts_do_net_exits will be called when net_kernel is terminated !! */ net_kernel->flags |= F_DISTRIBUTION; - if (net_kernel != BIF_P) - erts_proc_unlock(net_kernel, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(net_kernel, + (ERTS_PROC_LOCK_STATUS + | ((net_kernel != BIF_P) + ? ERTS_PROC_LOCK_MAIN + : 0))); #ifdef DEBUG erts_rwmtx_rlock(&erts_dist_table_rwmtx); @@ -2662,6 +3178,14 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2) erts_thr_progress_unblock(); erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + /* + * Note erts_this_dist_entry is changed by erts_set_this_node(), + * so we *need* to use the new one after erts_set_this_node() + * is called. + */ + erts_ref_dist_entry(erts_this_dist_entry); + ERTS_PROC_SET_DIST_ENTRY(net_kernel, erts_this_dist_entry); + BIF_RET(am_true); error: @@ -2691,18 +3215,18 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) Eterm ic, oc; Eterm *tp; DistEntry *dep = NULL; + ErtsProcLocks proc_unlock = 0; + Process *proc; Port *pp = NULL; - /* Prepare for success */ - ERTS_BIF_PREP_RET(ret, am_true); - /* * Check and pick out arguments */ if (!is_node_name_atom(BIF_ARG_1) || - is_not_internal_port(BIF_ARG_2) || - (erts_this_node->sysname == am_Noname)) { + !(is_internal_port(BIF_ARG_2) + || is_internal_pid(BIF_ARG_2)) + || (erts_this_node->sysname == am_Noname)) { goto badarg; } @@ -2746,77 +3270,124 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) else if (!dep) goto system_limit; /* Should never happen!!! */ - pp = erts_id2port_sflgs(BIF_ARG_2, - BIF_P, - ERTS_PROC_LOCK_MAIN, - ERTS_PORT_SFLGS_INVALID_LOOKUP); - erts_de_rwlock(dep); + if (is_internal_pid(BIF_ARG_2)) { + if (BIF_P->common.id == BIF_ARG_2) { + proc_unlock = 0; + proc = BIF_P; + } + else { + proc_unlock = ERTS_PROC_LOCK_MAIN; + proc = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN, + BIF_ARG_2, proc_unlock); + } + erts_de_rwlock(dep); - if (!pp || (erts_atomic32_read_nob(&pp->state) - & ERTS_PORT_SFLG_EXITING)) - goto badarg; + if (!proc) + goto badarg; + else if (proc == ERTS_PROC_LOCK_BUSY) { + proc_unlock = 0; + goto yield; + } - if ((pp->drv_ptr->flags & ERL_DRV_FLAG_SOFT_BUSY) == 0) - goto badarg; + erts_proc_lock(proc, ERTS_PROC_LOCK_STATUS); + proc_unlock |= ERTS_PROC_LOCK_STATUS; + + if (ERTS_PROC_GET_DIST_ENTRY(proc)) { + if (dep == ERTS_PROC_GET_DIST_ENTRY(proc) + && (proc->flags & F_DISTRIBUTION) + && dep->cid == BIF_ARG_2) { + ERTS_BIF_PREP_RET(ret, erts_make_dhandle(BIF_P, dep)); + goto done; + } + goto badarg; + } - if (dep->cid == BIF_ARG_2 && pp->dist_entry == dep) - goto done; /* Already set */ + if (is_not_nil(dep->cid)) + goto badarg; + + proc->flags |= F_DISTRIBUTION; + ERTS_PROC_SET_DIST_ENTRY(proc, dep); + + proc_unlock &= ~ERTS_PROC_LOCK_STATUS; + erts_proc_unlock(proc, ERTS_PROC_LOCK_STATUS); + + dep->send = NULL; /* Only for distr ports... */ - if (dep->status & ERTS_DE_SFLG_EXITING) { - /* Suspend on dist entry waiting for the exit to finish */ - ErtsProcList *plp = erts_proclist_create(BIF_P); - plp->next = NULL; - erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); - erts_mtx_lock(&dep->qlock); - erts_proclist_store_last(&dep->suspended, plp); - erts_mtx_unlock(&dep->qlock); - goto yield; } + else { - ASSERT(!(dep->status & ERTS_DE_SFLG_EXITING)); + pp = erts_id2port_sflgs(BIF_ARG_2, + BIF_P, + ERTS_PROC_LOCK_MAIN, + ERTS_PORT_SFLGS_INVALID_LOOKUP); + erts_de_rwlock(dep); - if (pp->dist_entry || is_not_nil(dep->cid)) - goto badarg; + if (!pp || (erts_atomic32_read_nob(&pp->state) + & ERTS_PORT_SFLG_EXITING)) + goto badarg; - erts_atomic32_read_bor_nob(&pp->state, ERTS_PORT_SFLG_DISTRIBUTION); + if ((pp->drv_ptr->flags & ERL_DRV_FLAG_SOFT_BUSY) == 0) + goto badarg; - /* - * Dist-ports do not use the "busy port message queue" functionality, but - * instead use "busy dist entry" functionality. - */ - { - ErlDrvSizeT disable = ERL_DRV_BUSY_MSGQ_DISABLED; - erl_drv_busy_msgq_limits(ERTS_Port2ErlDrvPort(pp), &disable, NULL); - } + if (dep->cid == BIF_ARG_2 && pp->dist_entry == dep) { + ERTS_BIF_PREP_RET(ret, erts_make_dhandle(BIF_P, dep)); + goto done; /* Already set */ + } - pp->dist_entry = dep; + if (dep->status & ERTS_DE_SFLG_EXITING) { + /* Suspend on dist entry waiting for the exit to finish */ + ErtsProcList *plp = erts_proclist_create(BIF_P); + plp->next = NULL; + erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); + erts_mtx_lock(&dep->qlock); + erts_proclist_store_last(&dep->suspended, plp); + erts_mtx_unlock(&dep->qlock); + goto yield; + } - dep->version = version; - dep->creation = 0; + ASSERT(!(dep->status & ERTS_DE_SFLG_EXITING)); - ASSERT(pp->drv_ptr->outputv || pp->drv_ptr->output); + if (pp->dist_entry || is_not_nil(dep->cid)) + goto badarg; -#if 1 - dep->send = (pp->drv_ptr->outputv - ? dist_port_commandv - : dist_port_command); -#else - dep->send = dist_port_command; -#endif - ASSERT(dep->send); + erts_atomic32_read_bor_nob(&pp->state, ERTS_PORT_SFLG_DISTRIBUTION); + + pp->dist_entry = dep; + + ASSERT(pp->drv_ptr->outputv || pp->drv_ptr->output); + + dep->send = (pp->drv_ptr->outputv + ? dist_port_commandv + : dist_port_command); + ASSERT(dep->send); + + /* + * Dist-ports do not use the "busy port message queue" functionality, but + * instead use "busy dist entry" functionality. + */ + { + ErlDrvSizeT disable = ERL_DRV_BUSY_MSGQ_DISABLED; + erl_drv_busy_msgq_limits(ERTS_Port2ErlDrvPort(pp), &disable, NULL); + } + + } + + dep->version = version; + dep->creation = 0; #ifdef DEBUG - erts_mtx_lock(&dep->qlock); - ASSERT(dep->qsize == 0); - erts_mtx_unlock(&dep->qlock); + ASSERT(erts_atomic_read_nob(&dep->qsize) == 0); #endif - erts_set_dist_entry_connected(dep, BIF_ARG_2, flags); - if (flags & DFLAG_DIST_HDR_ATOM_CACHE) create_cache(dep); + erts_set_dist_entry_connected(dep, BIF_ARG_2, flags); + erts_de_rwunlock(dep); + + ERTS_BIF_PREP_RET(ret, erts_make_dhandle(BIF_P, dep)); + dep = NULL; /* inc of refc transferred to port (dist_entry field) */ inc_no_nodes(); @@ -2836,6 +3407,9 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) if (pp) erts_port_release(pp); + if (proc_unlock) + erts_proc_unlock(proc, proc_unlock); + return ret; yield: @@ -3138,7 +3712,6 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options) erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); done: - erts_deref_dist_entry(dep); BIF_RET(am_true); } diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index 05016cafc5..d4765c50b8 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -44,6 +44,7 @@ #define DFLAG_UTF8_ATOMS 0x10000 #define DFLAG_MAP_TAG 0x20000 #define DFLAG_BIG_CREATION 0x40000 +#define DFLAG_SEND_SENDER 0x80000 /* All flags that should be enabled when term_to_binary/1 is used. */ #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES \ @@ -74,6 +75,9 @@ #define DOP_DEMONITOR_P 20 #define DOP_MONITOR_P_EXIT 21 +#define DOP_SEND_SENDER 22 +#define DOP_SEND_SENDER_TT 23 + /* distribution trap functions */ extern Export* dsend2_trap; extern Export* dsend3_trap; @@ -161,13 +165,10 @@ erts_dsig_prepare(ErtsDSigData *dsdp, goto fail; } if (no_suspend) { - failure = ERTS_DSIG_PREP_CONNECTED; - erts_mtx_lock(&dep->qlock); - if (dep->qflgs & ERTS_DE_QFLG_BUSY) + if (erts_atomic32_read_acqb(&dep->qflgs) & ERTS_DE_QFLG_BUSY) { failure = ERTS_DSIG_PREP_WOULD_SUSPEND; - erts_mtx_unlock(&dep->qlock); - if (failure == ERTS_DSIG_PREP_WOULD_SUSPEND) goto fail; + } } dsdp->proc = proc; dsdp->dep = dep; @@ -349,6 +350,7 @@ typedef struct { Eterm ctl_heap[6]; ErtsDSigData dsd; DistEntry* dep_to_deref; + DistEntry *dep; struct erts_dsig_send_context dss; Eterm return_term; diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index aee54ad0a8..88285d8be6 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -385,6 +385,7 @@ set_default_temp_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; + ip->disable_allowed = 0; ip->carrier_migration_allowed = 0; ip->atype = AFIT; ip->init.util.name_prefix = "temp_"; @@ -644,8 +645,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) = ERTS_MONITOR_SH_SIZE * sizeof(Uint); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NLINK_SH)] = ERTS_LINK_SH_SIZE * sizeof(Uint); - fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_EV_D_STATE)] - = sizeof(ErtsDrvEventDataState); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_SEL_D_STATE)] = sizeof(ErtsDrvSelectDataState); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_SEL_D_STATE)] @@ -1494,8 +1493,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) &init->ll_alloc, &init->driver_alloc, &init->fix_alloc, - &init->sl_alloc, - &init->temp_alloc + &init->sl_alloc /* test_alloc not affected by +Mea??? or +Mu??? */ }; int aui_sz = (int) sizeof(aui)/sizeof(aui[0]); @@ -1841,9 +1839,7 @@ erts_alloc_register_scheduler(void *vesdp) int ix = (int) esdp->no; int aix; -#ifdef ERTS_DIRTY_SCHEDULERS ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); -#endif for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) { ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix]; esdp->alloc_data.deallctr[aix] = NULL; diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index c661d0b226..117f96a4ad 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -402,6 +402,32 @@ NAME##_free(TYPE *p) \ erts_free(ALCT, (void *) p); \ } +#define ERTS_THR_PREF_AUX(NAME, TYPE, PASZ) \ +ERTS_THR_PREF_PRE_ALLOC_IMPL(NAME##_pre, TYPE, PASZ) + +#define ERTS_THR_PREF_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT) \ +ERTS_THR_PREF_AUX(NAME, TYPE, PASZ) \ +static void \ +init_##NAME##_alloc(int nthreads) \ +{ \ + init_##NAME##_pre_alloc(nthreads); \ +} \ +static ERTS_INLINE TYPE * \ +NAME##_alloc(void) \ +{ \ + TYPE *res = NAME##_pre_alloc(); \ + if (!res) \ + res = erts_alloc(ALCT, sizeof(TYPE)); \ + return res; \ +} \ +static ERTS_INLINE void \ +NAME##_free(TYPE *p) \ +{ \ + if (!NAME##_pre_free(p)) \ + erts_free(ALCT, (void *) p); \ +} + + #ifdef DEBUG #define ERTS_PRE_ALLOC_SIZE(SZ) ((SZ) < 1000 ? (SZ)/10 + 10 : 100) #define ERTS_PRE_ALLOC_CLOBBER(P, T) memset((void *) (P), 0xfd, sizeof(T)) @@ -482,7 +508,8 @@ init_##NAME##_alloc(void) \ { \ sspa_data_##NAME##__ = \ erts_sspa_create(sizeof(union erts_sspa_##NAME##__), \ - ERTS_PRE_ALLOC_SIZE((PASZ))); \ + ERTS_PRE_ALLOC_SIZE((PASZ)), \ + 0, NULL); \ } \ \ static TYPE * \ @@ -504,6 +531,57 @@ NAME##_free(TYPE *p) \ (char *) p); \ } + +#define ERTS_THR_PREF_PRE_ALLOC_IMPL(NAME, TYPE, PASZ) \ +union erts_sspa_##NAME##__ { \ + erts_sspa_blk_t next; \ + TYPE type; \ +}; \ + \ +static erts_sspa_data_t *sspa_data_##NAME##__; \ + \ +static void \ +init_##NAME##_alloc(int nthreads) \ +{ \ + sspa_data_##NAME##__ = \ + erts_sspa_create(sizeof(union erts_sspa_##NAME##__), \ + ERTS_PRE_ALLOC_SIZE((PASZ)), \ + nthreads, \ + #NAME); \ +} \ + \ +void \ +erts_##NAME##_alloc_init_thread(void) \ +{ \ + int id = erts_atomic_inc_read_nob(&sspa_data_##NAME##__->id_generator);\ + if (id > sspa_data_##NAME##__->nthreads) { \ + erts_exit(ERTS_ABORT_EXIT, \ + "%s:%d:%s(): Too many threads for '" #NAME "'\n", \ + __FILE__, __LINE__, __func__); \ + } \ + erts_tsd_set(sspa_data_##NAME##__->tsd_key, (void*)(SWord)id); \ +} \ + \ +static TYPE * \ +NAME##_alloc(void) \ +{ \ + int id = (int)(SWord)erts_tsd_get(sspa_data_##NAME##__->tsd_key); \ + if (id == 0) \ + return NULL; \ + return (TYPE *) erts_sspa_alloc(sspa_data_##NAME##__, \ + id-1); \ +} \ + \ +static int \ +NAME##_free(TYPE *p) \ +{ \ + int id = (int)(SWord)erts_tsd_get(sspa_data_##NAME##__->tsd_key); \ + return erts_sspa_free(sspa_data_##NAME##__, \ + id - 1, \ + (char *) p); \ +} + + #ifdef DEBUG #define ERTS_ALC_DBG_BLK_SZ(PTR) (*(((UWord *) (PTR)) - 2)) #endif /* #ifdef DEBUG */ diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 8142ea8893..878b971b07 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -258,6 +258,8 @@ type MREF_ENT STANDARD SYSTEM magic_ref_entry type MREF_TAB_BKTS STANDARD SYSTEM magic_ref_table_buckets type MREF_TAB LONG_LIVED SYSTEM magic_ref_table type MINDIRECTION FIXED_SIZE SYSTEM magic_indirection +type BINARY_FIND SHORT_LIVED PROCESSES binary_find +type OPEN_PORT_ENV TEMPORARY SYSTEM open_port_env type THR_Q_EL STANDARD SYSTEM thr_q_element type THR_Q_EL_SL FIXED_SIZE SYSTEM sl_thr_q_element @@ -350,11 +352,9 @@ type SYS_CHECK_REQ SHORT_LIVED SYSTEM system_check_request type TEMP_TERM TEMPORARY SYSTEM temp_term type DRV_TAB LONG_LIVED SYSTEM drv_tab type DRV_EV_STATE LONG_LIVED SYSTEM driver_event_state -type DRV_EV_D_STATE FIXED_SIZE SYSTEM driver_event_data_state type DRV_SEL_D_STATE FIXED_SIZE SYSTEM driver_select_data_state type NIF_SEL_D_STATE FIXED_SIZE SYSTEM enif_select_data_state type FD_LIST SHORT_LIVED SYSTEM fd_list -type ACTIVE_FD_ARR SHORT_LIVED SYSTEM active_fd_array type POLLSET LONG_LIVED SYSTEM pollset type POLLSET_UPDREQ SHORT_LIVED SYSTEM pollset_update_req type POLL_FDS LONG_LIVED SYSTEM poll_fds diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index 30d7baf769..faeb5ef368 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -230,10 +230,6 @@ void erts_lcnt_update_allocator_locks(int enable); # endif #endif -#undef MIN -#undef MAX -#define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) -#define MAX(X, Y) ((X) > (Y) ? (X) : (Y)) #define FLOOR(X, I) (((X)/(I))*(I)) #define CEILING(X, I) ((((X) - 1)/(I) + 1)*(I)) diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c index f2a3e411ec..b6625db0d3 100644 --- a/erts/emulator/beam/erl_arith.c +++ b/erts/emulator/beam/erl_arith.c @@ -114,7 +114,7 @@ BIF_RETTYPE intdiv_2(BIF_ALIST_2) } if (is_both_small(BIF_ARG_1,BIF_ARG_2)){ Sint ires = signed_val(BIF_ARG_1) / signed_val(BIF_ARG_2); - if (MY_IS_SSMALL(ires)) + if (IS_SSMALL(ires)) BIF_RET(make_small(ires)); } BIF_RET(erts_int_div(BIF_P, BIF_ARG_1, BIF_ARG_2)); @@ -340,8 +340,7 @@ erts_mixed_plus(Process* p, Eterm arg1, Eterm arg2) switch ((arg2 & _TAG_IMMED1_MASK) >> _TAG_PRIMARY_SIZE) { case (_TAG_IMMED1_SMALL >> _TAG_PRIMARY_SIZE): ires = signed_val(arg1) + signed_val(arg2); - ASSERT(MY_IS_SSMALL(ires) == IS_SSMALL(ires)); - if (MY_IS_SSMALL(ires)) { + if (IS_SSMALL(ires)) { return make_small(ires); } else { hp = HAlloc(p, 2); @@ -486,8 +485,7 @@ erts_mixed_minus(Process* p, Eterm arg1, Eterm arg2) switch ((arg2 & _TAG_IMMED1_MASK) >> _TAG_PRIMARY_SIZE) { case (_TAG_IMMED1_SMALL >> _TAG_PRIMARY_SIZE): ires = signed_val(arg1) - signed_val(arg2); - ASSERT(MY_IS_SSMALL(ires) == IS_SSMALL(ires)); - if (MY_IS_SSMALL(ires)) { + if (IS_SSMALL(ires)) { return make_small(ires); } else { hp = HAlloc(p, 2); @@ -1181,8 +1179,7 @@ erts_gc_mixed_plus(Process* p, Eterm* reg, Uint live) switch ((arg2 & _TAG_IMMED1_MASK) >> _TAG_PRIMARY_SIZE) { case (_TAG_IMMED1_SMALL >> _TAG_PRIMARY_SIZE): ires = signed_val(arg1) + signed_val(arg2); - ASSERT(MY_IS_SSMALL(ires) == IS_SSMALL(ires)); - if (MY_IS_SSMALL(ires)) { + if (IS_SSMALL(ires)) { return make_small(ires); } else { if (ERTS_NEED_GC(p, 2)) { @@ -1349,8 +1346,7 @@ erts_gc_mixed_minus(Process* p, Eterm* reg, Uint live) switch ((arg2 & _TAG_IMMED1_MASK) >> _TAG_PRIMARY_SIZE) { case (_TAG_IMMED1_SMALL >> _TAG_PRIMARY_SIZE): ires = signed_val(arg1) - signed_val(arg2); - ASSERT(MY_IS_SSMALL(ires) == IS_SSMALL(ires)); - if (MY_IS_SSMALL(ires)) { + if (IS_SSMALL(ires)) { return make_small(ires); } else { if (ERTS_NEED_GC(p, 2)) { diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index 756c7dce05..4cafa499a9 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -171,6 +171,16 @@ static void *my_alloc(MyAllocator *my, Uint size) #define ALPHABET_SIZE 256 +typedef struct _findall_data { + Uint pos; + Uint len; +#ifdef HARDDEBUG + Uint id; +#endif + Eterm epos; + Eterm elen; +} FindallData; + typedef struct _ac_node { #ifdef HARDDEBUG Uint32 id; /* To identify h pointer targets when @@ -208,6 +218,103 @@ typedef struct _bm_data { Sint badshift[ALPHABET_SIZE]; } BMData; +typedef struct _ac_find_all_state { + ACNode *q; + Uint pos; + Uint len; + Uint m; + Uint allocated; + FindallData *out; +} ACFindAllState; + +typedef struct _ac_find_first_state { + ACNode *q; + Uint pos; + Uint len; + ACNode *candidate; + Uint candidate_start; +} ACFindFirstState; + +typedef struct _bm_find_all_state { + Sint pos; + Sint len; + Uint m; + Uint allocated; + FindallData *out; +} BMFindAllState; + +typedef struct _bm_find_first_state { + Sint pos; + Sint len; +} BMFindFirstState; + +typedef enum _bf_return { + BF_RESTART = -3, + BF_NOT_FOUND, + BF_BADARG, + BF_OK +} BFReturn; + +typedef struct _binary_find_all_context { + ErtsHeapFactory factory; + Eterm term; + Sint head; + Sint tail; + Uint end_pos; + Uint size; + FindallData *data; + union { + ACFindAllState ac; + BMFindAllState bm; + } d; +} BinaryFindAllContext; + +typedef struct _binary_find_first_context { + Uint pos; + Uint len; + union { + ACFindFirstState ac; + BMFindFirstState bm; + } d; +} BinaryFindFirstContext; + +typedef struct _binary_find_context BinaryFindContext; + +typedef struct _binary_find_search { + void (*init) (BinaryFindContext *); + BFReturn (*find) (BinaryFindContext *, byte *); + void (*done) (BinaryFindContext *); +} BinaryFindSearch; + +typedef Eterm (*BinaryFindResult)(Process *, Eterm, BinaryFindContext **); + +typedef enum _binary_find_state { + BFSearch, + BFResult, + BFDone +} BinaryFindState; + +struct _binary_find_context { + Eterm pat_type; + Eterm pat_term; + Binary *pat_bin; + Uint flags; + Uint hsstart; + Uint hsend; + int loop_factor; + int exported; + Uint reds; + BinaryFindState state; + Eterm trap_term; + BinaryFindSearch *search; + BinaryFindResult not_found; + BinaryFindResult found; + union { + BinaryFindAllContext fa; + BinaryFindFirstContext ff; + } u; +}; + #ifdef HARDDEBUG static void dump_bm_data(BMData *bm); static void dump_ac_trie(ACTrie *act); @@ -229,13 +336,6 @@ static void dump_ac_node(ACNode *node, int indent, int ch); MYALIGN(sizeof(ACTrie))) /* Structure */ -#ifndef MAX -#define MAX(A,B) (((A) > (B)) ? (A) : (B)) -#endif - -#ifndef MIN -#define MIN(A,B) (((A) > (B)) ? (B) : (A)) -#endif /* * Callback for the magic binary */ @@ -421,32 +521,25 @@ static void ac_compute_failure_functions(ACTrie *act, ACNode **qbuff) * Basic AC finds the first end before the first start... * */ -typedef struct { - ACNode *q; - Uint pos; - Uint len; - ACNode *candidate; - Uint candidate_start; -} ACFindFirstState; - - -static void ac_init_find_first_match(ACFindFirstState *state, ACTrie *act, Sint startpos, Uint len) +static void ac_init_find_first_match(BinaryFindContext *ctx) { + ACFindFirstState *state = &(ctx->u.ff.d.ac); + ACTrie *act = ERTS_MAGIC_BIN_DATA(ctx->pat_bin); state->q = act->root; - state->pos = startpos; - state->len = len; + state->pos = ctx->hsstart; + state->len = ctx->hsend; state->candidate = NULL; state->candidate_start = 0; } -#define AC_OK 0 -#define AC_NOT_FOUND -1 -#define AC_RESTART -2 #define AC_LOOP_FACTOR 10 -static int ac_find_first_match(ACFindFirstState *state, byte *haystack, - Uint *mpos, Uint *mlen, Uint *reductions) +static BFReturn ac_find_first_match(BinaryFindContext *ctx, byte *haystack) { + ACFindFirstState *state = &(ctx->u.ff.d.ac); + Uint *mpos = &(ctx->u.ff.pos); + Uint *mlen = &(ctx->u.ff.len); + Uint *reductions = &(ctx->reds); ACNode *q = state->q; Uint i = state->pos; ACNode *candidate = state->candidate, *r; @@ -462,7 +555,7 @@ static int ac_find_first_match(ACFindFirstState *state, byte *haystack, state->len = len; state->candidate = candidate; state->candidate_start = candidate_start; - return AC_RESTART; + return BF_RESTART; } while (q->g[haystack[i]] == NULL && q->h != q) { @@ -492,68 +585,33 @@ static int ac_find_first_match(ACFindFirstState *state, byte *haystack, } *reductions = reds; if (!candidate) { - return AC_NOT_FOUND; + return BF_NOT_FOUND; } #ifdef HARDDEBUG dump_ac_node(candidate,0,'?'); #endif *mpos = candidate_start; *mlen = candidate->d; - return AC_OK; + return BF_OK; } -typedef struct _findall_data { - Uint pos; - Uint len; -#ifdef HARDDEBUG - Uint id; -#endif - Eterm epos; - Eterm elen; -} FindallData; - -typedef struct { - ACNode *q; - Uint pos; - Uint len; - Uint m; - Uint allocated; - FindallData *out; -} ACFindAllState; - -static void ac_init_find_all(ACFindAllState *state, ACTrie *act, Sint startpos, Uint len) +static void ac_init_find_all(BinaryFindContext *ctx) { + ACFindAllState *state = &(ctx->u.fa.d.ac); + ACTrie *act = ERTS_MAGIC_BIN_DATA(ctx->pat_bin); state->q = act->root; - state->pos = startpos; - state->len = len; + state->pos = ctx->hsstart; + state->len = ctx->hsend; state->m = 0; state->allocated = 0; state->out = NULL; } -static void ac_restore_find_all(ACFindAllState *state, - const ACFindAllState *src) -{ - memcpy(state, src, sizeof(ACFindAllState)); - if (state->allocated > 0) { - state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * (state->allocated)); - memcpy(state->out, src+1, sizeof(FindallData)*state->m); - } else { - state->out = NULL; - } -} - -static void ac_serialize_find_all(const ACFindAllState *state, - ACFindAllState *dst) -{ - memcpy(dst, state, sizeof(ACFindAllState)); - memcpy(dst+1, state->out, sizeof(FindallData)*state->m); -} - -static void ac_clean_find_all(ACFindAllState *state) +static void ac_clean_find_all(BinaryFindContext *ctx) { + ACFindAllState *state = &(ctx->u.fa.d.ac); if (state->out != NULL) { - erts_free(ERTS_ALC_T_TMP, state->out); + erts_free(ERTS_ALC_T_BINARY_FIND, state->out); } #ifdef HARDDEBUG state->out = NULL; @@ -565,9 +623,10 @@ static void ac_clean_find_all(ACFindAllState *state) * Differs to the find_first function in that it stores all matches and the values * arte returned only in the state. */ -static int ac_find_all_non_overlapping(ACFindAllState *state, byte *haystack, - Uint *reductions) +static BFReturn ac_find_all_non_overlapping(BinaryFindContext *ctx, byte *haystack) { + ACFindAllState *state = &(ctx->u.fa.d.ac); + Uint *reductions = &(ctx->reds); ACNode *q = state->q; Uint i = state->pos; Uint rstart; @@ -578,7 +637,6 @@ static int ac_find_all_non_overlapping(ACFindAllState *state, byte *haystack, FindallData *out = state->out; register Uint reds = *reductions; - while (i < len) { if (--reds == 0) { state->q = q; @@ -587,7 +645,7 @@ static int ac_find_all_non_overlapping(ACFindAllState *state, byte *haystack, state->m = m; state->allocated = allocated; state->out = out; - return AC_RESTART; + return BF_RESTART; } while (q->g[haystack[i]] == NULL && q->h != q) { q = q->h; @@ -625,11 +683,11 @@ static int ac_find_all_non_overlapping(ACFindAllState *state, byte *haystack, if (m >= allocated) { if (!allocated) { allocated = 10; - out = erts_alloc(ERTS_ALC_T_TMP, + out = erts_alloc(ERTS_ALC_T_BINARY_FIND, sizeof(FindallData) * allocated); } else { allocated *= 2; - out = erts_realloc(ERTS_ALC_T_TMP, out, + out = erts_realloc(ERTS_ALC_T_BINARY_FIND, out, sizeof(FindallData) * allocated); } @@ -656,7 +714,7 @@ static int ac_find_all_non_overlapping(ACFindAllState *state, byte *haystack, *reductions = reds; state->m = m; state->out = out; - return (m == 0) ? AC_NOT_FOUND : AC_OK; + return (m == 0) ? BF_NOT_FOUND : BF_OK; } /* @@ -743,27 +801,22 @@ static void compute_goodshifts(BMData *bmd) erts_free(ERTS_ALC_T_TMP, suffixes); } -typedef struct { - Sint pos; - Sint len; -} BMFindFirstState; - -#define BM_OK 0 /* used only for find_all */ -#define BM_NOT_FOUND -1 -#define BM_RESTART -2 #define BM_LOOP_FACTOR 10 /* Should we have a higher value? */ -static void bm_init_find_first_match(BMFindFirstState *state, Sint startpos, - Uint len) +static void bm_init_find_first_match(BinaryFindContext *ctx) { - state->pos = startpos; - state->len = (Sint) len; + BMFindFirstState *state = &(ctx->u.ff.d.bm); + state->pos = ctx->hsstart; + state->len = ctx->hsend; } - -static Sint bm_find_first_match(BMFindFirstState *state, BMData *bmd, - byte *haystack, Uint *reductions) +static BFReturn bm_find_first_match(BinaryFindContext *ctx, byte *haystack) { + BMFindFirstState *state = &(ctx->u.ff.d.bm); + BMData *bmd = ERTS_MAGIC_BIN_DATA(ctx->pat_bin); + Uint *mpos = &(ctx->u.ff.pos); + Uint *mlen = &(ctx->u.ff.len); + Uint *reductions = &(ctx->reds); Sint blen = bmd->len; Sint len = state->len; Sint *gs = bmd->goodshift; @@ -776,61 +829,37 @@ static Sint bm_find_first_match(BMFindFirstState *state, BMData *bmd, while (j <= len - blen) { if (--reds == 0) { state->pos = j; - return BM_RESTART; + return BF_RESTART; } for (i = blen - 1; i >= 0 && needle[i] == haystack[i + j]; --i) ; if (i < 0) { /* found */ *reductions = reds; - return j; + *mpos = (Uint) j; + *mlen = (Uint) blen; + return BF_OK; } j += MAX(gs[i],bs[haystack[i+j]] - blen + 1 + i); } *reductions = reds; - return BM_NOT_FOUND; + return BF_NOT_FOUND; } -typedef struct { - Sint pos; - Sint len; - Uint m; - Uint allocated; - FindallData *out; -} BMFindAllState; - -static void bm_init_find_all(BMFindAllState *state, Sint startpos, Uint len) +static void bm_init_find_all(BinaryFindContext *ctx) { - state->pos = startpos; - state->len = (Sint) len; + BMFindAllState *state = &(ctx->u.fa.d.bm); + state->pos = ctx->hsstart; + state->len = ctx->hsend; state->m = 0; state->allocated = 0; state->out = NULL; } -static void bm_restore_find_all(BMFindAllState *state, - const BMFindAllState *src) -{ - memcpy(state, src, sizeof(BMFindAllState)); - if (state->allocated > 0) { - state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * - (state->allocated)); - memcpy(state->out, src+1, sizeof(FindallData)*state->m); - } else { - state->out = NULL; - } -} - -static void bm_serialize_find_all(const BMFindAllState *state, - BMFindAllState *dst) -{ - memcpy(dst, state, sizeof(BMFindAllState)); - memcpy(dst+1, state->out, sizeof(FindallData)*state->m); -} - -static void bm_clean_find_all(BMFindAllState *state) +static void bm_clean_find_all(BinaryFindContext *ctx) { + BMFindAllState *state = &(ctx->u.fa.d.bm); if (state->out != NULL) { - erts_free(ERTS_ALC_T_TMP, state->out); + erts_free(ERTS_ALC_T_BINARY_FIND, state->out); } #ifdef HARDDEBUG state->out = NULL; @@ -842,10 +871,11 @@ static void bm_clean_find_all(BMFindAllState *state) * Differs to the find_first function in that it stores all matches and the * values are returned only in the state. */ -static Sint bm_find_all_non_overlapping(BMFindAllState *state, - BMData *bmd, byte *haystack, - Uint *reductions) +static BFReturn bm_find_all_non_overlapping(BinaryFindContext *ctx, byte *haystack) { + BMFindAllState *state = &(ctx->u.fa.d.bm); + BMData *bmd = ERTS_MAGIC_BIN_DATA(ctx->pat_bin); + Uint *reductions = &(ctx->reds); Sint blen = bmd->len; Sint len = state->len; Sint *gs = bmd->goodshift; @@ -864,7 +894,7 @@ static Sint bm_find_all_non_overlapping(BMFindAllState *state, state->m = m; state->allocated = allocated; state->out = out; - return BM_RESTART; + return BF_RESTART; } for (i = blen - 1; i >= 0 && needle[i] == haystack[i + j]; --i) ; @@ -872,10 +902,11 @@ static Sint bm_find_all_non_overlapping(BMFindAllState *state, if (m >= allocated) { if (!allocated) { allocated = 10; - out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * allocated); + out = erts_alloc(ERTS_ALC_T_BINARY_FIND, + sizeof(FindallData) * allocated); } else { allocated *= 2; - out = erts_realloc(ERTS_ALC_T_TMP, out, + out = erts_realloc(ERTS_ALC_T_BINARY_FIND, out, sizeof(FindallData) * allocated); } } @@ -890,7 +921,7 @@ static Sint bm_find_all_non_overlapping(BMFindAllState *state, state->m = m; state->out = out; *reductions = reds; - return (m == 0) ? BM_NOT_FOUND : BM_OK; + return (m == 0) ? BF_NOT_FOUND : BF_OK; } /* @@ -1016,51 +1047,160 @@ BIF_RETTYPE binary_compile_pattern_1(BIF_ALIST_1) BIF_RET(ret); } -#define DO_BIN_MATCH_OK 0 -#define DO_BIN_MATCH_BADARG -1 -#define DO_BIN_MATCH_RESTART -2 +#define BF_FLAG_GLOBAL 0x01 +#define BF_FLAG_SPLIT_TRIM 0x02 +#define BF_FLAG_SPLIT_TRIM_ALL 0x04 -#define BINARY_FIND_ALL 0x01 -#define BINARY_SPLIT_TRIM 0x02 -#define BINARY_SPLIT_TRIM_ALL 0x04 +static void bf_context_init(BinaryFindContext *ctx, BinaryFindResult not_found, + BinaryFindResult single, BinaryFindResult global, + Binary *pat_bin); +static BinaryFindContext *bf_context_export(Process *p, BinaryFindContext *src); +static int bf_context_destructor(Binary *ctx_bin); +#ifdef HARDDEBUG +static void bf_context_dump(BinaryFindContext *ctx); +#endif -typedef struct BinaryFindState { - Eterm type; - Uint flags; - Uint hsstart; - Uint hsend; - Eterm (*not_found_result) (Process *, Eterm, struct BinaryFindState *); - Eterm (*single_result) (Process *, Eterm, struct BinaryFindState *, Sint, Sint); - Eterm (*global_result) (Process *, Eterm, struct BinaryFindState *, FindallData *, Uint); -} BinaryFindState; +static BinaryFindSearch bf_search_ac_global = { + ac_init_find_all, + ac_find_all_non_overlapping, + ac_clean_find_all +}; + +static BinaryFindSearch bf_search_ac_single = { + ac_init_find_first_match, + ac_find_first_match, + NULL +}; + +static BinaryFindSearch bf_search_bm_global = { + bm_init_find_all, + bm_find_all_non_overlapping, + bm_clean_find_all +}; + +static BinaryFindSearch bf_search_bm_single = { + bm_init_find_first_match, + bm_find_first_match, + NULL +}; + +static void bf_context_init(BinaryFindContext *ctx, BinaryFindResult not_found, + BinaryFindResult single, BinaryFindResult global, + Binary *pat_bin) +{ + ctx->exported = 0; + ctx->state = BFSearch; + ctx->not_found = not_found; + if (ctx->flags & BF_FLAG_GLOBAL) { + ctx->found = global; + if (ctx->pat_type == am_bm) { + ctx->search = &bf_search_bm_global; + ctx->loop_factor = BM_LOOP_FACTOR; + } else if (ctx->pat_type == am_ac) { + ctx->search = &bf_search_ac_global; + ctx->loop_factor = AC_LOOP_FACTOR; + } + } else { + ctx->found = single; + if (ctx->pat_type == am_bm) { + ctx->search = &bf_search_bm_single; + ctx->loop_factor = BM_LOOP_FACTOR; + } else if (ctx->pat_type == am_ac) { + ctx->search = &bf_search_ac_single; + ctx->loop_factor = AC_LOOP_FACTOR; + } + } + ctx->trap_term = THE_NON_VALUE; + ctx->pat_bin = pat_bin; + ctx->search->init(ctx); +} -typedef struct BinaryFindState_bignum { - Eterm bignum_hdr; - BinaryFindState bfs; - union { - BMFindFirstState bmffs; - BMFindAllState bmfas; - ACFindFirstState acffs; - ACFindAllState acfas; - } data; -} BinaryFindState_bignum; - -#define SIZEOF_BINARY_FIND_STATE(S) \ - (sizeof(BinaryFindState)+sizeof(S)) - -#define SIZEOF_BINARY_FIND_ALL_STATE(S) \ - (sizeof(BinaryFindState)+sizeof(S)+(sizeof(FindallData)*(S).m)) - -static Eterm do_match_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs); -static Eterm do_match_single_result(Process *p, Eterm subject, BinaryFindState *bfs, - Sint pos, Sint len); -static Eterm do_match_global_result(Process *p, Eterm subject, BinaryFindState *bfs, - FindallData *fad, Uint fad_sz); -static Eterm do_split_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs); -static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindState *bfs, - Sint pos, Sint len); -static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindState *bfs, - FindallData *fad, Uint fad_sz); +static BinaryFindContext *bf_context_export(Process *p, BinaryFindContext *src) +{ + Binary *ctx_bin; + BinaryFindContext *ctx; + Eterm *hp; + + ASSERT(src->exported == 0); + ctx_bin = erts_create_magic_binary(sizeof(BinaryFindContext), + bf_context_destructor); + ctx = ERTS_MAGIC_BIN_DATA(ctx_bin); + sys_memcpy(ctx, src, sizeof(BinaryFindContext)); + if (ctx->pat_bin != NULL && ctx->pat_term == THE_NON_VALUE) { + hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE * 2); + ctx->pat_term = erts_mk_magic_ref(&hp, &MSO(p), ctx->pat_bin); + } else { + hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); + } + ctx->trap_term = erts_mk_magic_ref(&hp, &MSO(p), ctx_bin); + ctx->exported = 1; + return ctx; +} + +static int bf_context_destructor(Binary *ctx_bin) +{ + BinaryFindContext *ctx; + + ctx = ERTS_MAGIC_BIN_DATA(ctx_bin); + if (ctx->state != BFDone) { + if (ctx->search->done != NULL) { + ctx->search->done(ctx); + } + ctx->state = BFDone; + } + return 1; +} + +#ifdef HARDDEBUG +static void bf_context_dump(BinaryFindContext *ctx) +{ + if (ctx->pat_type == am_bm) { + BMData *bm; + bm = ERTS_MAGIC_BIN_DATA(ctx->pat_bin); + dump_bm_data(bm); + } else { + ACTrie *act; + act = ERTS_MAGIC_BIN_DATA(ctx->pat_bin); + dump_ac_trie(act); + } +} +#endif + +static Eterm do_match_not_found_result(Process *p, Eterm subject, BinaryFindContext **ctxp); +static Eterm do_match_single_result(Process *p, Eterm subject, BinaryFindContext **ctxp); +static Eterm do_match_global_result(Process *p, Eterm subject, BinaryFindContext **ctxp); +static Eterm do_split_not_found_result(Process *p, Eterm subject, BinaryFindContext **ctxp); +static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindContext **ctxp); +static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindContext **ctxp); + +static BFReturn maybe_binary_match_compile(BinaryFindContext *ctx, Eterm arg, Binary **pat_bin) +{ + Eterm *tp; + ctx->pat_term = THE_NON_VALUE; + if (is_tuple(arg)) { + tp = tuple_val(arg); + if (arityval(*tp) != 2 || is_not_atom(tp[1])) { + return BF_BADARG; + } + if (((tp[1] != am_bm) && (tp[1] != am_ac)) || + !is_internal_magic_ref(tp[2])) { + return BF_BADARG; + } + *pat_bin = erts_magic_ref2bin(tp[2]); + if ((tp[1] == am_bm && + ERTS_MAGIC_BIN_DESTRUCTOR(*pat_bin) != cleanup_my_data_bm) || + (tp[1] == am_ac && + ERTS_MAGIC_BIN_DESTRUCTOR(*pat_bin) != cleanup_my_data_ac)) { + *pat_bin = NULL; + return BF_BADARG; + } + ctx->pat_type = tp[1]; + ctx->pat_term = tp[2]; + } else if (do_binary_match_compile(arg, &(ctx->pat_type), pat_bin) != 0) { + return BF_BADARG; + } + return BF_OK; +} static int parse_match_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp) { @@ -1141,17 +1281,17 @@ static int parse_split_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp, Uin Uint orig_size; if (is_atom(t)) { if (t == am_global) { - *optp |= BINARY_FIND_ALL; + *optp |= BF_FLAG_GLOBAL; l = CDR(list_val(l)); continue; } if (t == am_trim) { - *optp |= BINARY_SPLIT_TRIM; + *optp |= BF_FLAG_SPLIT_TRIM; l = CDR(list_val(l)); continue; } if (t == am_trim_all) { - *optp |= BINARY_SPLIT_TRIM_ALL; + *optp |= BF_FLAG_SPLIT_TRIM_ALL; l = CDR(list_val(l)); continue; } @@ -1204,266 +1344,160 @@ static int parse_split_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp, Uin } } -static int do_binary_find(Process *p, Eterm subject, BinaryFindState *bfs, Binary *bin, - Eterm state_term, Eterm *res_term) +static BFReturn do_binary_find(Process *p, Eterm subject, BinaryFindContext **ctxp, + Binary *pat_bin, Binary *ctx_bin, Eterm *res_term) { - byte *bytes; - Uint bitoffs, bitsize; - byte *temp_alloc = NULL; - BinaryFindState_bignum *state_ptr = NULL; + BinaryFindContext *ctx; + int is_first_call; + Uint initial_reds; + BFReturn runres; - ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize); - if (bitsize != 0) { - goto badarg; - } - if (bitoffs != 0) { - bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc); - } - if (state_term != NIL) { - state_ptr = (BinaryFindState_bignum *)(big_val(state_term)); - bfs = &(state_ptr->bfs); + if (ctx_bin == NULL) { + is_first_call = 1; + ctx = *ctxp; + } else { + is_first_call = 0; + ctx = ERTS_MAGIC_BIN_DATA(ctx_bin); + ctx->pat_bin = pat_bin; + *ctxp = ctx; } - if (bfs->flags & BINARY_FIND_ALL) { - if (bfs->type == am_bm) { - BMData *bm; - Sint pos; - BMFindAllState state; - Uint reds = get_reds(p, BM_LOOP_FACTOR); - Uint save_reds = reds; + initial_reds = ctx->reds = get_reds(p, ctx->loop_factor); - bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin); -#ifdef HARDDEBUG - dump_bm_data(bm); -#endif - if (state_term == NIL) { - bm_init_find_all(&state, bfs->hsstart, bfs->hsend); - } else { - bm_restore_find_all(&state, &(state_ptr->data.bmfas)); - } + switch (ctx->state) { + case BFSearch: { + byte *bytes; + Uint bitoffs, bitsize; + byte *temp_alloc = NULL; - pos = bm_find_all_non_overlapping(&state, bm, bytes, &reds); - if (pos == BM_NOT_FOUND) { - *res_term = bfs->not_found_result(p, subject, bfs); - } else if (pos == BM_RESTART) { - int x = - (SIZEOF_BINARY_FIND_ALL_STATE(state) / sizeof(Eterm)) + - !!(SIZEOF_BINARY_FIND_ALL_STATE(state) % sizeof(Eterm)); -#ifdef HARDDEBUG - erts_printf("Trap bm!\n"); -#endif - state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1); - state_ptr->bignum_hdr = make_pos_bignum_header(x); - memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState)); - bm_serialize_find_all(&state, &state_ptr->data.bmfas); - *res_term = make_big(&state_ptr->bignum_hdr); - erts_free_aligned_binary_bytes(temp_alloc); - bm_clean_find_all(&state); - return DO_BIN_MATCH_RESTART; - } else { - *res_term = bfs->global_result(p, subject, bfs, state.out, state.m); - } - erts_free_aligned_binary_bytes(temp_alloc); - bm_clean_find_all(&state); - BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR); - return DO_BIN_MATCH_OK; - } else if (bfs->type == am_ac) { - ACTrie *act; - int acr; - ACFindAllState state; - Uint reds = get_reds(p, AC_LOOP_FACTOR); - Uint save_reds = reds; - - act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin); + ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize); + if (bitsize != 0) { + goto badarg; + } + if (bitoffs != 0) { + bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc); + } #ifdef HARDDEBUG - dump_ac_trie(act); + bf_context_dump(ctx); #endif - if (state_term == NIL) { - ac_init_find_all(&state, act, bfs->hsstart, bfs->hsend); - } else { - ac_restore_find_all(&state, &(state_ptr->data.acfas)); - } - acr = ac_find_all_non_overlapping(&state, bytes, &reds); - if (acr == AC_NOT_FOUND) { - *res_term = bfs->not_found_result(p, subject, bfs); - } else if (acr == AC_RESTART) { - int x = - (SIZEOF_BINARY_FIND_ALL_STATE(state) / sizeof(Eterm)) + - !!(SIZEOF_BINARY_FIND_ALL_STATE(state) % sizeof(Eterm)); + runres = ctx->search->find(ctx, bytes); + if (runres == BF_NOT_FOUND) { + *res_term = ctx->not_found(p, subject, &ctx); + *ctxp = ctx; + } else if (runres == BF_RESTART) { #ifdef HARDDEBUG + if (ctx->pat_type == am_ac) { erts_printf("Trap ac!\n"); -#endif - state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1); - state_ptr->bignum_hdr = make_pos_bignum_header(x); - memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState)); - ac_serialize_find_all(&state, &state_ptr->data.acfas); - *res_term = make_big(&state_ptr->bignum_hdr); - erts_free_aligned_binary_bytes(temp_alloc); - ac_clean_find_all(&state); - return DO_BIN_MATCH_RESTART; - } else { - *res_term = bfs->global_result(p, subject, bfs, state.out, state.m); - } - erts_free_aligned_binary_bytes(temp_alloc); - ac_clean_find_all(&state); - BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR); - return DO_BIN_MATCH_OK; - } - } else { - if (bfs->type == am_bm) { - BMData *bm; - Sint pos; - BMFindFirstState state; - Uint reds = get_reds(p, BM_LOOP_FACTOR); - Uint save_reds = reds; - - bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin); -#ifdef HARDDEBUG - dump_bm_data(bm); -#endif - if (state_term == NIL) { - bm_init_find_first_match(&state, bfs->hsstart, bfs->hsend); } else { - memcpy(&state, &state_ptr->data.bmffs, sizeof(BMFindFirstState)); - } - -#ifdef HARDDEBUG - erts_printf("(bm) state->pos = %ld, state->len = %lu\n",state.pos, - state.len); -#endif - pos = bm_find_first_match(&state, bm, bytes, &reds); - if (pos == BM_NOT_FOUND) { - *res_term = bfs->not_found_result(p, subject, bfs); - } else if (pos == BM_RESTART) { - int x = - (SIZEOF_BINARY_FIND_STATE(state) / sizeof(Eterm)) + - !!(SIZEOF_BINARY_FIND_STATE(state) % sizeof(Eterm)); -#ifdef HARDDEBUG erts_printf("Trap bm!\n"); + } #endif - state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1); - state_ptr->bignum_hdr = make_pos_bignum_header(x); - memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState)); - memcpy(&state_ptr->data.acffs, &state, sizeof(BMFindFirstState)); - *res_term = make_big(&state_ptr->bignum_hdr); - erts_free_aligned_binary_bytes(temp_alloc); - return DO_BIN_MATCH_RESTART; - } else { - *res_term = bfs->single_result(p, subject, bfs, pos, bm->len); + if (is_first_call) { + ctx = bf_context_export(p, ctx); + *ctxp = ctx; + erts_set_gc_state(p, 0); } erts_free_aligned_binary_bytes(temp_alloc); - BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR); - return DO_BIN_MATCH_OK; - } else if (bfs->type == am_ac) { - ACTrie *act; - Uint pos, rlen; - int acr; - ACFindFirstState state; - Uint reds = get_reds(p, AC_LOOP_FACTOR); - Uint save_reds = reds; - - act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin); -#ifdef HARDDEBUG - dump_ac_trie(act); -#endif - if (state_term == NIL) { - ac_init_find_first_match(&state, act, bfs->hsstart, bfs->hsend); - } else { - memcpy(&state, &state_ptr->data.acffs, sizeof(ACFindFirstState)); + *res_term = THE_NON_VALUE; + BUMP_ALL_REDS(p); + return BF_RESTART; + } else { + *res_term = ctx->found(p, subject, &ctx); + *ctxp = ctx; + } + erts_free_aligned_binary_bytes(temp_alloc); + if (*res_term == THE_NON_VALUE) { + if (is_first_call) { + erts_set_gc_state(p, 0); } - acr = ac_find_first_match(&state, bytes, &pos, &rlen, &reds); - if (acr == AC_NOT_FOUND) { - *res_term = bfs->not_found_result(p, subject, bfs); - } else if (acr == AC_RESTART) { - int x = - (SIZEOF_BINARY_FIND_STATE(state) / sizeof(Eterm)) + - !!(SIZEOF_BINARY_FIND_STATE(state) % sizeof(Eterm)); -#ifdef HARDDEBUG - erts_printf("Trap ac!\n"); -#endif - state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1); - state_ptr->bignum_hdr = make_pos_bignum_header(x); - memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState)); - memcpy(&state_ptr->data.acffs, &state, sizeof(ACFindFirstState)); - *res_term = make_big(&state_ptr->bignum_hdr); - erts_free_aligned_binary_bytes(temp_alloc); - return DO_BIN_MATCH_RESTART; - } else { - *res_term = bfs->single_result(p, subject, bfs, pos, rlen); + BUMP_ALL_REDS(p); + return BF_RESTART; + } + if (ctx->search->done != NULL) { + ctx->search->done(ctx); + } + ctx->state = BFDone; + if (!is_first_call) { + erts_set_gc_state(p, 1); + } + BUMP_REDS(p, (initial_reds - ctx->reds) / ctx->loop_factor); + return BF_OK; + } + case BFResult: { + *res_term = ctx->found(p, subject, &ctx); + *ctxp = ctx; + if (*res_term == THE_NON_VALUE) { + if (is_first_call) { + erts_set_gc_state(p, 0); } - erts_free_aligned_binary_bytes(temp_alloc); - BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR); - return DO_BIN_MATCH_OK; + BUMP_ALL_REDS(p); + return BF_RESTART; } + if (ctx->search->done != NULL) { + ctx->search->done(ctx); + } + ctx->state = BFDone; + if (!is_first_call) { + erts_set_gc_state(p, 1); + } + BUMP_REDS(p, (initial_reds - ctx->reds) / ctx->loop_factor); + return BF_OK; } - badarg: - return DO_BIN_MATCH_BADARG; + default: + ASSERT(!"Unknown state in do_binary_find"); + } + +badarg: + if (!is_first_call) { + if (ctx->search->done != NULL) { + ctx->search->done(ctx); + } + ctx->state = BFDone; + erts_set_gc_state(p, 1); + } + return BF_BADARG; } static BIF_RETTYPE binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, Uint flags) { - BinaryFindState bfs; - Eterm *tp; - Binary *bin; - Eterm bin_term = NIL; + BinaryFindContext c_buff; + BinaryFindContext *ctx = &c_buff; + Binary *pat_bin; int runres; Eterm result; - if (is_not_binary(arg1)) { + if (is_not_binary(arg1) || binary_bitsize(arg1) != 0) { goto badarg; } - bfs.flags = flags; - if (parse_match_opts_list(arg3, arg1, &(bfs.hsstart), &(bfs.hsend))) { + ctx->flags = flags; + if (parse_match_opts_list(arg3, arg1, &(ctx->hsstart), &(ctx->hsend))) { goto badarg; } - if (bfs.hsend == 0) { - BIF_RET(do_match_not_found_result(p, arg1, &bfs)); + if (ctx->hsend == 0) { + result = do_match_not_found_result(p, arg1, &ctx); + BIF_RET(result); } - if (is_tuple(arg2)) { - tp = tuple_val(arg2); - if (arityval(*tp) != 2 || is_not_atom(tp[1])) { - goto badarg; - } - if (((tp[1] != am_bm) && (tp[1] != am_ac)) || - !is_internal_magic_ref(tp[2])) { - goto badarg; - } - bfs.type = tp[1]; - bin = erts_magic_ref2bin(tp[2]); - if (bfs.type == am_bm && - ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) { - goto badarg; - } - if (bfs.type == am_ac && - ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) { - goto badarg; - } - bin_term = tp[2]; - } else if (do_binary_match_compile(arg2, &(bfs.type), &bin)) { + if (maybe_binary_match_compile(ctx, arg2, &pat_bin) != BF_OK) { goto badarg; } - bfs.not_found_result = &do_match_not_found_result; - bfs.single_result = &do_match_single_result; - bfs.global_result = &do_match_global_result; - runres = do_binary_find(p, arg1, &bfs, bin, NIL, &result); - if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) { - Eterm *hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); - bin_term = erts_mk_magic_ref(&hp, &MSO(p), bin); - } else if (bin_term == NIL) { - erts_bin_free(bin); + bf_context_init(ctx, do_match_not_found_result, do_match_single_result, + do_match_global_result, pat_bin); + runres = do_binary_find(p, arg1, &ctx, pat_bin, NULL, &result); + if (runres == BF_OK && ctx->pat_term == THE_NON_VALUE) { + erts_bin_free(pat_bin); } switch (runres) { - case DO_BIN_MATCH_OK: + case BF_OK: BIF_RET(result); - case DO_BIN_MATCH_RESTART: - BUMP_ALL_REDS(p); - BIF_TRAP3(&binary_find_trap_export, p, arg1, result, bin_term); + case BF_RESTART: + ASSERT(result == THE_NON_VALUE && ctx->trap_term != result && ctx->pat_term != result); + BIF_TRAP3(&binary_find_trap_export, p, arg1, ctx->trap_term, ctx->pat_term); default: goto badarg; } - badarg: - BIF_ERROR(p,BADARG); +badarg: + BIF_ERROR(p, BADARG); } BIF_RETTYPE binary_match_2(BIF_ALIST_2) @@ -1478,76 +1512,52 @@ BIF_RETTYPE binary_match_3(BIF_ALIST_3) BIF_RETTYPE binary_matches_2(BIF_ALIST_2) { - return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, THE_NON_VALUE, BINARY_FIND_ALL); + return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, THE_NON_VALUE, BF_FLAG_GLOBAL); } BIF_RETTYPE binary_matches_3(BIF_ALIST_3) { - return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, BINARY_FIND_ALL); + return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, BF_FLAG_GLOBAL); } static BIF_RETTYPE binary_split(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) { - BinaryFindState bfs; - Eterm *tp; - Binary *bin; - Eterm bin_term = NIL; + BinaryFindContext c_buff; + BinaryFindContext *ctx = &c_buff; + Binary *pat_bin; int runres; Eterm result; - if (is_not_binary(arg1)) { + if (is_not_binary(arg1) || binary_bitsize(arg1) != 0) { goto badarg; } - if (parse_split_opts_list(arg3, arg1, &(bfs.hsstart), &(bfs.hsend), &(bfs.flags))) { + if (parse_split_opts_list(arg3, arg1, &(ctx->hsstart), &(ctx->hsend), &(ctx->flags))) { goto badarg; } - if (bfs.hsend == 0) { - result = do_split_not_found_result(p, arg1, &bfs); + if (ctx->hsend == 0) { + result = do_split_not_found_result(p, arg1, &ctx); BIF_RET(result); } - if (is_tuple(arg2)) { - tp = tuple_val(arg2); - if (arityval(*tp) != 2 || is_not_atom(tp[1])) { - goto badarg; - } - if (((tp[1] != am_bm) && (tp[1] != am_ac)) || - !is_internal_magic_ref(tp[2])) { - goto badarg; - } - bfs.type = tp[1]; - bin = erts_magic_ref2bin(tp[2]); - if (bfs.type == am_bm && - ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) { - goto badarg; - } - if (bfs.type == am_ac && - ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) { - goto badarg; - } - bin_term = tp[2]; - } else if (do_binary_match_compile(arg2, &(bfs.type), &bin)) { + if (maybe_binary_match_compile(ctx, arg2, &pat_bin) != BF_OK) { goto badarg; } - bfs.not_found_result = &do_split_not_found_result; - bfs.single_result = &do_split_single_result; - bfs.global_result = &do_split_global_result; - runres = do_binary_find(p, arg1, &bfs, bin, NIL, &result); - if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) { - Eterm *hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); - bin_term = erts_mk_magic_ref(&hp, &MSO(p), bin); - } else if (bin_term == NIL) { - erts_bin_free(bin); - } - switch(runres) { - case DO_BIN_MATCH_OK: + bf_context_init(ctx, do_split_not_found_result, do_split_single_result, + do_split_global_result, pat_bin); + runres = do_binary_find(p, arg1, &ctx, pat_bin, NULL, &result); + if (runres == BF_OK && ctx->pat_term == THE_NON_VALUE) { + erts_bin_free(pat_bin); + } + switch (runres) { + case BF_OK: BIF_RET(result); - case DO_BIN_MATCH_RESTART: - BIF_TRAP3(&binary_find_trap_export, p, arg1, result, bin_term); + case BF_RESTART: + ASSERT(result == THE_NON_VALUE && ctx->trap_term != result && ctx->pat_term != result); + BIF_TRAP3(&binary_find_trap_export, p, arg1, ctx->trap_term, ctx->pat_term); default: goto badarg; } - badarg: +badarg: BIF_ERROR(p, BADARG); } @@ -1561,72 +1571,117 @@ BIF_RETTYPE binary_split_3(BIF_ALIST_3) return binary_split(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } -static Eterm do_match_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs) +static Eterm do_match_not_found_result(Process *p, Eterm subject, BinaryFindContext **ctxp) { - if (bfs->flags & BINARY_FIND_ALL) { + if ((*ctxp)->flags & BF_FLAG_GLOBAL) { return NIL; } else { return am_nomatch; } } -static Eterm do_match_single_result(Process *p, Eterm subject, BinaryFindState *bfs, - Sint pos, Sint len) +static Eterm do_match_single_result(Process *p, Eterm subject, BinaryFindContext **ctxp) { + BinaryFindContext *ctx = (*ctxp); + BinaryFindFirstContext *ff = &(ctx->u.ff); Eterm erlen; Eterm *hp; Eterm ret; - erlen = erts_make_integer((Uint)(len), p); - ret = erts_make_integer(pos, p); + erlen = erts_make_integer((Uint)(ff->len), p); + ret = erts_make_integer(ff->pos, p); hp = HAlloc(p, 3); ret = TUPLE2(hp, ret, erlen); return ret; } -static Eterm do_match_global_result(Process *p, Eterm subject, BinaryFindState *bfs, - FindallData *fad, Uint fad_sz) +static Eterm do_match_global_result(Process *p, Eterm subject, BinaryFindContext **ctxp) { - Sint i; + BinaryFindContext *ctx = (*ctxp); + BinaryFindAllContext *fa = &(ctx->u.fa); + FindallData *fad; Eterm tpl; - Eterm *hp; - Eterm ret; + Sint i; + register Uint reds = ctx->reds; - for (i = 0; i < fad_sz; ++i) { - fad[i].epos = erts_make_integer(fad[i].pos, p); - fad[i].elen = erts_make_integer(fad[i].len, p); + if (ctx->state == BFSearch) { + if (ctx->pat_type == am_ac) { + fa->data = fa->d.ac.out; + fa->size = fa->d.ac.m; + } else { + fa->data = fa->d.bm.out; + fa->size = fa->d.bm.m; + } + fa->tail = fa->size - 1; + fa->head = 0; + fa->end_pos = 0; + fa->term = NIL; + if (ctx->exported == 0 && ((fa->size * 2) >= reds)) { + ctx = bf_context_export(p, ctx); + *ctxp = ctx; + fa = &(ctx->u.fa); + } + erts_factory_proc_prealloc_init(&(fa->factory), p, fa->size * (3 + 2)); + ctx->state = BFResult; + } + + fad = fa->data; + + if (fa->end_pos == 0) { + for (i = fa->head; i < fa->size; ++i) { + if (--reds == 0) { + ASSERT(ctx->exported == 1); + fa->head = i; + ctx->reds = reds; + return THE_NON_VALUE; + } + fad[i].epos = erts_make_integer(fad[i].pos, p); + fad[i].elen = erts_make_integer(fad[i].len, p); + } + fa->end_pos = 1; + fa->head = fa->tail; } - hp = HAlloc(p, fad_sz * (3 + 2)); - ret = NIL; - for (i = fad_sz - 1; i >= 0; --i) { - tpl = TUPLE2(hp, fad[i].epos, fad[i].elen); - hp += 3; - ret = CONS(hp, tpl, ret); - hp += 2; + + for (i = fa->head; i >= 0; --i) { + if (--reds == 0) { + ASSERT(ctx->exported == 1); + fa->head = i; + ctx->reds = reds; + return THE_NON_VALUE; + } + tpl = TUPLE2(fa->factory.hp, fad[i].epos, fad[i].elen); + fa->factory.hp += 3; + fa->term = CONS(fa->factory.hp, tpl, fa->term); + fa->factory.hp += 2; } + ctx->reds = reds; + erts_factory_close(&(fa->factory)); - return ret; + return fa->term; } -static Eterm do_split_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs) +static Eterm do_split_not_found_result(Process *p, Eterm subject, BinaryFindContext **ctxp) { + BinaryFindContext *ctx = (*ctxp); Eterm *hp; Eterm ret; - if (bfs->flags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL) + if (ctx->flags & (BF_FLAG_SPLIT_TRIM | BF_FLAG_SPLIT_TRIM_ALL) && binary_size(subject) == 0) { - return NIL; + return NIL; } hp = HAlloc(p, 2); ret = CONS(hp, subject, NIL); - return ret; } -static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindState *bfs, - Sint pos, Sint len) +static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindContext **ctxp) { + BinaryFindContext *ctx = (*ctxp); + BinaryFindFirstContext *ff = &(ctx->u.ff); + Sint pos; + Sint len; size_t orig_size; Eterm orig; Uint offset; @@ -1637,9 +1692,12 @@ static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindState * Eterm *hp; Eterm ret; + pos = ff->pos; + len = ff->len; + orig_size = binary_size(subject); - if ((bfs->flags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL)) && + if ((ctx->flags & (BF_FLAG_SPLIT_TRIM | BF_FLAG_SPLIT_TRIM_ALL)) && (orig_size - pos - len) == 0) { if (pos == 0) { ret = NIL; @@ -1660,7 +1718,7 @@ static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindState * hp += 2; } } else { - if ((bfs->flags & BINARY_SPLIT_TRIM_ALL) && (pos == 0)) { + if ((ctx->flags & BF_FLAG_SPLIT_TRIM_ALL) && (pos == 0)) { hp = HAlloc(p, 1 * (ERL_SUB_BIN_SIZE + 2)); ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size); sb1 = NULL; @@ -1698,39 +1756,60 @@ static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindState * return ret; } -static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindState *bfs, - FindallData *fad, Uint fad_sz) +static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindContext **ctxp) { - size_t orig_size; + BinaryFindContext *ctx = (*ctxp); + BinaryFindAllContext *fa = &(ctx->u.fa); + FindallData *fad; Eterm orig; + size_t orig_size; Uint offset; Uint bit_offset; Uint bit_size; ErlSubBin *sb; + Uint do_trim; Sint i; - Sint tail; - Uint list_size; - Uint end_pos; - Uint do_trim = bfs->flags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL); - Eterm *hp; - Eterm *hendp; - Eterm ret; + register Uint reds = ctx->reds; - tail = fad_sz - 1; - list_size = fad_sz + 1; - orig_size = binary_size(subject); - end_pos = (Uint)(orig_size); + if (ctx->state == BFSearch) { + if (ctx->pat_type == am_ac) { + fa->data = fa->d.ac.out; + fa->size = fa->d.ac.m; + } else { + fa->data = fa->d.bm.out; + fa->size = fa->d.bm.m; + } + fa->tail = fa->size - 1; + fa->head = fa->tail; + orig_size = binary_size(subject); + fa->end_pos = (Uint)(orig_size); + fa->term = NIL; + if (ctx->exported == 0 && ((fa->head + 1) >= reds)) { + ctx = bf_context_export(p, ctx); + *ctxp = ctx; + fa = &(ctx->u.fa); + } + erts_factory_proc_prealloc_init(&(fa->factory), p, (fa->size + 1) * (ERL_SUB_BIN_SIZE + 2)); + ctx->state = BFResult; + } - hp = HAlloc(p, list_size * (ERL_SUB_BIN_SIZE + 2)); - hendp = hp + list_size * (ERL_SUB_BIN_SIZE + 2); ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size); ASSERT(bit_size == 0); + fad = fa->data; + do_trim = ctx->flags & (BF_FLAG_SPLIT_TRIM | BF_FLAG_SPLIT_TRIM_ALL); - ret = NIL; - - for (i = tail; i >= 0; --i) { - sb = (ErlSubBin *)(hp); - sb->size = end_pos - (fad[i].pos + fad[i].len); + for (i = fa->head; i >= 0; --i) { + if (--reds == 0) { + ASSERT(ctx->exported == 1); + fa->head = i; + ctx->reds = reds; + if (!do_trim && (ctx->flags & BF_FLAG_SPLIT_TRIM)) { + ctx->flags &= ~BF_FLAG_SPLIT_TRIM; + } + return THE_NON_VALUE; + } + sb = (ErlSubBin *)(fa->factory.hp); + sb->size = fa->end_pos - (fad[i].pos + fad[i].len); if (!(sb->size == 0 && do_trim)) { sb->thing_word = HEADER_SUB_BIN; sb->offs = offset + fad[i].pos + fad[i].len; @@ -1738,15 +1817,18 @@ static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindState * sb->bitoffs = bit_offset; sb->bitsize = 0; sb->is_writable = 0; - hp += ERL_SUB_BIN_SIZE; - ret = CONS(hp, make_binary(sb), ret); - hp += 2; - do_trim &= ~BINARY_SPLIT_TRIM; + fa->factory.hp += ERL_SUB_BIN_SIZE; + fa->term = CONS(fa->factory.hp, make_binary(sb), fa->term); + fa->factory.hp += 2; + do_trim &= ~BF_FLAG_SPLIT_TRIM; } - end_pos = fad[i].pos; + fa->end_pos = fad[i].pos; } - sb = (ErlSubBin *)(hp); + fa->head = i; + ctx->reds = reds; + + sb = (ErlSubBin *)(fa->factory.hp); sb->size = fad[0].pos; if (!(sb->size == 0 && do_trim)) { sb->thing_word = HEADER_SUB_BIN; @@ -1755,26 +1837,31 @@ static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindState * sb->bitoffs = bit_offset; sb->bitsize = 0; sb->is_writable = 0; - hp += ERL_SUB_BIN_SIZE; - ret = CONS(hp, make_binary(sb), ret); - hp += 2; + fa->factory.hp += ERL_SUB_BIN_SIZE; + fa->term = CONS(fa->factory.hp, make_binary(sb), fa->term); + fa->factory.hp += 2; } - HRelease(p, hendp, hp); - return ret; + erts_factory_close(&(fa->factory)); + + return fa->term; } static BIF_RETTYPE binary_find_trap(BIF_ALIST_3) { int runres; Eterm result; - Binary *bin = erts_magic_ref2bin(BIF_ARG_3); - - runres = do_binary_find(BIF_P, BIF_ARG_1, NULL, bin, BIF_ARG_2, &result); - if (runres == DO_BIN_MATCH_OK) { + Binary *ctx_bin = erts_magic_ref2bin(BIF_ARG_2); + Binary *pat_bin = erts_magic_ref2bin(BIF_ARG_3); + BinaryFindContext *ctx = NULL; + + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(ctx_bin) == bf_context_destructor); + runres = do_binary_find(BIF_P, BIF_ARG_1, &ctx, pat_bin, ctx_bin, &result); + if (runres == BF_OK) { + ASSERT(result != THE_NON_VALUE); BIF_RET(result); } else { - BUMP_ALL_REDS(BIF_P); - BIF_TRAP3(&binary_find_trap_export, BIF_P, BIF_ARG_1, result, BIF_ARG_3); + ASSERT(result == THE_NON_VALUE && ctx->trap_term != result && ctx->pat_term != result); + BIF_TRAP3(&binary_find_trap_export, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } } diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 162d179982..27bbf70c0b 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -46,8 +46,10 @@ #include "erl_thr_progress.h" #include "erl_bif_unique.h" #include "erl_map.h" +#include "erl_check_io.h" #define ERTS_PTAB_WANT_DEBUG_FUNCS__ #include "erl_ptab.h" +#include "erl_time.h" #ifdef HIPE #include "hipe_arch.h" #endif @@ -88,9 +90,7 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE " [64-bit]" #endif " [smp:%beu:%beu]" -#if defined(ERTS_DIRTY_SCHEDULERS) " [ds:%beu:%beu:%beu]" -#endif #if defined(ERTS_DIRTY_SCHEDULERS_TEST) " [dirty-schedulers-TEST]" #endif @@ -98,9 +98,6 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE #ifdef HIPE " [hipe]" #endif -#ifdef ERTS_ENABLE_KERNEL_POLL - " [kernel-poll:%s]" -#endif #ifdef ET_DEBUG #if ET_DEBUG " [type-assertions]" @@ -370,13 +367,8 @@ erts_print_system_version(fmtfn_t to, void *arg, Process *c_p) return erts_print(to, arg, erts_system_version, rc_str , total, online -#ifdef ERTS_DIRTY_SCHEDULERS , dirty_cpu, dirty_cpu_onln, dirty_io -#endif , erts_async_max_threads -#ifdef ERTS_ENABLE_KERNEL_POLL - , erts_use_kernel_poll ? "true" : "false" -#endif ); } @@ -2130,11 +2122,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) ASSERT(erts_compat_rel > 0); BIF_RET(make_small(erts_compat_rel)); } else if (BIF_ARG_1 == am_multi_scheduling) { -#ifndef ERTS_DIRTY_SCHEDULERS - if (erts_no_schedulers == 1) - BIF_RET(am_disabled); - else -#endif { int msb = erts_is_multi_scheduling_blocked(); BIF_RET(!msb @@ -2673,27 +2660,15 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(make_small(active)); } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers", BIF_ARG_1)) { Uint dirty_cpu; -#ifdef ERTS_DIRTY_SCHEDULERS erts_schedulers_state(NULL, NULL, NULL, &dirty_cpu, NULL, NULL, NULL, NULL); -#else - dirty_cpu = 0; -#endif BIF_RET(make_small(dirty_cpu)); } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers_online", BIF_ARG_1)) { Uint dirty_cpu_onln; -#ifdef ERTS_DIRTY_SCHEDULERS erts_schedulers_state(NULL, NULL, NULL, NULL, &dirty_cpu_onln, NULL, NULL, NULL); -#else - dirty_cpu_onln = 0; -#endif BIF_RET(make_small(dirty_cpu_onln)); } else if (ERTS_IS_ATOM_STR("dirty_io_schedulers", BIF_ARG_1)) { Uint dirty_io; -#ifdef ERTS_DIRTY_SCHEDULERS erts_schedulers_state(NULL, NULL, NULL, NULL, NULL, NULL, &dirty_io, NULL); -#else - dirty_io = 0; -#endif BIF_RET(make_small(dirty_io)); } else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) { res = make_small(erts_no_run_queues); @@ -2715,7 +2690,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(make_small(CONTEXT_REDS)); } else if (ERTS_IS_ATOM_STR("kernel_poll", BIF_ARG_1)) { #ifdef ERTS_ENABLE_KERNEL_POLL - BIF_RET(erts_use_kernel_poll ? am_true : am_false); + BIF_RET(am_true); #else BIF_RET(am_false); #endif @@ -2868,7 +2843,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(am_disabled); } else if (ERTS_IS_ATOM_STR("eager_check_io",BIF_ARG_1)) { - BIF_RET(erts_eager_check_io ? am_true : am_false); + BIF_RET(am_true); } else if (ERTS_IS_ATOM_STR("literal_test",BIF_ARG_1)) { #ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE @@ -3467,7 +3442,7 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) ErtsMonotonicTime u1, u2; Eterm b1, b2; Uint hsz; - elapsed_time_both(&u1, NULL, &u2, NULL); + erts_runtime_elapsed_both(&u1, NULL, &u2, NULL); hsz = 3; /* 2-tuple */ (void) erts_bld_monotonic_time(NULL, &hsz, u1); (void) erts_bld_monotonic_time(NULL, &hsz, u2); @@ -3483,7 +3458,7 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) ErtsMonotonicTime w1, w2; Eterm b1, b2; Uint hsz; - wall_clock_elapsed_time_both(&w1, &w2); + erts_wall_clock_elapsed_both(&w1, &w2); hsz = 3; /* 2-tuple */ (void) erts_bld_monotonic_time(NULL, &hsz, w1); (void) erts_bld_monotonic_time(NULL, &hsz, w2); @@ -3600,8 +3575,8 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) (Uint) ciodi.no_used_fds), erts_bld_uint(hpp, szp, (Uint) ciodi.no_driver_select_structs), - erts_bld_uint(hpp, szp, - (Uint) ciodi.no_driver_event_structs)); + erts_bld_uint(hpp, szp, + (Uint) ciodi.no_enif_select_structs)); if (hpp) break; hp = HAlloc(BIF_P, sz); @@ -3756,7 +3731,6 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) subres = make_link_list(BIF_P, dep->nlinks, NIL); subres = make_link_list(BIF_P, dep->node_links, subres); erts_de_links_unlock(dep); - erts_deref_dist_entry(dep); BIF_RET(subres); } else { BIF_RET(am_undefined); @@ -3787,7 +3761,6 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) erts_de_links_lock(dep); ml = make_monitor_list(BIF_P, dep->monitors); erts_de_links_unlock(dep); - erts_deref_dist_entry(dep); BIF_RET(ml); } else { BIF_RET(am_undefined); @@ -3802,7 +3775,6 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) else { Uint cno = dist_entry_channel_no(dep); res = make_small(cno); - erts_deref_dist_entry(dep); } BIF_RET(res); } @@ -3864,15 +3836,14 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) DFLAG_BIT_BINARIES); BIF_RET(erts_term_to_binary(BIF_P, tp[2], 0, dflags)); } - else if (ERTS_IS_ATOM_STR("dist_port", tp[1])) { + else if (ERTS_IS_ATOM_STR("dist_ctrl", tp[1])) { Eterm res = am_undefined; DistEntry *dep = erts_sysname_to_connected_dist_entry(tp[2]); if (dep) { erts_de_rlock(dep); - if (is_internal_port(dep->cid)) + if (is_internal_port(dep->cid) || is_internal_pid(dep->cid)) res = dep->cid; erts_de_runlock(dep); - erts_deref_dist_entry(dep); } BIF_RET(res); } @@ -4281,7 +4252,6 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) con_id = dep->connection_id; erts_de_runlock(dep); erts_kill_dist_connection(dep, con_id); - erts_deref_dist_entry(dep); BIF_RET(am_true); } } diff --git a/erts/emulator/beam/erl_bif_os.c b/erts/emulator/beam/erl_bif_os.c index 5cd172c26f..910325a2f4 100644 --- a/erts/emulator/beam/erl_bif_os.c +++ b/erts/emulator/beam/erl_bif_os.c @@ -37,6 +37,8 @@ #include "dist.h" #include "erl_version.h" +static int check_env_name(char *name); + /* * Return the pid for the Erlang process in the host OS. */ @@ -101,8 +103,10 @@ BIF_RETTYPE os_getenv_1(BIF_ALIST_1) key_str = erts_convert_filename_to_native(BIF_ARG_1,buf,STATIC_BUF_SIZE, ERTS_ALC_T_TMP,1,0,&len); - if (!key_str) { - BIF_ERROR(p, BADARG); + if (!check_env_name(key_str)) { + if (key_str && key_str != &buf[0]) + erts_free(ERTS_ALC_T_TMP, key_str); + BIF_ERROR(p, BADARG); } if (key_str != &buf[0]) @@ -143,25 +147,20 @@ BIF_RETTYPE os_putenv_2(BIF_ALIST_2) { char def_buf_key[STATIC_BUF_SIZE]; char def_buf_value[STATIC_BUF_SIZE]; - char *key_buf, *value_buf; + char *key_buf = NULL, *value_buf = NULL; key_buf = erts_convert_filename_to_native(BIF_ARG_1,def_buf_key, STATIC_BUF_SIZE, ERTS_ALC_T_TMP,0,0,NULL); - if (!key_buf) { - BIF_ERROR(BIF_P, BADARG); - } + if (!check_env_name(key_buf)) + goto badarg; + value_buf = erts_convert_filename_to_native(BIF_ARG_2,def_buf_value, STATIC_BUF_SIZE, ERTS_ALC_T_TMP,1,0, NULL); - if (!value_buf) { - if (key_buf != def_buf_key) { - erts_free(ERTS_ALC_T_TMP, key_buf); - } - BIF_ERROR(BIF_P, BADARG); - } - + if (!value_buf) + goto badarg; if (erts_sys_putenv(key_buf, value_buf)) { if (key_buf != def_buf_key) { @@ -179,6 +178,13 @@ BIF_RETTYPE os_putenv_2(BIF_ALIST_2) erts_free(ERTS_ALC_T_TMP, value_buf); } BIF_RET(am_true); + +badarg: + if (key_buf && key_buf != def_buf_key) + erts_free(ERTS_ALC_T_TMP, key_buf); + if (value_buf && value_buf != def_buf_value) + erts_free(ERTS_ALC_T_TMP, value_buf); + BIF_ERROR(BIF_P, BADARG); } BIF_RETTYPE os_unsetenv_1(BIF_ALIST_1) @@ -188,20 +194,21 @@ BIF_RETTYPE os_unsetenv_1(BIF_ALIST_1) key_buf = erts_convert_filename_to_native(BIF_ARG_1,buf,STATIC_BUF_SIZE, ERTS_ALC_T_TMP,0,0,NULL); - if (!key_buf) { - BIF_ERROR(BIF_P, BADARG); - } + if (!check_env_name(key_buf)) + goto badarg; + + if (erts_sys_unsetenv(key_buf)) + goto badarg; - if (erts_sys_unsetenv(key_buf)) { - if (key_buf != buf) { - erts_free(ERTS_ALC_T_TMP, key_buf); - } - BIF_ERROR(BIF_P, BADARG); - } if (key_buf != buf) { erts_free(ERTS_ALC_T_TMP, key_buf); } BIF_RET(am_true); + +badarg: + if (key_buf && key_buf != buf) + erts_free(ERTS_ALC_T_TMP, key_buf); + BIF_ERROR(BIF_P, BADARG); } BIF_RETTYPE os_set_signal_2(BIF_ALIST_2) { @@ -217,3 +224,27 @@ BIF_RETTYPE os_set_signal_2(BIF_ALIST_2) { error: BIF_ERROR(BIF_P, BADARG); } + +static int +check_env_name(char *raw_name) +{ + byte *c = (byte *) raw_name; + int encoding; + + if (!c) + return 0; + + encoding = erts_get_native_filename_encoding(); + + if (erts_raw_env_char_is_7bit_ascii_char('\0', c, encoding)) + return 0; /* Do not allow empty name... */ + + /* Verify no '=' characters in variable name... */ + do { + if (erts_raw_env_char_is_7bit_ascii_char('=', c, encoding)) + return 0; + c = erts_raw_env_next_char(c, encoding); + } while (!erts_raw_env_char_is_7bit_ascii_char('\0', c, encoding)); + + return 1; /* Seems ok... */ +} diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 4b73be55c6..c4a4dd5863 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -45,7 +45,7 @@ #include "dtrace-wrapper.h" static Port *open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump); -static byte* convert_environment(Process* p, Eterm env); +static char* convert_environment(Eterm env); static char **convert_args(Eterm); static void free_args(char **); @@ -718,11 +718,11 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) goto badarg; } } else if (option == am_env) { - byte* bytes; - if ((bytes = convert_environment(p, *tp)) == NULL) { + if (opts.envir) /* ignore previous env option... */ + erts_free(ERTS_ALC_T_OPEN_PORT_ENV, opts.envir); + opts.envir = convert_environment(*tp); + if (!opts.envir) goto badarg; - } - opts.envir = (char *) bytes; } else if (option == am_args) { char **av; char **oav = opts.argv; @@ -956,6 +956,8 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) erts_atomic32_read_bor_relb(&port->state, sflgs); do_return: + if (opts.envir) + erts_free(ERTS_ALC_T_OPEN_PORT_ENV, opts.envir); if (name_buf) erts_free(ERTS_ALC_T_TMP, (void *) name_buf); if (opts.argv) { @@ -1021,74 +1023,129 @@ static void free_args(char **av) } erts_free(ERTS_ALC_T_TMP, av); } - -static byte* convert_environment(Process* p, Eterm env) +#ifdef DEBUG +#define ERTS_CONV_ENV_BUF_EXTRA 2 +#else +#define ERTS_CONV_ENV_BUF_EXTRA 1024 +#endif + +static char* convert_environment(Eterm env) { - Eterm all; - Eterm* temp_heap; - Eterm* hp; - Uint heap_size; - Sint n; - Sint size; + /* + * Returns environment buffer in memory allocated + * as ERTS_ALC_T_OPEN_PORT_ENV. Caller *needs* + * to deallocate... + */ + + Sint size, alloc_size; byte* bytes; int encoding = erts_get_native_filename_encoding(); - if ((n = erts_list_length(env)) < 0) { - return NULL; - } - heap_size = 2*(5*n+1); - temp_heap = hp = (Eterm *) erts_alloc(ERTS_ALC_T_TMP, heap_size*sizeof(Eterm)); - bytes = NULL; /* Indicating error */ + alloc_size = ERTS_CONV_ENV_BUF_EXTRA; + bytes = erts_alloc(ERTS_ALC_T_OPEN_PORT_ENV, + alloc_size); + size = 0; - /* - * All errors below are handled by jumping to 'done', to ensure that the memory - * gets deallocated. Do NOT return directly from this function. - */ + /* ERTS_CONV_ENV_BUF_EXTRA >= for end delimiter... */ + ERTS_CT_ASSERT(ERTS_CONV_ENV_BUF_EXTRA >= 2); - all = CONS(hp, make_small(0), NIL); - hp += 2; + while (is_list(env)) { + Sint var_sz, val_sz, need; + byte *str, *limit; + Eterm tmp, *tp, *consp; - while(is_list(env)) { - Eterm tmp; - Eterm* tp; + consp = list_val(env); + tmp = CAR(consp); + if (is_not_tuple_arity(tmp, 2)) + goto error; - tmp = CAR(list_val(env)); - if (is_not_tuple_arity(tmp, 2)) { - goto done; - } tp = tuple_val(tmp); - tmp = CONS(hp, make_small(0), NIL); - hp += 2; - if (tp[2] != am_false) { - tmp = CONS(hp, tp[2], tmp); - hp += 2; - } - tmp = CONS(hp, make_small('='), tmp); - hp += 2; - tmp = CONS(hp, tp[1], tmp); - hp += 2; - all = CONS(hp, tmp, all); - hp += 2; - env = CDR(list_val(env)); - } - if (is_not_nil(env)) { - goto done; - } - if ((size = erts_native_filename_need(all,encoding)) < 0) { - goto done; + /* Check encoding of env variable... */ + if (is_not_list(tp[1])) + goto error; + var_sz = erts_native_filename_need(tp[1], encoding); + if (var_sz <= 0) + goto error; + /* Check encoding of value... */ + if (tp[2] == am_false || is_nil(tp[2])) + val_sz = 0; + else if (is_not_list(tp[2])) + goto error; + else { + val_sz = erts_native_filename_need(tp[2], encoding); + if (val_sz < 0) + goto error; + } + + /* Ensure enough memory... */ + need = size; + need += var_sz + val_sz; + /* '=' and '\0' */ + need += 2 * erts_raw_env_7bit_ascii_char_need(encoding); + if (need > alloc_size) { + alloc_size = (need - alloc_size) + alloc_size; + alloc_size += ERTS_CONV_ENV_BUF_EXTRA; + bytes = erts_realloc(ERTS_ALC_T_OPEN_PORT_ENV, + bytes, alloc_size); + } + + /* Write environment variable name... */ + str = bytes + size; + erts_native_filename_put(tp[1], encoding, str); + /* empty variable name is not allowed... */ + if (erts_raw_env_char_is_7bit_ascii_char('\0', str, encoding)) + goto error; + + /* + * Drop null characters at the end and verify that we do + * not have any '=' characters in the name... + */ + limit = str + var_sz; + while (str < limit) { + if (erts_raw_env_char_is_7bit_ascii_char('\0', str, encoding)) + break; + if (erts_raw_env_char_is_7bit_ascii_char('=', str, encoding)) + goto error; + str = erts_raw_env_next_char(str, encoding); + } + + /* Write the equals sign... */ + str = erts_raw_env_7bit_ascii_char_put('=', str, encoding); + + /* Write the value... */ + if (val_sz > 0) { + limit = str + val_sz; + erts_native_filename_put(tp[2], encoding, str); + while (str < limit) { + if (erts_raw_env_char_is_7bit_ascii_char('\0', str, encoding)) + break; + str = erts_raw_env_next_char(str, encoding); + } + } + + /* Delimit... */ + str = erts_raw_env_7bit_ascii_char_put('\0', str, encoding); + + size = str - bytes; + ASSERT(size <= alloc_size); + + env = CDR(consp); } - /* - * Put the result in a binary (no risk for a memory leak that way). - */ - (void) erts_new_heap_binary(p, NULL, size, &bytes); - erts_native_filename_put(all,encoding,bytes); + /* End delimit... */ + (void) erts_raw_env_7bit_ascii_char_put('\0', &bytes[size], encoding); + + if (is_nil(env)) + return (char *) bytes; + +error: + + if (bytes) + erts_free(ERTS_ALC_T_OPEN_PORT_ENV, bytes); - done: - erts_free(ERTS_ALC_T_TMP, temp_heap); - return bytes; + return (char *) NULL; /* error... */ } /* ------------ decode_packet() and friends: */ diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index b02f966558..22942b40c4 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -966,12 +966,12 @@ static int function_is_traced(Process *p, if ((ep = export_get(&e)) != NULL) { pc = ep->beam; if (ep->addressv[erts_active_code_ix()] == pc && - *pc != (BeamInstr) em_call_error_handler) { + ! BeamIsOpCode(*pc, op_call_error_handler)) { int r = 0; - ASSERT(*pc == (BeamInstr) em_apply_bif || - *pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)); + ASSERT(BeamIsOpCode(*pc, op_apply_bif) || + BeamIsOpCode(*pc, op_i_generic_breakpoint)); if (erts_is_trace_break(&ep->info, ms, 0)) { return FUNC_TRACE_GLOBAL_TRACE; @@ -1043,16 +1043,12 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN); erts_thr_progress_block(); } -#ifdef ERTS_DIRTY_SCHEDULERS erts_mtx_lock(&erts_dirty_bp_ix_mtx); -#endif r = function_is_traced(p, mfa, &ms, &ms_meta, &meta, &count, &call_time); -#ifdef ERTS_DIRTY_SCHEDULERS erts_mtx_unlock(&erts_dirty_bp_ix_mtx); -#endif if ( (key == am_call_time) || (key == am_all)) { erts_thr_progress_unblock(); erts_proc_lock(p, ERTS_PROC_LOCK_MAIN); @@ -1365,14 +1361,14 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified, if (ep->addressv[code_ix] != pc) { fp[i].mod->curr.num_traced_exports++; #ifdef DEBUG - ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI); + ep->info.op = BeamOpCodeAddr(op_i_func_info_IaaI); #endif - ep->beam[0] = (BeamInstr) BeamOp(op_jump_f); + ep->beam[0] = BeamOpCodeAddr(op_trace_jump_W); ep->beam[1] = (BeamInstr) ep->addressv[code_ix]; } erts_set_call_trace_bif(ci, match_prog_set, 0); if (ep->addressv[code_ix] != pc) { - ep->beam[0] = (BeamInstr) BeamOp(op_i_generic_breakpoint); + ep->beam[0] = BeamOpCodeAddr(op_i_generic_breakpoint); } } else if (!on && flags.breakpoint) { /* Turn off breakpoint tracing -- nothing to do here. */ @@ -1382,8 +1378,8 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified, * before turning on breakpoint tracing. */ erts_clear_call_trace_bif(ci, 0); - if (ep->beam[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { - ep->beam[0] = (BeamInstr) BeamOp(op_jump_f); + if (BeamIsOpCode(ep->beam[0], op_i_generic_breakpoint)) { + ep->beam[0] = BeamOpCodeAddr(op_trace_jump_W); } } } @@ -1675,7 +1671,7 @@ uninstall_exp_breakpoints(BpFunctions* f) if (ep->addressv[code_ix] != ep->beam) { continue; } - ASSERT(ep->beam[0] == (BeamInstr) BeamOp(op_jump_f)); + ASSERT(BeamIsOpCode(ep->beam[0], op_trace_jump_W)); ep->addressv[code_ix] = (BeamInstr *) ep->beam[1]; } } @@ -1694,7 +1690,7 @@ clean_export_entries(BpFunctions* f) if (ep->addressv[code_ix] == ep->beam) { continue; } - if (ep->beam[0] == (BeamInstr) BeamOp(op_jump_f)) { + if (BeamIsOpCode(ep->beam[0], op_trace_jump_W)) { ep->beam[0] = (BeamInstr) 0; ep->beam[1] = (BeamInstr) 0; } diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c index aa79503819..19d46537f9 100644 --- a/erts/emulator/beam/erl_bif_unique.c +++ b/erts/emulator/beam/erl_bif_unique.c @@ -77,11 +77,9 @@ init_reference(void) ref_init_value += (Uint64) tv.tv_usec; #ifdef DEBUG max_thr_id = (Uint32) erts_no_schedulers; -#ifdef ERTS_DIRTY_SCHEDULERS max_thr_id += (Uint32) erts_no_dirty_cpu_schedulers; max_thr_id += (Uint32) erts_no_dirty_io_schedulers; #endif -#endif erts_atomic64_init_nob(&global_reference.count, (erts_aint64_t) ref_init_value); init_magic_ref_tables(); @@ -439,10 +437,8 @@ init_unique_integer(void) { int bits; unique_data.r.o.val0_max = (Uint64) erts_no_schedulers; -#ifdef ERTS_DIRTY_SCHEDULERS unique_data.r.o.val0_max += (Uint64) erts_no_dirty_cpu_schedulers; unique_data.r.o.val0_max += (Uint64) erts_no_dirty_io_schedulers; -#endif bits = erts_fit_in_bits_int64(unique_data.r.o.val0_max); unique_data.r.o.left_shift = bits; unique_data.r.o.right_shift = 64 - bits; diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index 2035b56eb5..3a16913473 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -32,15 +32,6 @@ #include "erl_bits.h" #include "erl_binary.h" -#ifdef MAX -#undef MAX -#endif -#define MAX(x,y) (((x)>(y))?(x):(y)) -#ifdef MIN -#undef MIN -#endif -#define MIN(x,y) (((x)<(y))?(x):(y)) - #if defined(WORDS_BIGENDIAN) # define BIT_ENDIAN_MACHINE 0 #else @@ -1303,7 +1294,14 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term, goto badarg; } } + + if((ERTS_UINT_MAX - build_size_in_bits) < erts_bin_offset) { + c_p->freason = SYSTEM_LIMIT; + return THE_NON_VALUE; + } + used_size_in_bits = erts_bin_offset + build_size_in_bits; + sb->is_writable = 0; /* Make sure that no one else can write. */ pb->size = NBYTES(used_size_in_bits); pb->flags |= PB_ACTIVE_WRITER; @@ -1377,9 +1375,21 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term, goto badarg; } } - used_size_in_bits = erts_bin_offset + build_size_in_bits; - used_size_in_bytes = NBYTES(used_size_in_bits); - bin_size = 2*used_size_in_bytes; + + if((ERTS_UINT_MAX - build_size_in_bits) < erts_bin_offset) { + c_p->freason = SYSTEM_LIMIT; + return THE_NON_VALUE; + } + + used_size_in_bits = erts_bin_offset + build_size_in_bits; + used_size_in_bytes = NBYTES(used_size_in_bits); + + if(used_size_in_bits < (ERTS_UINT_MAX / 2)) { + bin_size = 2 * used_size_in_bytes; + } else { + bin_size = NBYTES(ERTS_UINT_MAX); + } + bin_size = (bin_size < 256) ? 256 : bin_size; /* @@ -1469,6 +1479,12 @@ erts_bs_private_append(Process* p, Eterm bin, Eterm build_size_term, Uint unit) * Calculate new size in bytes. */ erts_bin_offset = 8*sb->size + sb->bitsize; + + if((ERTS_UINT_MAX - build_size_in_bits) < erts_bin_offset) { + p->freason = SYSTEM_LIMIT; + return THE_NON_VALUE; + } + pos_in_bits_after_build = erts_bin_offset + build_size_in_bits; pb->size = (pos_in_bits_after_build+7) >> 3; pb->flags |= PB_ACTIVE_WRITER; diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index a21b9b9c0c..3ba0886464 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -3550,14 +3550,8 @@ static SWord proc_cleanup_fixed_table(Process* p, DbFixation* fix) ASSERT(sizeof(DbFixation) == ERTS_ALC_DBG_BLK_SZ(fix)); ERTS_DB_ALC_MEM_UPDATE_(tb, sizeof(DbFixation), 0); } - else { - ASSERT(fix->counter == 0); - } db_unlock(tb, LCK_WRITE_REC); } - else { - ASSERT(fix->counter == 0); - } erts_bin_release(fix->tabs.btid); erts_free(ERTS_ALC_T_DB_FIXATION, fix); @@ -3785,11 +3779,8 @@ static void free_fixations_op(DbFixation* fix, void* vctx) { struct free_fixations_ctx* ctx = (struct free_fixations_ctx*) vctx; erts_aint_t diff; -#ifdef DEBUG - DbTable* dbg_tb = btid2tab(fix->tabs.btid); -#endif - ASSERT(!dbg_tb || dbg_tb == ctx->tb); + ASSERT(!btid2tab(fix->tabs.btid)); ASSERT(fix->counter > 0); ASSERT(ctx->tb->common.status & DB_DELETE); diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 25072ede97..5d49b2ea14 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -269,9 +269,6 @@ static ERTS_INLINE Sint next_slot_w(DbTableHash* tb, Uint ix, return ix; } -#ifndef MIN -#define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) -#endif /* * Some special binary flags diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 038f6602bf..5a276b9d88 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -85,9 +85,6 @@ #define EMPTY_NODE(Dtt) (TOP_NODE(Dtt) == NULL) -#ifndef MIN -#define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) -#endif /* Obtain table static stack if available. NULL if not. ** Must be released with release_stack() diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 1c99b661e4..6b126f35d6 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -220,6 +220,9 @@ typedef struct db_fixation { Process* p; } procs; + /* Number of fixations on table from procs.p + * Protected by table write lock or read lock + fixlock + */ Uint counter; } DbFixation; diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 0e8ebf0c98..d5379a40d5 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -40,7 +40,6 @@ #include "erl_drv_nif.h" #include <stdlib.h> -#include <sys/types.h> /* ssize_t */ #if defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_) #ifndef STATIC_ERLANG_DRIVER @@ -48,24 +47,6 @@ #define ERL_DRIVER_TYPES_ONLY #define WIN32_DYNAMIC_ERL_DRIVER #endif -/* - * This structure can be cast to a WSABUF structure. - */ -typedef struct _SysIOVec { - unsigned long iov_len; - char* iov_base; -} SysIOVec; -#else /* Unix */ -# ifdef HAVE_SYS_UIO_H -# include <sys/types.h> -# include <sys/uio.h> -typedef struct iovec SysIOVec; -# else -typedef struct { - char* iov_base; - size_t iov_len; -} SysIOVec; -# endif #endif #ifndef EXTERN @@ -167,14 +148,6 @@ typedef struct _erl_drv_event* ErlDrvEvent; /* An event to be selected on. */ typedef struct _erl_drv_port* ErlDrvPort; /* A port descriptor. */ typedef struct _erl_drv_port* ErlDrvThreadData; /* Thread data. */ -#if !defined(__WIN32__) && !defined(_WIN32) && !defined(_WIN32_) && !defined(USE_SELECT) -struct erl_drv_event_data { - short events; - short revents; -}; -#endif -typedef struct erl_drv_event_data *ErlDrvEventData; /* Event data */ - typedef struct { unsigned long megasecs; unsigned long secs; @@ -289,10 +262,7 @@ typedef struct erl_drv_entry { unsigned int *flags); /* Works mostly like 'control', a synchronous call into the driver. */ - void (*event)(ErlDrvData drv_data, ErlDrvEvent event, - ErlDrvEventData event_data); - /* Called when an event selected by - driver_event() has occurred */ + void (*unused_event_callback)(void); int extended_marker; /* ERL_DRV_EXTENDED_MARKER */ int major_version; /* ERL_DRV_EXTENDED_MAJOR_VERSION */ int minor_version; /* ERL_DRV_EXTENDED_MINOR_VERSION */ @@ -359,8 +329,6 @@ EXTERN void erl_drv_busy_msgq_limits(ErlDrvPort port, ErlDrvSizeT *high); EXTERN int driver_select(ErlDrvPort port, ErlDrvEvent event, int mode, int on); -EXTERN int driver_event(ErlDrvPort port, ErlDrvEvent event, - ErlDrvEventData event_data); EXTERN int driver_output(ErlDrvPort port, char *buf, ErlDrvSizeT len); EXTERN int driver_output2(ErlDrvPort port, char *hbuf, ErlDrvSizeT hlen, diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h index f88138063e..31b4817fb1 100644 --- a/erts/emulator/beam/erl_drv_nif.h +++ b/erts/emulator/beam/erl_drv_nif.h @@ -144,8 +144,25 @@ typedef signed int ErlNapiSInt; #define ERTS_NAPI_USEC__ 2 #define ERTS_NAPI_NSEC__ 3 -#endif /* __ERL_DRV_NIF_H__ */ - - - +#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) +/* + * This structure can be cast to a WSABUF structure. + */ +typedef struct _SysIOVec { + unsigned long iov_len; + char* iov_base; +} SysIOVec; +#else /* Unix */ +# include <sys/types.h> +# ifdef HAVE_SYS_UIO_H +# include <sys/uio.h> +typedef struct iovec SysIOVec; +# else +typedef struct { + char* iov_base; + size_t iov_len; +} SysIOVec; +# endif +#endif +#endif /* __ERL_DRV_NIF_H__ */ diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index bc3bcdc9ad..97a1ca915f 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -183,12 +183,10 @@ typedef struct { erts_atomic32_t refc; } ErtsGCInfoReq; -#ifdef ERTS_DIRTY_SCHEDULERS static struct { erts_mtx_t mtx; ErtsGCInfo info; } dirty_gc; -#endif static ERTS_INLINE int gc_cost(Uint gc_moved_live_words, Uint resize_moved_words) @@ -273,11 +271,9 @@ erts_init_gc(void) init_gc_info(&esdp->gc_info); } -#ifdef ERTS_DIRTY_SCHEDULERS erts_mtx_init(&dirty_gc.mtx, "dirty_gc_info", NIL, ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); init_gc_info(&dirty_gc.info); -#endif init_gcireq_alloc(); } @@ -341,7 +337,7 @@ erts_heap_sizes(Process* p) for (i = num_heap_sizes-1; i >= 0; i--) { n += 2; - if (!MY_IS_SSMALL(heap_sizes[i])) { + if (!IS_SSMALL(heap_sizes[i])) { big += BIG_UINT_HEAP_SIZE; } } @@ -356,7 +352,7 @@ erts_heap_sizes(Process* p) Eterm num; Sint sz = heap_sizes[i]; - if (MY_IS_SSMALL(sz)) { + if (IS_SSMALL(sz)) { num = make_small(sz); } else { num = uint_to_big(sz, bigp); @@ -481,12 +477,10 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int } if (need == 0) { -#ifdef ERTS_DIRTY_SCHEDULERS if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) { ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p))); goto force_reschedule; } -#endif return 1; } /* @@ -541,9 +535,7 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int p->heap_hfrag = hfrag; #endif -#ifdef ERTS_DIRTY_SCHEDULERS force_reschedule: -#endif /* Make sure that we do a proper GC as soon as possible... */ p->flags |= F_FORCE_GC; @@ -616,7 +608,6 @@ young_gen_usage(Process *p) } \ } while (0) -#ifdef ERTS_DIRTY_SCHEDULERS static ERTS_INLINE void check_for_possibly_long_gc(Process *p, Uint ygen_usage) @@ -640,7 +631,6 @@ check_for_possibly_long_gc(Process *p, Uint ygen_usage) } } -#endif /* * Garbage collect a process. @@ -675,21 +665,17 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, state = erts_atomic32_read_nob(&p->state); if ((p->flags & (F_DISABLE_GC|F_DELAY_GC)) || state & ERTS_PSFLG_EXITING) { -#ifdef ERTS_DIRTY_SCHEDULERS delay_gc_before_start: -#endif return delay_garbage_collection(p, live_hf_end, need, fcalls); } ygen_usage = max_young_gen_usage ? max_young_gen_usage : young_gen_usage(p); -#ifdef ERTS_DIRTY_SCHEDULERS if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { check_for_possibly_long_gc(p, ygen_usage); if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) goto delay_gc_before_start; } -#endif if (p->abandoned_heap) live_hf_end = ERTS_INVALID_HFRAG_PTR; @@ -731,14 +717,12 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, if (IS_TRACED_FL(p, F_TRACE_GC)) { trace_gc(p, am_gc_minor_end, reclaimed_now, THE_NON_VALUE); } -#ifdef ERTS_DIRTY_SCHEDULERS if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { p->flags |= F_NEED_FULLSWEEP; check_for_possibly_long_gc(p, ygen_usage); if (p->flags & F_DIRTY_MAJOR_GC) goto delay_gc_after_start; } -#endif goto do_major_collection; } if (ERTS_SCHEDULER_IS_DIRTY(esdp)) @@ -784,9 +768,7 @@ do_major_collection: am_kill, NIL, NULL, 0); erts_proc_unlock(p, locks & ERTS_PROC_LOCKS_ALL_MINOR); -#ifdef ERTS_DIRTY_SCHEDULERS delay_gc_after_start: -#endif /* erts_send_exit_signal looks for ERTS_PSFLG_GC, so we have to remove it after the signal is sent */ erts_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); @@ -821,7 +803,6 @@ do_major_collection: monitor_large_heap(p); } -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { erts_mtx_lock(&dirty_gc.mtx); dirty_gc.info.garbage_cols++; @@ -829,7 +810,6 @@ do_major_collection: erts_mtx_unlock(&dirty_gc.mtx); } else -#endif { esdp->gc_info.garbage_cols++; esdp->gc_info.reclaimed += reclaimed_now; @@ -907,7 +887,6 @@ garbage_collect_hibernate(Process* p, int check_long_gc) if (p->flags & F_DISABLE_GC) ERTS_INTERNAL_ERROR("GC disabled"); -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p))) p->flags &= ~(F_DIRTY_GC_HIBERNATE|F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC); else if (check_long_gc) { @@ -920,7 +899,6 @@ garbage_collect_hibernate(Process* p, int check_long_gc) } p->flags = flags; } -#endif /* * Preliminaries. */ @@ -1110,7 +1088,6 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, p->flags |= F_NEED_FULLSWEEP; -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p))) p->flags &= ~F_DIRTY_CLA; else { @@ -1126,7 +1103,6 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, return 10; } } -#endif reds = (Sint64) garbage_collect(p, ERTS_INVALID_HFRAG_PTR, 0, p->arg_reg, p->arity, fcalls, @@ -3230,7 +3206,6 @@ reply_gc_info(void *vgcirp) reclaimed = esdp->gc_info.reclaimed; garbage_cols = esdp->gc_info.garbage_cols; -#ifdef ERTS_DIRTY_SCHEDULERS /* * Add dirty schedulers info on requesting * schedulers info @@ -3241,7 +3216,6 @@ reply_gc_info(void *vgcirp) garbage_cols += dirty_gc.info.garbage_cols; erts_mtx_unlock(&dirty_gc.mtx); } -#endif sz = 0; hpp = NULL; diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index f8cbe6f49a..bda2c9b94d 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -96,14 +96,14 @@ typedef enum { #define ERTS_BIF_TIMER_SHORT_TIME 5000 -/* Bit 0 to 9 contains scheduler id (see mask below) */ -#define ERTS_TMR_ROFLG_HLT (((Uint32) 1) << 10) -#define ERTS_TMR_ROFLG_BIF_TMR (((Uint32) 1) << 11) -#define ERTS_TMR_ROFLG_PRE_ALC (((Uint32) 1) << 12) -#define ERTS_TMR_ROFLG_REG_NAME (((Uint32) 1) << 13) -#define ERTS_TMR_ROFLG_PROC (((Uint32) 1) << 14) -#define ERTS_TMR_ROFLG_PORT (((Uint32) 1) << 15) -#define ERTS_TMR_ROFLG_CALLBACK (((Uint32) 1) << 16) +/* Bit 0 to 10 contains scheduler id (see mask below) */ +#define ERTS_TMR_ROFLG_HLT (((Uint32) 1) << 11) +#define ERTS_TMR_ROFLG_BIF_TMR (((Uint32) 1) << 12) +#define ERTS_TMR_ROFLG_PRE_ALC (((Uint32) 1) << 13) +#define ERTS_TMR_ROFLG_REG_NAME (((Uint32) 1) << 14) +#define ERTS_TMR_ROFLG_PROC (((Uint32) 1) << 15) +#define ERTS_TMR_ROFLG_PORT (((Uint32) 1) << 16) +#define ERTS_TMR_ROFLG_CALLBACK (((Uint32) 1) << 17) #define ERTS_TMR_ROFLG_SID_MASK \ (ERTS_TMR_ROFLG_HLT - (Uint32) 1) @@ -1256,14 +1256,15 @@ bif_timer_timeout(ErtsHLTimerService *srv, ERTS_HLT_ASSERT(proc); } if (proc) { + int dec_refc = 0; + ErtsMessage *mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = tmr->btm.bp; + tmr->btm.bp = NULL; + erts_queue_message(proc, 0, mp, tmr->btm.message, + am_clock_service); + erts_proc_lock(proc, ERTS_PROC_LOCK_BTM); + /* If the process is exiting do not disturb the cleanup... */ if (!ERTS_PROC_IS_EXITING(proc)) { - int dec_refc = 0; - ErtsMessage *mp = erts_alloc_message(0, NULL); - mp->data.heap_frag = tmr->btm.bp; - tmr->btm.bp = NULL; - erts_queue_message(proc, 0, mp, tmr->btm.message, - am_clock_service); - erts_proc_lock(proc, ERTS_PROC_LOCK_BTM); #ifdef ERTS_MAGIC_REF_BIF_TIMERS if (tmr->btm.proc_list.next) { proc_btm_list_delete(&proc->bif_timers, tmr); @@ -1276,10 +1277,10 @@ bif_timer_timeout(ErtsHLTimerService *srv, dec_refc = 1; } #endif - erts_proc_unlock(proc, ERTS_PROC_LOCK_BTM); - if (dec_refc) - timer_pre_dec_refc((ErtsTimer *) tmr); } + erts_proc_unlock(proc, ERTS_PROC_LOCK_BTM); + if (dec_refc) + timer_pre_dec_refc((ErtsTimer *) tmr); } if (tmr->btm.bp) free_message_buffer(tmr->btm.bp); diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 34affaa015..6cef9bd0e3 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -49,6 +49,7 @@ #include "erl_bif_unique.h" #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" +#include "erl_check_io.h" #ifdef HIPE #include "hipe_mode_switch.h" /* for hipe_mode_switch_init() */ @@ -76,8 +77,10 @@ const char etp_otp_release[] = ERLANG_OTP_RELEASE; const char etp_compile_date[] = ERLANG_COMPILE_DATE; const char etp_arch[] = ERLANG_ARCHITECTURE; #ifdef ERTS_ENABLE_KERNEL_POLL +const int erts_use_kernel_poll = 1; const int etp_kernel_poll_support = 1; #else +const int erts_use_kernel_poll = 0; const int etp_kernel_poll_support = 0; #endif #if defined(ARCH_64) @@ -180,11 +183,9 @@ int erts_compat_rel; static int no_schedulers; static int no_schedulers_online; -#ifdef ERTS_DIRTY_SCHEDULERS static int no_dirty_cpu_schedulers; static int no_dirty_cpu_schedulers_online; static int no_dirty_io_schedulers; -#endif #ifdef DEBUG Uint32 verbose; /* See erl_debug.h for information about verbose */ @@ -205,16 +206,16 @@ int erts_no_line_info = 0; /* -L: Don't load line information */ */ ErtsModifiedTimings erts_modified_timings[] = { - /* 0 */ {make_small(0), CONTEXT_REDS, INPUT_REDUCTIONS}, - /* 1 */ {make_small(0), (3*CONTEXT_REDS)/4, 2*INPUT_REDUCTIONS}, - /* 2 */ {make_small(0), CONTEXT_REDS/2, INPUT_REDUCTIONS/2}, - /* 3 */ {make_small(0), (7*CONTEXT_REDS)/8, 3*INPUT_REDUCTIONS}, - /* 4 */ {make_small(0), CONTEXT_REDS/3, 3*INPUT_REDUCTIONS}, - /* 5 */ {make_small(0), (10*CONTEXT_REDS)/11, INPUT_REDUCTIONS/2}, - /* 6 */ {make_small(1), CONTEXT_REDS/4, 2*INPUT_REDUCTIONS}, - /* 7 */ {make_small(1), (5*CONTEXT_REDS)/7, INPUT_REDUCTIONS/3}, - /* 8 */ {make_small(10), CONTEXT_REDS/5, 3*INPUT_REDUCTIONS}, - /* 9 */ {make_small(10), (6*CONTEXT_REDS)/7, INPUT_REDUCTIONS/4} + /* 0 */ {make_small(0), CONTEXT_REDS}, + /* 1 */ {make_small(0), (3*CONTEXT_REDS)/4}, + /* 2 */ {make_small(0), CONTEXT_REDS/2}, + /* 3 */ {make_small(0), (7*CONTEXT_REDS)/8}, + /* 4 */ {make_small(0), CONTEXT_REDS/3}, + /* 5 */ {make_small(0), (10*CONTEXT_REDS)/11}, + /* 6 */ {make_small(1), CONTEXT_REDS/4}, + /* 7 */ {make_small(1), (5*CONTEXT_REDS)/7}, + /* 8 */ {make_small(10), CONTEXT_REDS/5}, + /* 9 */ {make_small(10), (6*CONTEXT_REDS)/7} }; #define ERTS_MODIFIED_TIMING_LEVELS \ @@ -312,12 +313,11 @@ erl_init(int ncpu, erts_init_sys_common_misc(); erts_init_process(ncpu, proc_tab_sz, legacy_proc_tab); erts_init_scheduling(no_schedulers, - no_schedulers_online -#ifdef ERTS_DIRTY_SCHEDULERS - , no_dirty_cpu_schedulers, + no_schedulers_online, + erts_no_poll_threads, + no_dirty_cpu_schedulers, no_dirty_cpu_schedulers_online, no_dirty_io_schedulers -#endif ); erts_late_init_time_sup(); erts_init_cpu_topology(); /* Must be after init_scheduling */ @@ -562,9 +562,19 @@ void erts_usage(void) erts_fprintf(stderr, "-hmqd val set default message queue data flag for processes,\n"); erts_fprintf(stderr, " valid values are: off_heap | on_heap\n"); + erts_fprintf(stderr, "-IOp number set number of pollsets to be used to poll for I/O,\n"); + erts_fprintf(stderr, " This value has to be equal or smaller than the\n"); + erts_fprintf(stderr, " number of poll threads. If the current platform\n"); + erts_fprintf(stderr, " does not support concurrent update of pollsets\n"); + erts_fprintf(stderr, " this value is ignored.\n"); + erts_fprintf(stderr, "-IOt number set number of threads to be used to poll for I/O\n"); + erts_fprintf(stderr, "-IOPp number set number of pollsets as a percentage of the\n"); + erts_fprintf(stderr, " number of poll threads."); + erts_fprintf(stderr, "-IOPt number set number of threads to be used to poll for I/O\n"); + erts_fprintf(stderr, " as a percentage of the number of schedulers."); + /* erts_fprintf(stderr, "-i module set the boot module (default init)\n"); */ - erts_fprintf(stderr, "-K boolean enable or disable kernel poll\n"); erts_fprintf(stderr, "-n[s|a|d] Control behavior of signals to ports\n"); erts_fprintf(stderr, " Note that this flag is deprecated!\n"); erts_fprintf(stderr, "-M<X> <Y> memory allocator switches,\n"); @@ -609,7 +619,6 @@ void erts_usage(void) ERTS_SCHED_THREAD_MIN_STACK_SIZE, ERTS_SCHED_THREAD_MAX_STACK_SIZE, ERTS_DEFAULT_SCHED_STACK_SIZE); -#ifdef ERTS_DIRTY_SCHEDULERS erts_fprintf(stderr, "-sssdcpu size suggested stack size in kilo words for dirty CPU scheduler\n"); erts_fprintf(stderr, " threads, valid range is [%d-%d] (default %d)\n", ERTS_SCHED_THREAD_MIN_STACK_SIZE, @@ -620,7 +629,6 @@ void erts_usage(void) ERTS_SCHED_THREAD_MIN_STACK_SIZE, ERTS_SCHED_THREAD_MAX_STACK_SIZE, ERTS_DEFAULT_DIO_SCHED_STACK_SIZE); -#endif erts_fprintf(stderr, "-spp Bool set port parallelism scheduling hint\n"); erts_fprintf(stderr, "-S n1:n2 set number of schedulers (n1), and number of\n"); erts_fprintf(stderr, " schedulers online (n2), maximum for both\n"); @@ -629,7 +637,6 @@ void erts_usage(void) erts_fprintf(stderr, "-SP p1:p2 specify schedulers (p1) and schedulers online (p2)\n"); erts_fprintf(stderr, " as percentages of logical processors configured and logical\n"); erts_fprintf(stderr, " processors available, respectively\n"); -#ifdef ERTS_DIRTY_SCHEDULERS erts_fprintf(stderr, "-SDcpu n1:n2 set number of dirty CPU schedulers (n1), and number of\n"); erts_fprintf(stderr, " dirty CPU schedulers online (n2), valid range for both\n"); erts_fprintf(stderr, " numbers is [1-%d], and n2 must be less than or equal to n1\n", @@ -639,7 +646,6 @@ void erts_usage(void) erts_fprintf(stderr, " and logical processors available, respectively\n"); erts_fprintf(stderr, "-SDio n set number of dirty I/O schedulers, valid range is [0-%d]\n", ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS); -#endif erts_fprintf(stderr, "-t size set the maximum number of atoms the emulator can handle\n"); erts_fprintf(stderr, " valid range is [%d-%d]\n", MIN_ATOM_TABLE_SIZE, MAX_ATOM_TABLE_SIZE); @@ -725,19 +731,16 @@ early_init(int *argc, char **argv) /* int schdlrs_percentage = 100; int schdlrs_onln_percentage = 100; int max_main_threads; -#ifdef ERTS_DIRTY_SCHEDULERS int dirty_cpu_scheds; int dirty_cpu_scheds_online; int dirty_cpu_scheds_pctg = 100; int dirty_cpu_scheds_onln_pctg = 100; int dirty_io_scheds; -#endif int max_reader_groups; int reader_groups; char envbuf[21]; /* enough for any 64-bit integer */ size_t envbufsz; - erts_save_emu_args(*argc, argv); erts_sched_compact_load = 1; @@ -794,11 +797,9 @@ early_init(int *argc, char **argv) /* schdlrs = no_schedulers; schdlrs_onln = no_schedulers_online; -#ifdef ERTS_DIRTY_SCHEDULERS dirty_cpu_scheds = no_schedulers; dirty_cpu_scheds_online = no_schedulers_online; dirty_io_scheds = 10; -#endif envbufsz = sizeof(envbuf); @@ -851,6 +852,7 @@ early_init(int *argc, char **argv) /* } break; } + case 'S' : if (argv[i][2] == 'P') { int ptot, ponln; @@ -891,7 +893,6 @@ early_init(int *argc, char **argv) /* ("using %d:%d scheduler percentages\n", schdlrs_percentage, schdlrs_onln_percentage)); } -#ifdef ERTS_DIRTY_SCHEDULERS else if (argv[i][2] == 'D') { char *arg; char *type = argv[i]+3; @@ -1003,7 +1004,6 @@ early_init(int *argc, char **argv) /* break; } } -#endif else { int tot, onln; char *arg = get_arg(argv[i]+2, argv[i+1], &i); @@ -1085,7 +1085,6 @@ early_init(int *argc, char **argv) /* erts_usage(); } } -#ifdef ERTS_DIRTY_SCHEDULERS /* apply any dirty scheduler precentages */ if (dirty_cpu_scheds_pctg != 100 || dirty_cpu_scheds_onln_pctg != 100) { dirty_cpu_scheds = dirty_cpu_scheds * dirty_cpu_scheds_pctg / 100; @@ -1099,7 +1098,6 @@ early_init(int *argc, char **argv) /* dirty_cpu_scheds_online = schdlrs_onln; if (dirty_cpu_scheds_online < 1) dirty_cpu_scheds_online = 1; -#endif } @@ -1107,17 +1105,18 @@ early_init(int *argc, char **argv) /* no_schedulers_online = schdlrs_onln; erts_no_schedulers = (Uint) no_schedulers; -#ifdef ERTS_DIRTY_SCHEDULERS erts_no_dirty_cpu_schedulers = no_dirty_cpu_schedulers = dirty_cpu_scheds; no_dirty_cpu_schedulers_online = dirty_cpu_scheds_online; erts_no_dirty_io_schedulers = no_dirty_io_schedulers = dirty_io_scheds; -#endif erts_early_init_scheduling(no_schedulers); alloc_opts.ncpu = ncpu; erts_alloc_init(argc, argv, &alloc_opts); /* Handles (and removes) -M flags. */ /* Require allocators */ + + erts_init_check_io(argc, argv); + /* * Thread progress management: * @@ -1125,20 +1124,17 @@ early_init(int *argc, char **argv) /* * ** Scheduler threads (see erl_process.c) * ** Aux thread (see erl_process.c) * ** Sys message dispatcher thread (see erl_trace.c) + * ** IO Poll threads (see erl_check_io.c) * * * Unmanaged threads that need to register: * ** Async threads (see erl_async.c) * ** Dirty scheduler threads */ erts_thr_progress_init(no_schedulers, - no_schedulers+2, -#ifndef ERTS_DIRTY_SCHEDULERS - erts_async_max_threads -#else + no_schedulers+2+erts_no_poll_threads, erts_async_max_threads + erts_no_dirty_cpu_schedulers + erts_no_dirty_io_schedulers -#endif ); erts_thr_q_init(); erts_init_utils(); @@ -1237,10 +1233,8 @@ erl_start(int argc, char **argv) * a lot of stack. */ erts_sched_thread_suggested_stack_size = ERTS_DEFAULT_SCHED_STACK_SIZE; -#ifdef ERTS_DIRTY_SCHEDULERS erts_dcpu_sched_thread_suggested_stack_size = ERTS_DEFAULT_DCPU_SCHED_STACK_SIZE; erts_dio_sched_thread_suggested_stack_size = ERTS_DEFAULT_DIO_SCHED_STACK_SIZE; -#endif #ifdef DEBUG verbose = DEBUG_DEFAULT; @@ -1585,16 +1579,6 @@ erl_start(int argc, char **argv) have_break_handler = 0; break; - case 'K': - /* If kernel poll support is present, - erl_sys_args() will remove the K parameter - and value */ - get_arg(argv[i]+2, argv[i+1], &i); - erts_fprintf(stderr, - "kernel-poll not supported; \"K\" parameter ignored\n", - arg); - break; - case 'n': arg = get_arg(argv[i]+2, argv[i+1], &i); switch (arg[0]) { @@ -1764,22 +1748,9 @@ erl_start(int argc, char **argv) erts_usage(); } } - else if (has_prefix("ecio", sub_param)) { - arg = get_arg(sub_param+4, argv[i+1], &i); -#ifndef __OSE__ - if (sys_strcmp("true", arg) == 0) - erts_eager_check_io = 1; - else -#endif - if (sys_strcmp("false", arg) == 0) - erts_eager_check_io = 0; - else { - erts_fprintf(stderr, - "bad schedule eager check I/O value '%s'\n", - arg); - erts_usage(); - } - } + else if (has_prefix("ecio", sub_param)) { + /* ignore argument, eager check io no longer used */ + } else if (has_prefix("pp", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); if (sys_strcmp(arg, "true") == 0) @@ -1855,7 +1826,6 @@ erl_start(int argc, char **argv) VERBOSE(DEBUG_SYSTEM, ("scheduler wakeup threshold: %s\n", arg)); } -#ifdef ERTS_DIRTY_SCHEDULERS else if (has_prefix("ssdcpu", sub_param)) { /* suggested stack size (Kilo Words) for dirty CPU scheduler threads */ arg = get_arg(sub_param+6, argv[i+1], &i); @@ -1890,7 +1860,6 @@ erl_start(int argc, char **argv) ("suggested dirty IO scheduler thread stack size %d kilo words\n", erts_dio_sched_thread_suggested_stack_size)); } -#endif else if (has_prefix("ss", sub_param)) { /* suggested stack size (Kilo Words) for scheduler threads */ arg = get_arg(sub_param+2, argv[i+1], &i); @@ -2205,12 +2174,10 @@ erl_start(int argc, char **argv) if (erts_sched_thread_suggested_stack_size < ERTS_SCHED_THREAD_MIN_STACK_SIZE) erts_sched_thread_suggested_stack_size = ERTS_SCHED_THREAD_MIN_STACK_SIZE; -#ifdef ERTS_DIRTY_SCHEDULERS if (erts_dcpu_sched_thread_suggested_stack_size < ERTS_SCHED_THREAD_MIN_STACK_SIZE) erts_dcpu_sched_thread_suggested_stack_size = ERTS_SCHED_THREAD_MIN_STACK_SIZE; if (erts_dio_sched_thread_suggested_stack_size < ERTS_SCHED_THREAD_MIN_STACK_SIZE) erts_dio_sched_thread_suggested_stack_size = ERTS_SCHED_THREAD_MIN_STACK_SIZE; -#endif erl_init(ncpu, proc_tab_sz, @@ -2255,7 +2222,6 @@ erl_start(int argc, char **argv) && erts_literal_area_collector->common.id == pid); erts_proc_inc_refc(erts_literal_area_collector); -#ifdef ERTS_DIRTY_SCHEDULERS pid = erl_system_process_otp(otp_ring0_pid, "erts_dirty_process_code_checker", !0); erts_dirty_process_code_checker = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, @@ -2263,7 +2229,6 @@ erl_start(int argc, char **argv) ASSERT(erts_dirty_process_code_checker && erts_dirty_process_code_checker->common.id == pid); erts_proc_inc_refc(erts_dirty_process_code_checker); -#endif } diff --git a/erts/emulator/beam/erl_io_queue.c b/erts/emulator/beam/erl_io_queue.c new file mode 100644 index 0000000000..190ba6bbb9 --- /dev/null +++ b/erts/emulator/beam/erl_io_queue.c @@ -0,0 +1,1231 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2017. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "global.h" + +#define ERL_WANT_HIPE_BIF_WRAPPER__ +#include "bif.h" +#undef ERL_WANT_HIPE_BIF_WRAPPER__ + +#include "erl_bits.h" +#include "erl_io_queue.h" + +#define IOL2V_SMALL_BIN_LIMIT (ERL_ONHEAP_BIN_LIMIT * 4) + +static void free_binary(ErtsIOQBinary *b, int driver); +static ErtsIOQBinary *alloc_binary(Uint size, char *source, void **iov_base, int driver); + +void erts_ioq_init(ErtsIOQueue *q, ErtsAlcType_t alct, int driver) +{ + + ERTS_CT_ASSERT(offsetof(ErlNifIOVec,flags) == sizeof(ErtsIOVecCommon)); + ERTS_CT_ASSERT(sizeof(ErlIOVec) == sizeof(ErtsIOVecCommon)); + ERTS_CT_ASSERT(sizeof(size_t) == sizeof(ErlDrvSizeT)); + ERTS_CT_ASSERT(sizeof(size_t) == sizeof(Uint)); + + q->alct = alct; + q->driver = driver; + q->size = 0; + q->v_head = q->v_tail = q->v_start = q->v_small; + q->v_end = q->v_small + ERTS_SMALL_IO_QUEUE; + q->b_head = q->b_tail = q->b_start = q->b_small; + q->b_end = q->b_small + ERTS_SMALL_IO_QUEUE; +} + +void erts_ioq_clear(ErtsIOQueue *q) +{ + ErtsIOQBinary** binp = q->b_head; + int driver = q->driver; + + if (q->v_start != q->v_small) + erts_free(q->alct, (void *) q->v_start); + + while(binp < q->b_tail) { + if (*binp != NULL) + free_binary(*binp, driver); + binp++; + } + if (q->b_start != q->b_small) + erts_free(q->alct, (void *) q->b_start); + q->v_start = q->v_end = q->v_head = q->v_tail = NULL; + q->b_start = q->b_end = q->b_head = q->b_tail = NULL; + q->size = 0; +} + +static void free_binary(ErtsIOQBinary *b, int driver) +{ + if (driver) + driver_free_binary(&b->driver); + else if (erts_refc_dectest(&b->nif.intern.refc, 0) == 0) + erts_bin_free(&b->nif); +} + +static ErtsIOQBinary *alloc_binary(Uint size, char *source, void **iov_base, int driver) +{ + if (driver) { + ErlDrvBinary *bin = driver_alloc_binary(size); + if (!bin) return NULL; + sys_memcpy(bin->orig_bytes, source, size); + *iov_base = bin->orig_bytes; + return (ErtsIOQBinary *)bin; + } else { + /* This clause can be triggered in enif_ioq_enq_binary is used */ + Binary *bin = erts_bin_nrml_alloc(size); + if (!bin) return NULL; + erts_refc_init(&bin->intern.refc, 1); + sys_memcpy(bin->orig_bytes, source, size); + *iov_base = bin->orig_bytes; + return (ErtsIOQBinary *)bin; + } +} + +Uint erts_ioq_size(ErtsIOQueue *q) +{ + return q->size; +} + +/* expand queue to hold n elements in tail or head */ +static int expandq(ErtsIOQueue* q, int n, int tail) +/* tail: 0 if make room in head, make room in tail otherwise */ +{ + int h_sz; /* room before header */ + int t_sz; /* room after tail */ + int q_sz; /* occupied */ + int nvsz; + SysIOVec* niov; + ErtsIOQBinary** nbinv; + + h_sz = q->v_head - q->v_start; + t_sz = q->v_end - q->v_tail; + q_sz = q->v_tail - q->v_head; + + if (tail && (n <= t_sz)) /* do we need to expand tail? */ + return 0; + else if (!tail && (n <= h_sz)) /* do we need to expand head? */ + return 0; + else if (n > (h_sz + t_sz)) { /* need to allocate */ + /* we may get little extra but it ok */ + nvsz = (q->v_end - q->v_start) + n; + + niov = erts_alloc_fnf(q->alct, nvsz * sizeof(SysIOVec)); + if (!niov) + return -1; + nbinv = erts_alloc_fnf(q->alct, nvsz * sizeof(ErtsIOQBinary**)); + if (!nbinv) { + erts_free(q->alct, (void *) niov); + return -1; + } + if (tail) { + sys_memcpy(niov, q->v_head, q_sz*sizeof(SysIOVec)); + if (q->v_start != q->v_small) + erts_free(q->alct, (void *) q->v_start); + q->v_start = niov; + q->v_end = niov + nvsz; + q->v_head = q->v_start; + q->v_tail = q->v_head + q_sz; + + sys_memcpy(nbinv, q->b_head, q_sz*sizeof(ErtsIOQBinary*)); + if (q->b_start != q->b_small) + erts_free(q->alct, (void *) q->b_start); + q->b_start = nbinv; + q->b_end = nbinv + nvsz; + q->b_head = q->b_start; + q->b_tail = q->b_head + q_sz; + } + else { + sys_memcpy(niov+nvsz-q_sz, q->v_head, q_sz*sizeof(SysIOVec)); + if (q->v_start != q->v_small) + erts_free(q->alct, (void *) q->v_start); + q->v_start = niov; + q->v_end = niov + nvsz; + q->v_tail = q->v_end; + q->v_head = q->v_tail - q_sz; + + sys_memcpy(nbinv+nvsz-q_sz, q->b_head, q_sz*sizeof(ErtsIOQBinary*)); + if (q->b_start != q->b_small) + erts_free(q->alct, (void *) q->b_start); + q->b_start = nbinv; + q->b_end = nbinv + nvsz; + q->b_tail = q->b_end; + q->b_head = q->b_tail - q_sz; + } + } + else if (tail) { /* move to beginning to make room in tail */ + sys_memmove(q->v_start, q->v_head, q_sz*sizeof(SysIOVec)); + q->v_head = q->v_start; + q->v_tail = q->v_head + q_sz; + sys_memmove(q->b_start, q->b_head, q_sz*sizeof(ErtsIOQBinary*)); + q->b_head = q->b_start; + q->b_tail = q->b_head + q_sz; + } + else { /* move to end to make room */ + sys_memmove(q->v_end-q_sz, q->v_head, q_sz*sizeof(SysIOVec)); + q->v_tail = q->v_end; + q->v_head = q->v_tail-q_sz; + sys_memmove(q->b_end-q_sz, q->b_head, q_sz*sizeof(ErtsIOQBinary*)); + q->b_tail = q->b_end; + q->b_head = q->b_tail-q_sz; + } + + return 0; +} + +static +int skip(ErtsIOVec* vec, Uint skipbytes, + SysIOVec **iovp, ErtsIOQBinary ***binvp, + Uint *lenp) +{ + int n; + Uint len; + SysIOVec* iov; + ErtsIOQBinary** binv; + + if (vec->common.size <= skipbytes) + return -1; + + iov = vec->common.iov; + binv = vec->common.binv; + n = vec->common.vsize; + /* we use do here to strip iov_len=0 from beginning */ + do { + len = iov->iov_len; + if (len <= skipbytes) { + skipbytes -= len; + iov++; + binv++; + n--; + } + else { + iov->iov_base = ((char *)(iov->iov_base)) + skipbytes; + iov->iov_len -= skipbytes; + skipbytes = 0; + } + } while(skipbytes > 0); + + *binvp = binv; + *iovp = iov; + *lenp = len; + + return n; +} + +/* Put elements from vec at q tail */ +int erts_ioq_enqv(ErtsIOQueue *q, ErtsIOVec *eiov, Uint skipbytes) +{ + int n; + Uint len; + Uint size = eiov->common.size - skipbytes; + SysIOVec *iov; + ErtsIOQBinary** binv; + ErtsIOQBinary* b; + + if (q == NULL) + return -1; + + ASSERT(eiov->common.size >= skipbytes); + if (eiov->common.size <= skipbytes) + return 0; + + n = skip(eiov, skipbytes, &iov, &binv, &len); + + if (n < 0) + return n; + + if (q->v_tail + n >= q->v_end) + if (expandq(q, n, 1)) + return -1; + + /* Queue and reference all binaries (remove zero length items) */ + while(n--) { + if ((len = iov->iov_len) > 0) { + if ((b = *binv) == NULL) { /* special case create binary ! */ + b = alloc_binary(len, iov->iov_base, (void**)&q->v_tail->iov_base, + q->driver); + if (!b) return -1; + *q->b_tail++ = b; + q->v_tail->iov_len = len; + q->v_tail++; + } + else { + if (q->driver) + driver_binary_inc_refc(&b->driver); + else + erts_refc_inc(&b->nif.intern.refc, 1); + *q->b_tail++ = b; + *q->v_tail++ = *iov; + } + } + iov++; + binv++; + } + q->size += size; /* update total size in queue */ + return 0; +} + +/* Put elements from vec at q head */ +int erts_ioq_pushqv(ErtsIOQueue *q, ErtsIOVec* vec, Uint skipbytes) +{ + int n; + Uint len; + Uint size = vec->common.size - skipbytes; + SysIOVec* iov; + ErtsIOQBinary** binv; + ErtsIOQBinary* b; + + if (q == NULL) + return -1; + + ASSERT(vec->common.size >= skipbytes); + if (vec->common.size <= skipbytes) + return 0; + + n = skip(vec, skipbytes, &iov, &binv, &len); + + if (n < 0) + return n; + + if (q->v_head - n < q->v_start) + if (expandq(q, n, 0)) + return -1; + + /* Queue and reference all binaries (remove zero length items) */ + iov += (n-1); /* move to end */ + binv += (n-1); /* move to end */ + while(n--) { + if ((len = iov->iov_len) > 0) { + if ((b = *binv) == NULL) { /* special case create binary ! */ + if (q->driver) { + ErlDrvBinary *bin = driver_alloc_binary(len); + if (!bin) return -1; + sys_memcpy(bin->orig_bytes, iov->iov_base, len); + b = (ErtsIOQBinary *)bin; + q->v_head->iov_base = bin->orig_bytes; + } + *--q->b_head = b; + q->v_head--; + q->v_head->iov_len = len; + } + else { + if (q->driver) + driver_binary_inc_refc(&b->driver); + else + erts_refc_inc(&b->nif.intern.refc, 1); + *--q->b_head = b; + *--q->v_head = *iov; + } + } + iov--; + binv--; + } + q->size += size; /* update total size in queue */ + return 0; +} + + +/* +** Remove size bytes from queue head +** Return number of bytes that remain in queue +*/ +int erts_ioq_deq(ErtsIOQueue *q, Uint size) +{ + Uint len; + + if ((q == NULL) || (q->size < size)) + return -1; + q->size -= size; + while (size > 0) { + ASSERT(q->v_head != q->v_tail); + + len = q->v_head->iov_len; + if (len <= size) { + size -= len; + free_binary(*q->b_head, q->driver); + *q->b_head++ = NULL; + q->v_head++; + } + else { + q->v_head->iov_base = ((char *)(q->v_head->iov_base)) + size; + q->v_head->iov_len -= size; + size = 0; + } + } + + /* restart pointers (optimised for enq) */ + if (q->v_head == q->v_tail) { + q->v_head = q->v_tail = q->v_start; + q->b_head = q->b_tail = q->b_start; + } + return 0; +} + + +Uint erts_ioq_peekqv(ErtsIOQueue *q, ErtsIOVec *ev) { + ASSERT(ev); + + if (! q) { + return (Uint) -1; + } else { + if ((ev->common.vsize = q->v_tail - q->v_head) == 0) { + ev->common.size = 0; + ev->common.iov = NULL; + ev->common.binv = NULL; + } else { + ev->common.size = q->size; + ev->common.iov = q->v_head; + ev->common.binv = q->b_head; + } + return q->size; + } +} + +SysIOVec* erts_ioq_peekq(ErtsIOQueue *q, int* vlenp) /* length of io-vector */ +{ + + if (q == NULL) { + *vlenp = -1; + return NULL; + } + if ((*vlenp = (q->v_tail - q->v_head)) == 0) + return NULL; + return q->v_head; +} + +/* Fills a possibly deep list of chars and binaries into vec +** Small characters are first stored in the buffer buf of length ln +** binaries found are copied and linked into msoh +** Return vector length on succsess, +** -1 on overflow +** -2 on type error +*/ + +static ERTS_INLINE void +io_list_to_vec_set_vec(SysIOVec **iov, ErtsIOQBinary ***binv, + ErtsIOQBinary *bin, byte *ptr, Uint len, + int *vlen) +{ + while (len > MAX_SYSIOVEC_IOVLEN) { + (*iov)->iov_base = ptr; + (*iov)->iov_len = MAX_SYSIOVEC_IOVLEN; + ptr += MAX_SYSIOVEC_IOVLEN; + len -= MAX_SYSIOVEC_IOVLEN; + (*iov)++; + (*vlen)++; + *(*binv)++ = bin; + } + (*iov)->iov_base = ptr; + (*iov)->iov_len = len; + *(*binv)++ = bin; + (*iov)++; + (*vlen)++; +} + +int +erts_ioq_iolist_to_vec(Eterm obj, /* io-list */ + SysIOVec* iov, /* io vector */ + ErtsIOQBinary** binv, /* binary reference vector */ + ErtsIOQBinary* cbin, /* binary to store characters */ + Uint bin_limit, /* small binaries limit */ + int driver) +{ + DECLARE_ESTACK(s); + Eterm* objp; + byte *buf = NULL; + Uint len = 0; + Uint csize = 0; + int vlen = 0; + byte* cptr; + + if (cbin) { + if (driver) { + buf = (byte*)cbin->driver.orig_bytes; + len = cbin->driver.orig_size; + } else { + buf = (byte*)cbin->nif.orig_bytes; + len = cbin->nif.orig_size; + } + } + cptr = buf; + + goto L_jump_start; /* avoid push */ + + while (!ESTACK_ISEMPTY(s)) { + obj = ESTACK_POP(s); + L_jump_start: + if (is_list(obj)) { + L_iter_list: + objp = list_val(obj); + obj = CAR(objp); + if (is_byte(obj)) { + if (len == 0) + goto L_overflow; + *buf++ = unsigned_val(obj); + csize++; + len--; + } else if (is_binary(obj)) { + ESTACK_PUSH(s, CDR(objp)); + goto handle_binary; + } else if (is_list(obj)) { + ESTACK_PUSH(s, CDR(objp)); + goto L_iter_list; /* on head */ + } else if (!is_nil(obj)) { + goto L_type_error; + } + obj = CDR(objp); + if (is_list(obj)) + goto L_iter_list; /* on tail */ + else if (is_binary(obj)) { + goto handle_binary; + } else if (!is_nil(obj)) { + goto L_type_error; + } + } else if (is_binary(obj)) { + Eterm real_bin; + Uint offset; + Eterm* bptr; + Uint size; + int bitoffs; + int bitsize; + + handle_binary: + size = binary_size(obj); + ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize); + ASSERT(bitsize == 0); + bptr = binary_val(real_bin); + if (*bptr == HEADER_PROC_BIN) { + ProcBin* pb = (ProcBin *) bptr; + if (bitoffs != 0) { + if (len < size) { + goto L_overflow; + } + erts_copy_bits(pb->bytes+offset, bitoffs, 1, + (byte *) buf, 0, 1, size*8); + csize += size; + buf += size; + len -= size; + } else if (bin_limit && size < bin_limit) { + if (len < size) { + goto L_overflow; + } + sys_memcpy(buf, pb->bytes+offset, size); + csize += size; + buf += size; + len -= size; + } else { + ErtsIOQBinary *qbin; + if (csize != 0) { + io_list_to_vec_set_vec(&iov, &binv, cbin, + cptr, csize, &vlen); + cptr = buf; + csize = 0; + } + if (pb->flags) { + erts_emasculate_writable_binary(pb); + } + if (driver) + qbin = (ErtsIOQBinary*)Binary2ErlDrvBinary(pb->val); + else + qbin = (ErtsIOQBinary*)pb->val; + + io_list_to_vec_set_vec( + &iov, &binv, qbin, + pb->bytes+offset, size, &vlen); + } + } else { + ErlHeapBin* hb = (ErlHeapBin *) bptr; + if (len < size) { + goto L_overflow; + } + copy_binary_to_buffer(buf, 0, + ((byte *) hb->data)+offset, bitoffs, + 8*size); + csize += size; + buf += size; + len -= size; + } + } else if (!is_nil(obj)) { + goto L_type_error; + } + } + + if (csize != 0) { + io_list_to_vec_set_vec(&iov, &binv, cbin, cptr, csize, &vlen); + } + + DESTROY_ESTACK(s); + return vlen; + + L_type_error: + DESTROY_ESTACK(s); + return -2; + + L_overflow: + DESTROY_ESTACK(s); + return -1; +} + +static ERTS_INLINE int +io_list_vec_count(Eterm obj, Uint *v_size, + Uint *c_size, Uint *b_size, Uint *in_clist, + Uint *p_v_size, Uint *p_c_size, Uint *p_in_clist, + Uint blimit) +{ + Uint size = binary_size(obj); + Eterm real; + ERTS_DECLARE_DUMMY(Uint offset); + int bitoffs; + int bitsize; + ERTS_GET_REAL_BIN(obj, real, offset, bitoffs, bitsize); + if (bitsize != 0) return 1; + if (thing_subtag(*binary_val(real)) == REFC_BINARY_SUBTAG && + bitoffs == 0) { + *b_size += size; + if (*b_size < size) return 2; + *in_clist = 0; + ++*v_size; + /* If iov_len is smaller then Uint we split the binary into*/ + /* multiple smaller (2GB) elements in the iolist.*/ + *v_size += size / MAX_SYSIOVEC_IOVLEN; + if (size >= blimit) { + *p_in_clist = 0; + ++*p_v_size; + } else { + *p_c_size += size; + if (!*p_in_clist) { + *p_in_clist = 1; + ++*p_v_size; + } + } + } else { + *c_size += size; + if (*c_size < size) return 2; + if (!*in_clist) { + *in_clist = 1; + ++*v_size; + } + *p_c_size += size; + if (!*p_in_clist) { + *p_in_clist = 1; + ++*p_v_size; + } + } + return 0; +} + +#define IO_LIST_VEC_COUNT(obj) \ + do { \ + switch (io_list_vec_count(obj, &v_size, &c_size, \ + &b_size, &in_clist, \ + &p_v_size, &p_c_size, &p_in_clist, \ + blimit)) { \ + case 1: goto L_type_error; \ + case 2: goto L_overflow_error; \ + default: break; \ + } \ + } while(0) + +/* + * Returns 0 if successful and a non-zero value otherwise. + * + * Return values through pointers: + * *vsize - SysIOVec size needed for a writev + * *csize - Number of bytes not in binary (in the common binary) + * *pvsize - SysIOVec size needed if packing small binaries + * *pcsize - Number of bytes in the common binary if packing + * *total_size - Total size of iolist in bytes + */ +int +erts_ioq_iolist_vec_len(Eterm obj, int* vsize, Uint* csize, + Uint* pvsize, Uint* pcsize, + Uint* total_size, Uint blimit) +{ + DECLARE_ESTACK(s); + Eterm* objp; + Uint v_size = 0; + Uint c_size = 0; + Uint b_size = 0; + Uint in_clist = 0; + Uint p_v_size = 0; + Uint p_c_size = 0; + Uint p_in_clist = 0; + Uint total; + + goto L_jump_start; /* avoid a push */ + + while (!ESTACK_ISEMPTY(s)) { + obj = ESTACK_POP(s); + L_jump_start: + if (is_list(obj)) { + L_iter_list: + objp = list_val(obj); + obj = CAR(objp); + + if (is_byte(obj)) { + c_size++; + if (c_size == 0) { + goto L_overflow_error; + } + if (!in_clist) { + in_clist = 1; + v_size++; + } + p_c_size++; + if (!p_in_clist) { + p_in_clist = 1; + p_v_size++; + } + } + else if (is_binary(obj)) { + IO_LIST_VEC_COUNT(obj); + } + else if (is_list(obj)) { + ESTACK_PUSH(s, CDR(objp)); + goto L_iter_list; /* on head */ + } + else if (!is_nil(obj)) { + goto L_type_error; + } + + obj = CDR(objp); + if (is_list(obj)) + goto L_iter_list; /* on tail */ + else if (is_binary(obj)) { /* binary tail is OK */ + IO_LIST_VEC_COUNT(obj); + } + else if (!is_nil(obj)) { + goto L_type_error; + } + } + else if (is_binary(obj)) { + IO_LIST_VEC_COUNT(obj); + } + else if (!is_nil(obj)) { + goto L_type_error; + } + } + + total = c_size + b_size; + if (total < c_size) { + goto L_overflow_error; + } + *total_size = total; + + DESTROY_ESTACK(s); + *vsize = v_size; + *csize = c_size; + *pvsize = p_v_size; + *pcsize = p_c_size; + return 0; + + L_type_error: + L_overflow_error: + DESTROY_ESTACK(s); + return 1; +} + +typedef struct { + Eterm result_head; + Eterm result_tail; + Eterm input_list; + + UWord acc_size; + Binary *acc; + + /* We yield after copying this many bytes into the accumulator (Minus + * eating a few on consing etc). Large binaries will only count to the + * extent their split (if any) resulted in a copy op. */ + UWord bytereds_available; + UWord bytereds_spent; + + Process *process; + ErtsEStack estack; + + Eterm magic_reference; +} iol2v_state_t; + +static int iol2v_state_destructor(Binary *data) { + iol2v_state_t *state = ERTS_MAGIC_BIN_UNALIGNED_DATA(data); + + DESTROY_SAVED_ESTACK(&state->estack); + + if (state->acc != NULL) { + erts_bin_free(state->acc); + } + + return 1; +} + +static void iol2v_init(iol2v_state_t *state, Process *process, Eterm input) { + state->process = process; + + state->result_head = NIL; + state->result_tail = NIL; + state->input_list = input; + + state->magic_reference = NIL; + state->acc_size = 0; + state->acc = NULL; + + CLEAR_SAVED_ESTACK(&state->estack); +} + +static Eterm iol2v_make_sub_bin(iol2v_state_t *state, Eterm bin_term, + UWord offset, UWord size) { + Uint byte_offset, bit_offset, bit_size; + ErlSubBin *sb; + Eterm orig_pb_term; + + sb = (ErlSubBin*)HAlloc(state->process, ERL_SUB_BIN_SIZE); + + ERTS_GET_REAL_BIN(bin_term, orig_pb_term, + byte_offset, bit_offset, bit_size); + + (void)bit_offset; + (void)bit_size; + + sb->thing_word = HEADER_SUB_BIN; + sb->bitsize = 0; + sb->bitoffs = 0; + sb->orig = orig_pb_term; + sb->is_writable = 0; + + sb->offs = byte_offset + offset; + sb->size = size; + + return make_binary(sb); +} + +static Eterm iol2v_promote_acc(iol2v_state_t *state) { + ProcBin *pb; + + state->acc = erts_bin_realloc(state->acc, state->acc_size); + + pb = (ProcBin*)HAlloc(state->process, PROC_BIN_SIZE); + pb->thing_word = HEADER_PROC_BIN; + pb->size = state->acc_size; + pb->val = state->acc; + pb->bytes = (byte*)(state->acc)->orig_bytes; + pb->flags = 0; + pb->next = MSO(state->process).first; + OH_OVERHEAD(&(MSO(state->process)), pb->size / sizeof(Eterm)); + MSO(state->process).first = (struct erl_off_heap_header*)pb; + + state->acc_size = 0; + state->acc = NULL; + + return make_binary(pb); +} + +/* Destructively enqueues a term to the result list, saving us the hassle of + * having to reverse it later. This is safe since GC is disabled and we never + * leak the unfinished term to the outside. */ +static void iol2v_enqueue_result(iol2v_state_t *state, Eterm term) { + Eterm prev_tail; + Eterm *hp; + + prev_tail = state->result_tail; + + hp = HAlloc(state->process, 2); + state->result_tail = CONS(hp, term, NIL); + + if(prev_tail != NIL) { + Eterm *prev_cell = list_val(prev_tail); + CDR(prev_cell) = state->result_tail; + } else { + state->result_head = state->result_tail; + } + + state->bytereds_spent += 1; +} + +#ifndef DEBUG + #define ACC_REALLOCATION_LIMIT (IOL2V_SMALL_BIN_LIMIT * 32) +#else + #define ACC_REALLOCATION_LIMIT (IOL2V_SMALL_BIN_LIMIT * 4) +#endif + +static void iol2v_expand_acc(iol2v_state_t *state, UWord extra) { + UWord required_bytes, acc_alloc_size; + + ERTS_CT_ASSERT(ERTS_UWORD_MAX > ACC_REALLOCATION_LIMIT / 2); + ASSERT(extra >= 1); + + acc_alloc_size = state->acc != NULL ? (state->acc)->orig_size : 0; + required_bytes = state->acc_size + extra; + + if (state->acc == NULL) { + UWord new_size = MAX(required_bytes, IOL2V_SMALL_BIN_LIMIT); + + state->acc = erts_bin_nrml_alloc(new_size); + } else if (required_bytes > acc_alloc_size) { + Binary *prev_acc; + UWord new_size; + + if (acc_alloc_size >= ACC_REALLOCATION_LIMIT) { + /* We skip reallocating once we hit a certain point; it often + * results in extra copying and we're very likely to overallocate + * on anything other than absurdly long byte/heapbin sequences. */ + iol2v_enqueue_result(state, iol2v_promote_acc(state)); + iol2v_expand_acc(state, extra); + return; + } + + new_size = MAX(required_bytes, acc_alloc_size * 2); + prev_acc = state->acc; + + state->acc = erts_bin_realloc(prev_acc, new_size); + + if (prev_acc != state->acc) { + state->bytereds_spent += state->acc_size; + } + } + + state->bytereds_spent += extra; +} + +static int iol2v_append_byte_seq(iol2v_state_t *state, Eterm seq_start, Eterm *seq_end) { + Eterm lookahead, iterator; + Uint observed_bits; + SWord seq_length; + char *acc_data; + + lookahead = seq_start; + seq_length = 0; + + ASSERT(state->bytereds_available > state->bytereds_spent); + + while (is_list(lookahead)) { + Eterm *cell = list_val(lookahead); + + if (!is_small(CAR(cell))) { + break; + } + + if (seq_length * 2 >= (state->bytereds_available - state->bytereds_spent)) { + break; + } + + lookahead = CDR(cell); + seq_length += 1; + } + + ASSERT(seq_length >= 1); + + iol2v_expand_acc(state, seq_length); + + /* Bump a few extra reductions to account for list traversal. */ + state->bytereds_spent += seq_length; + + acc_data = &(state->acc)->orig_bytes[state->acc_size]; + state->acc_size += seq_length; + + iterator = seq_start; + observed_bits = 0; + + while (iterator != lookahead) { + Eterm *cell; + Uint byte; + + cell = list_val(iterator); + iterator = CDR(cell); + + byte = unsigned_val(CAR(cell)); + observed_bits |= byte; + + ASSERT(acc_data < &(state->acc)->orig_bytes[state->acc_size]); + *(acc_data++) = byte; + } + + if (observed_bits > UCHAR_MAX) { + return 0; + } + + ASSERT(acc_data == &(state->acc)->orig_bytes[state->acc_size]); + *seq_end = iterator; + + return 1; +} + +static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) { + int is_acc_small, is_bin_small; + UWord combined_size; + UWord binary_size; + + Uint byte_offset, bit_offset, bit_size; + byte *binary_data; + + Eterm *parent_header; + Eterm parent_binary; + + ASSERT(state->bytereds_available > state->bytereds_spent); + + ERTS_GET_REAL_BIN(bin_term, parent_binary, byte_offset, bit_offset, bit_size); + parent_header = binary_val(parent_binary); + binary_size = binary_size(bin_term); + + if (bit_offset != 0 || bit_size != 0) { + return 0; + } else if (binary_size == 0) { + state->bytereds_spent += 1; + return 1; + } + + is_acc_small = state->acc_size < IOL2V_SMALL_BIN_LIMIT; + is_bin_small = binary_size < IOL2V_SMALL_BIN_LIMIT; + combined_size = binary_size + state->acc_size; + + if (thing_subtag(*parent_header) == REFC_BINARY_SUBTAG) { + ProcBin *pb = (ProcBin*)parent_header; + + if (pb->flags) { + erts_emasculate_writable_binary(pb); + } + + binary_data = &((byte*)pb->bytes)[byte_offset]; + } else { + ErlHeapBin *hb = (ErlHeapBin*)parent_header; + + ASSERT(thing_subtag(*parent_header) == HEAP_BINARY_SUBTAG); + ASSERT(is_bin_small); + + binary_data = &((byte*)&hb->data)[byte_offset]; + } + + if (!is_bin_small && (state->acc_size == 0 || !is_acc_small)) { + /* Avoid combining if we encounter an acceptably large binary while the + * accumulator is either empty or large enough to be returned on its + * own. */ + if (state->acc_size != 0) { + iol2v_enqueue_result(state, iol2v_promote_acc(state)); + } + + iol2v_enqueue_result(state, bin_term); + } else if (is_bin_small || combined_size < (IOL2V_SMALL_BIN_LIMIT * 2)) { + /* If the candidate is small or we can't split the combination in two, + * then just copy it into the accumulator. */ + iol2v_expand_acc(state, binary_size); + + sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], + binary_data, binary_size); + + state->acc_size += binary_size; + } else { + /* Otherwise, append enough data for the accumulator to be valid, and + * then return the rest as a sub-binary. */ + UWord spill = IOL2V_SMALL_BIN_LIMIT - state->acc_size; + Eterm binary_tail; + + iol2v_expand_acc(state, spill); + + sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], + binary_data, spill); + + state->acc_size += spill; + + binary_tail = iol2v_make_sub_bin(state, bin_term, spill, + binary_size - spill); + + iol2v_enqueue_result(state, iol2v_promote_acc(state)); + iol2v_enqueue_result(state, binary_tail); + } + + return 1; +} + +static BIF_RETTYPE iol2v_yield(iol2v_state_t *state) { + if (is_nil(state->magic_reference)) { + iol2v_state_t *boxed_state; + Binary *magic_binary; + Eterm *hp; + + magic_binary = erts_create_magic_binary_x(sizeof(*state), + &iol2v_state_destructor, ERTS_ALC_T_BINARY, 1); + + boxed_state = ERTS_MAGIC_BIN_UNALIGNED_DATA(magic_binary); + sys_memcpy(boxed_state, state, sizeof(*state)); + + hp = HAlloc(boxed_state->process, ERTS_MAGIC_REF_THING_SIZE); + boxed_state->magic_reference = + erts_mk_magic_ref(&hp, &MSO(boxed_state->process), magic_binary); + + state = boxed_state; + } + + ERTS_BIF_YIELD1(bif_export[BIF_iolist_to_iovec_1], + state->process, state->magic_reference); +} + +static BIF_RETTYPE iol2v_continue(iol2v_state_t *state) { + Eterm iterator; + + DECLARE_ESTACK(s); + ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + + state->bytereds_available = + ERTS_BIF_REDS_LEFT(state->process) * IOL2V_SMALL_BIN_LIMIT; + state->bytereds_spent = 0; + + if (state->estack.start) { + ESTACK_RESTORE(s, &state->estack); + } + + iterator = state->input_list; + + for(;;) { + if (state->bytereds_spent >= state->bytereds_available) { + ESTACK_SAVE(s, &state->estack); + state->input_list = iterator; + + return iol2v_yield(state); + } + + while (is_list(iterator)) { + Eterm *cell; + Eterm head; + + cell = list_val(iterator); + head = CAR(cell); + + if (is_binary(head)) { + if (!iol2v_append_binary(state, head)) { + goto l_badarg; + } + + iterator = CDR(cell); + } else if (is_small(head)) { + Eterm seq_end; + + if (!iol2v_append_byte_seq(state, iterator, &seq_end)) { + goto l_badarg; + } + + iterator = seq_end; + } else if (is_list(head) || is_nil(head)) { + Eterm tail = CDR(cell); + + if (!is_nil(tail)) { + ESTACK_PUSH(s, tail); + } + + state->bytereds_spent += 1; + iterator = head; + } else { + goto l_badarg; + } + + if (state->bytereds_spent >= state->bytereds_available) { + ESTACK_SAVE(s, &state->estack); + state->input_list = iterator; + + return iol2v_yield(state); + } + } + + if (is_binary(iterator)) { + if (!iol2v_append_binary(state, iterator)) { + goto l_badarg; + } + } else if (!is_nil(iterator)) { + goto l_badarg; + } + + if(ESTACK_ISEMPTY(s)) { + break; + } + + iterator = ESTACK_POP(s); + } + + if (state->acc_size != 0) { + iol2v_enqueue_result(state, iol2v_promote_acc(state)); + } + + BUMP_REDS(state->process, state->bytereds_spent / IOL2V_SMALL_BIN_LIMIT); + + CLEAR_SAVED_ESTACK(&state->estack); + DESTROY_ESTACK(s); + + BIF_RET(state->result_head); + +l_badarg: + CLEAR_SAVED_ESTACK(&state->estack); + DESTROY_ESTACK(s); + + if (state->acc != NULL) { + erts_bin_free(state->acc); + state->acc = NULL; + } + + BIF_ERROR(state->process, BADARG); +} + +HIPE_WRAPPER_BIF_DISABLE_GC(iolist_to_iovec, 1) + +BIF_RETTYPE iolist_to_iovec_1(BIF_ALIST_1) { + BIF_RETTYPE result; + + if (is_nil(BIF_ARG_1)) { + BIF_RET(NIL); + } else if (is_binary(BIF_ARG_1)) { + if (binary_size(BIF_ARG_1) != 0) { + Eterm *hp = HAlloc(BIF_P, 2); + + BIF_RET(CONS(hp, BIF_ARG_1, NIL)); + } else { + BIF_RET(NIL); + } + } else if (is_internal_magic_ref(BIF_ARG_1)) { + iol2v_state_t *state; + Binary *magic; + + magic = erts_magic_ref2bin(BIF_ARG_1); + + if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != &iol2v_state_destructor) { + ASSERT(!(BIF_P->flags & F_DISABLE_GC)); + BIF_ERROR(BIF_P, BADARG); + } + + ASSERT(BIF_P->flags & F_DISABLE_GC); + + state = ERTS_MAGIC_BIN_UNALIGNED_DATA(magic); + result = iol2v_continue(state); + } else if (!is_list(BIF_ARG_1)) { + ASSERT(!(BIF_P->flags & F_DISABLE_GC)); + BIF_ERROR(BIF_P, BADARG); + } else { + iol2v_state_t state; + + iol2v_init(&state, BIF_P, BIF_ARG_1); + + erts_set_gc_state(BIF_P, 0); + + result = iol2v_continue(&state); + } + + if (result != THE_NON_VALUE || BIF_P->freason != TRAP) { + erts_set_gc_state(BIF_P, 1); + } + + BIF_RET(result); +} diff --git a/erts/emulator/beam/erl_io_queue.h b/erts/emulator/beam/erl_io_queue.h new file mode 100644 index 0000000000..51abe99510 --- /dev/null +++ b/erts/emulator/beam/erl_io_queue.h @@ -0,0 +1,201 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2017. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: A queue used for storing binary data that should be + * passed to writev or similar functions. Used by both + * the nif and driver api. + * + * Author: Lukas Larsson + */ + +#ifndef ERL_IO_QUEUE_H__TYPES__ +#define ERL_IO_QUEUE_H__TYPES__ + +#define ERTS_BINARY_TYPES_ONLY__ +#include "erl_binary.h" +#undef ERTS_BINARY_TYPES_ONLY__ +#include "erl_nif.h" + +#ifdef DEBUG +#define MAX_SYSIOVEC_IOVLEN (1ull << (32 - 1)) +#else +#define MAX_SYSIOVEC_IOVLEN (1ull << (sizeof(((SysIOVec*)0)->iov_len) * 8 - 1)) +#endif + +#define ERTS_SMALL_IO_QUEUE 5 + +typedef union { + ErlDrvBinary driver; + Binary nif; +} ErtsIOQBinary; + +typedef struct { + int vsize; /* length of vectors */ + Uint size; /* total size in bytes */ + SysIOVec* iov; + ErtsIOQBinary** binv; +} ErtsIOVecCommon; + +typedef union { + ErtsIOVecCommon common; + ErlIOVec driver; + ErlNifIOVec nif; +} ErtsIOVec; + +/* head/tail represent the data in the queue + * start/end represent the edges of the allocated queue + * small is used when the number of iovec elements is < SMALL_IO_QUEUE + */ +typedef struct erts_io_queue { + ErtsAlcType_t alct; + int driver; + Uint size; /* total size in bytes */ + + SysIOVec* v_start; + SysIOVec* v_end; + SysIOVec* v_head; + SysIOVec* v_tail; + SysIOVec v_small[ERTS_SMALL_IO_QUEUE]; + + ErtsIOQBinary **b_start; + ErtsIOQBinary **b_end; + ErtsIOQBinary **b_head; + ErtsIOQBinary **b_tail; + ErtsIOQBinary *b_small[ERTS_SMALL_IO_QUEUE]; + +} ErtsIOQueue; + +#endif /* ERL_IO_QUEUE_H__TYPES__ */ + +#if !defined(ERL_IO_QUEUE_H) && !defined(ERTS_IO_QUEUE_TYPES_ONLY__) +#define ERL_IO_QUEUE_H + +#include "erl_binary.h" +#include "erl_bits.h" + +void erts_ioq_init(ErtsIOQueue *q, ErtsAlcType_t alct, int driver); +void erts_ioq_clear(ErtsIOQueue *q); +Uint erts_ioq_size(ErtsIOQueue *q); +int erts_ioq_enqv(ErtsIOQueue *q, ErtsIOVec *vec, Uint skip); +int erts_ioq_pushqv(ErtsIOQueue *q, ErtsIOVec *vec, Uint skip); +int erts_ioq_deq(ErtsIOQueue *q, Uint Uint); +Uint erts_ioq_peekqv(ErtsIOQueue *q, ErtsIOVec *ev); +SysIOVec *erts_ioq_peekq(ErtsIOQueue *q, int *vlenp); +Uint erts_ioq_sizeq(ErtsIOQueue *q); + +int erts_ioq_iolist_vec_len(Eterm obj, int* vsize, Uint* csize, + Uint* pvsize, Uint* pcsize, + Uint* total_size, Uint blimit); +int erts_ioq_iolist_to_vec(Eterm obj, SysIOVec* iov, + ErtsIOQBinary** binv, ErtsIOQBinary* cbin, + Uint bin_limit, int driver_binary); + +ERTS_GLB_INLINE +int erts_ioq_iodata_vec_len(Eterm obj, int* vsize, Uint* csize, + Uint* pvsize, Uint* pcsize, + Uint* total_size, Uint blimit); +ERTS_GLB_INLINE +int erts_ioq_iodata_to_vec(Eterm obj, SysIOVec* iov, + ErtsIOQBinary** binv, ErtsIOQBinary* cbin, + Uint bin_limit, int driver_binary); + + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE +int erts_ioq_iodata_vec_len(Eterm obj, int* vsize, Uint* csize, + Uint* pvsize, Uint* pcsize, + Uint* total_size, Uint blimit) { + if (is_binary(obj)) { + /* We optimize for when we get a procbin without a bit-offset + * that fits in one iov slot + */ + Eterm real_bin; + byte bitoffs; + byte bitsize; + ERTS_DECLARE_DUMMY(Uint offset); + Uint size = binary_size(obj); + ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize); + if (size < MAX_SYSIOVEC_IOVLEN && bitoffs == 0 && bitsize == 0) { + *vsize = 1; + *pvsize = 1; + if (thing_subtag(*binary_val(real_bin)) == REFC_BINARY_SUBTAG) { + *csize = 0; + *pcsize = 0; + } else { + *csize = size; + *pcsize = size; + } + *total_size = size; + return 0; + } + } + + return erts_ioq_iolist_vec_len(obj, vsize, csize, + pvsize, pcsize, total_size, blimit); +} + +ERTS_GLB_INLINE +int erts_ioq_iodata_to_vec(Eterm obj, + SysIOVec *iov, + ErtsIOQBinary **binv, + ErtsIOQBinary *cbin, + Uint bin_limit, + int driver) +{ + if (is_binary(obj)) { + Eterm real_bin; + byte bitoffs; + byte bitsize; + Uint offset; + Uint size = binary_size(obj); + ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize); + if (size < MAX_SYSIOVEC_IOVLEN && bitoffs == 0 && bitsize == 0) { + Eterm *bptr = binary_val(real_bin); + if (thing_subtag(*bptr) == REFC_BINARY_SUBTAG) { + ProcBin *pb = (ProcBin *)bptr; + if (pb->flags) + erts_emasculate_writable_binary(pb); + iov[0].iov_base = pb->bytes+offset; + iov[0].iov_len = size; + if (driver) + binv[0] = (ErtsIOQBinary*)Binary2ErlDrvBinary(pb->val); + else + binv[0] = (ErtsIOQBinary*)pb->val; + return 1; + } else { + ErlHeapBin* hb = (ErlHeapBin *)bptr; + byte *buf = driver ? (byte*)cbin->driver.orig_bytes : + (byte*)cbin->nif.orig_bytes; + copy_binary_to_buffer(buf, 0, ((byte *) hb->data)+offset, 0, 8*size); + iov[0].iov_base = buf; + iov[0].iov_len = size; + binv[0] = cbin; + return 1; + } + } + } + return erts_ioq_iolist_to_vec(obj, iov, binv, cbin, bin_limit, driver); +} + +#endif + +#endif /* ERL_IO_QUEUE_H */ diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index b68b48874d..4cdef0200f 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -113,17 +113,14 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "drv_ev_state_grow", NULL, }, { "drv_ev_state", "address" }, { "safe_hash", "address" }, - { "pollset_rm_list", NULL }, { "removed_fd_pre_alloc_lock", "address" }, { "state_prealloc", NULL }, { "schdlr_sspnd", NULL }, { "migration_info_update", NULL }, { "run_queue", "address" }, -#ifdef ERTS_DIRTY_SCHEDULERS { "dirty_run_queue_sleep_list", "address" }, { "dirty_gc_info", NULL }, { "dirty_break_point_index", NULL }, -#endif { "process_table", NULL }, { "cpu_info", NULL }, { "pollset", "address" }, @@ -160,9 +157,9 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "xports_list_pre_alloc_lock", "address" }, { "inet_buffer_stack_lock", NULL }, { "system_block", NULL }, - { "timeofday", NULL }, { "get_time", NULL }, { "get_corrected_time", NULL }, + { "runtime", NULL }, { "breakpoints", NULL }, { "pix_lock", "address" }, { "run_queues_lists", NULL }, diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index d2e8f47d59..1ae6076b12 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -554,16 +554,6 @@ erts_lock_flags_t erts_lcnt_get_category_mask() { return lcnt_category_mask__; } -#ifdef ERTS_ENABLE_KERNEL_POLL -/* erl_poll/erl_check_io only exports one of these variants at a time, and we - * may need to use either one depending on emulator startup flags. */ -void erts_lcnt_update_pollset_locks_nkp(int); -void erts_lcnt_update_pollset_locks_kp(int); - -void erts_lcnt_update_cio_locks_nkp(int); -void erts_lcnt_update_cio_locks_kp(int); -#endif - void erts_lcnt_set_category_mask(erts_lock_flags_t mask) { erts_lock_flags_t changed_categories; @@ -590,19 +580,7 @@ void erts_lcnt_set_category_mask(erts_lock_flags_t mask) { } if(changed_categories & ERTS_LOCK_FLAGS_CATEGORY_IO) { -#ifdef ERTS_ENABLE_KERNEL_POLL - if(erts_use_kernel_poll) { - erts_lcnt_update_pollset_locks_kp(mask & ERTS_LOCK_FLAGS_CATEGORY_IO); - erts_lcnt_update_cio_locks_kp(mask & ERTS_LOCK_FLAGS_CATEGORY_IO); - } else { - erts_lcnt_update_pollset_locks_nkp(mask & ERTS_LOCK_FLAGS_CATEGORY_IO); - erts_lcnt_update_cio_locks_nkp(mask & ERTS_LOCK_FLAGS_CATEGORY_IO); - } -#else - erts_lcnt_update_pollset_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_IO); erts_lcnt_update_cio_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_IO); -#endif - erts_lcnt_update_driver_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_IO); erts_lcnt_update_port_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_IO); } diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 3418a7f4df..abf194cf94 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -550,14 +550,11 @@ erts_msg_attached_data_size_aux(ErtsMessage *msg) sz = erts_decode_dist_ext_size(msg->data.dist_ext); if (sz < 0) { - /* Bad external; remove it */ - if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) { - ErlHeapFragment *heap_frag; - heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); - erts_cleanup_offheap(&heap_frag->off_heap); - } - erts_free_dist_ext_copy(msg->data.dist_ext); - msg->data.dist_ext = NULL; + /* Bad external + * We leave the message intact in this case as it's not worth the trouble + * to make all callers remove it from queue. It will be detected again + * and removed from message queue later anyway. + */ return 0; } diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c index 67c552b364..1c840d89f6 100644 --- a/erts/emulator/beam/erl_monitors.c +++ b/erts/emulator/beam/erl_monitors.c @@ -993,7 +993,6 @@ Eterm erts_debug_dump_monitors_1(BIF_ALIST_1) erts_dump_monitors(dep->monitors,0); erts_de_links_unlock(dep); erts_printf("Monitors dumped-------------------------\n"); - erts_deref_dist_entry(dep); BIF_RET(am_true); } else { BIF_ERROR(p,BADARG); @@ -1038,7 +1037,6 @@ Eterm erts_debug_dump_links_1(BIF_ALIST_1) erts_dump_links(dep->nlinks,0); erts_de_links_unlock(dep); erts_printf("Links dumped----------------------------\n"); - erts_deref_dist_entry(dep); BIF_RET(am_true); } else { BIF_ERROR(p,BADARG); diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h index 8349a7e297..2d4637f800 100644 --- a/erts/emulator/beam/erl_msacc.h +++ b/erts/emulator/beam/erl_msacc.h @@ -318,8 +318,8 @@ ERTS_GLB_INLINE void erts_msacc_set_state_um__(ErtsMsAcc *msacc, Uint new_state, int increment) { if (ERTS_UNLIKELY(msacc->unmanaged)) { erts_mtx_lock(&msacc->mtx); - msacc->state = new_state; if (ERTS_LIKELY(!msacc->perf_counter)) { + msacc->state = new_state; erts_mtx_unlock(&msacc->mtx); return; } diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h index 69008084df..b8a4e4ebc3 100644 --- a/erts/emulator/beam/erl_nfunc_sched.h +++ b/erts/emulator/beam/erl_nfunc_sched.h @@ -193,7 +193,6 @@ erts_nif_export_check_save_trace(Process *c_p, Eterm result, ERTS_GLB_INLINE Process * erts_proc_shadow2real(Process *c_p) { -#ifdef ERTS_DIRTY_SCHEDULERS if (c_p->static_flags & ERTS_STC_FLG_SHADOW_PROC) { Process *real_c_p = c_p->next; ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); @@ -201,7 +200,6 @@ erts_proc_shadow2real(Process *c_p) return real_c_p; } ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); -#endif return c_p; } @@ -213,11 +211,10 @@ erts_proc_shadow2real(Process *c_p) #define ERTS_NFUNC_SCHED_INTERNALS__ #define ERTS_I_BEAM_OP_TO_NIF_EXPORT(I) \ - (ASSERT(BeamOp(op_apply_bif) == (BeamInstr *) (*(I)) \ - || BeamOp(op_call_nif) == (BeamInstr *) (*(I))), \ + (ASSERT(BeamIsOpCode(*(I), op_apply_bif) || \ + BeamIsOpCode(*(I), op_call_nif)), \ ((NifExport *) (((char *) (I)) - offsetof(NifExport, exp.beam[0])))) -#ifdef ERTS_DIRTY_SCHEDULERS #include "erl_message.h" #include <stddef.h> @@ -326,7 +323,6 @@ erts_make_dirty_shadow_proc(ErtsSchedulerData *esdp, Process *c_p) #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ -#endif /* ERTS_DIRTY_SCHEDULERS */ #endif /* defined(ERTS_WANT_NFUNC_SCHED_INTERNALS__) && !defined(ERTS_NFUNC_SCHED_INTERNALS__) */ diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 05bcf73451..f7f12efe28 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -56,6 +56,7 @@ #include "erl_process.h" #include "erl_bif_unique.h" #include "erl_utils.h" +#include "erl_io_queue.h" #undef ERTS_WANT_NFUNC_SCHED_INTERNALS__ #define ERTS_WANT_NFUNC_SCHED_INTERNALS__ #include "erl_nfunc_sched.h" @@ -66,7 +67,6 @@ #include <limits.h> #include <stddef.h> /* offsetof */ - /* Information about a loaded nif library. * Each successful call to erlang:load_nif will allocate an instance of * erl_module_nif. Two calls opening the same library will thus have the same @@ -292,7 +292,7 @@ schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp, ep = erts_nif_export_schedule(c_p, dirty_shadow_proc, c_p->current, c_p->cp, - (BeamInstr) em_call_nif, + BeamOpCodeAddr(op_call_nif), direct_fp, indirect_fp, mod, func_name, argc, (const Eterm *) argv); @@ -304,7 +304,6 @@ schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp, return (ERL_NIF_TERM) THE_NON_VALUE; } -#ifdef ERTS_DIRTY_SCHEDULERS static ERL_NIF_TERM dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); @@ -394,24 +393,19 @@ erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * return exiting; } -#endif static void full_flush_env(ErlNifEnv* env) { flush_env(env); -#ifdef ERTS_DIRTY_SCHEDULERS if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) /* Dirty nif call using shadow process struct */ erts_flush_dirty_shadow_proc(env->proc); -#endif } static void full_cache_env(ErlNifEnv* env) { -#ifdef ERTS_DIRTY_SCHEDULERS if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) erts_cache_dirty_shadow_proc(env->proc); -#endif cache_env(env); } @@ -2591,7 +2585,6 @@ nif_export_restore(Process *c_p, NifExport *ep, Eterm res) } -#ifdef ERTS_DIRTY_SCHEDULERS /* * Finalize a dirty NIF call. This function is scheduled to cause the VM to @@ -2719,7 +2712,6 @@ static_schedule_dirty_cpu_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ return static_schedule_dirty_nif(env, ERTS_PSFLG_DIRTY_CPU_PROC, argc, argv); } -#endif /* ERTS_DIRTY_SCHEDULERS */ /* * NIF execution wrapper used by enif_schedule_nif() for regular NIFs. It @@ -2800,11 +2792,7 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, result = schedule(env, execute_nif, fp, proc->current->module, fun_name_atom, argc, argv); else if (!(flags & ~(ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND))) { -#ifdef ERTS_DIRTY_SCHEDULERS result = schedule_dirty_nif(env, flags, fp, fun_name_atom, argc, argv); -#else - result = enif_raise_exception(env, am_notsup); -#endif } else result = enif_make_badarg(env); @@ -2826,12 +2814,10 @@ enif_thread_type(void) switch (esdp->type) { case ERTS_SCHED_NORMAL: return ERL_NIF_THR_NORMAL_SCHEDULER; -#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: return ERL_NIF_THR_DIRTY_CPU_SCHEDULER; case ERTS_SCHED_DIRTY_IO: return ERL_NIF_THR_DIRTY_IO_SCHEDULER; -#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); return -1; @@ -3259,6 +3245,363 @@ int enif_compare_monitors(const ErlNifMonitor *monitor1, ERTS_REF_THING_SIZE*sizeof(Eterm)); } +ErlNifIOQueue *enif_ioq_create(ErlNifIOQueueOpts opts) +{ + ErlNifIOQueue *q; + + if (opts != ERL_NIF_IOQ_NORMAL) + return NULL; + + q = enif_alloc(sizeof(ErlNifIOQueue)); + if (!q) return NULL; + erts_ioq_init(q, ERTS_ALC_T_NIF, 0); + + return q; +} + +void enif_ioq_destroy(ErlNifIOQueue *q) +{ + erts_ioq_clear(q); + enif_free(q); +} + +/* If the iovec was preallocated (Stack or otherwise) it needs to be marked as + * such to perform a proper free. */ +#define ERL_NIF_IOVEC_FLAGS_PREALLOC (1 << 0) + +void enif_free_iovec(ErlNifIOVec *iov) +{ + int i; + /* Decrement the refc of all the binaries */ + for (i = 0; i < iov->iovcnt; i++) { + Binary *bptr = ((Binary**)iov->ref_bins)[i]; + /* bptr can be null if enq_binary was used */ + if (bptr && erts_refc_dectest(&bptr->intern.refc, 0) == 0) { + erts_bin_free(bptr); + } + } + + if (!(iov->flags & ERL_NIF_IOVEC_FLAGS_PREALLOC)) { + enif_free(iov); + } +} + +typedef struct { + UWord sublist_length; + Eterm sublist_start; + Eterm sublist_end; + + UWord offheap_size; + UWord onheap_size; + + UWord iovec_len; +} iovec_slice_t; + +static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *result) { + Eterm lookahead; + + result->sublist_start = list; + result->sublist_length = 0; + result->offheap_size = 0; + result->onheap_size = 0; + result->iovec_len = 0; + + lookahead = result->sublist_start; + + while (is_list(lookahead)) { + Eterm *binary_header, binary; + Eterm *cell; + UWord size; + + cell = list_val(lookahead); + binary = CAR(cell); + + if (!is_binary(binary)) { + return 0; + } + + size = binary_size(binary); + binary_header = binary_val(binary); + + /* If we're a sub-binary we'll need to check our underlying binary to + * determine whether we're on-heap or not. */ + if(thing_subtag(*binary_header) == SUB_BINARY_SUBTAG) { + ErlSubBin *sb = (ErlSubBin*)binary_header; + + /* Reject bitstrings */ + if((sb->bitoffs + sb->bitsize) > 0) { + return 0; + } + + ASSERT(size <= binary_size(sb->orig)); + binary_header = binary_val(sb->orig); + } + + if(thing_subtag(*binary_header) == HEAP_BINARY_SUBTAG) { + ASSERT(size <= ERL_ONHEAP_BIN_LIMIT); + + result->iovec_len += 1; + result->onheap_size += size; + } else { + ASSERT(thing_subtag(*binary_header) == REFC_BINARY_SUBTAG); + + result->iovec_len += 1 + size / MAX_SYSIOVEC_IOVLEN; + result->offheap_size += size; + } + + result->sublist_length += 1; + lookahead = CDR(cell); + + if(result->sublist_length >= max_length) { + break; + } + } + + if (!is_nil(lookahead) && !is_list(lookahead)) { + return 0; + } + + result->sublist_end = lookahead; + + return 1; +} + +static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) { + Eterm *parent_header; + Eterm parent_binary; + + int bit_offset, bit_size; + Uint byte_offset; + + ASSERT(is_binary(binary)); + + ERTS_GET_REAL_BIN(binary, parent_binary, byte_offset, bit_offset, bit_size); + + parent_header = binary_val(parent_binary); + + result->size = binary_size(binary); + result->bin_term = binary; + + if (thing_subtag(*parent_header) == REFC_BINARY_SUBTAG) { + ProcBin *pb = (ProcBin*)parent_header; + + ASSERT(pb->val != NULL); + ASSERT(byte_offset < pb->size); + ASSERT(&pb->bytes[byte_offset] >= (byte*)(pb->val)->orig_bytes); + + result->data = (unsigned char*)&pb->bytes[byte_offset]; + result->ref_bin = (void*)pb->val; + } else { + ErlHeapBin *hb = (ErlHeapBin*)parent_header; + + ASSERT(thing_subtag(*parent_header) == HEAP_BINARY_SUBTAG); + + result->data = &((unsigned char*)&hb->data)[byte_offset]; + result->ref_bin = NULL; + } +} + +static int fill_iovec_with_slice(ErlNifEnv *env, + iovec_slice_t *slice, + ErlNifIOVec *iovec) { + UWord onheap_offset, iovec_idx; + ErlNifBinary onheap_data; + Eterm sublist_iterator; + + /* Set up a common refc binary for all on-heap binaries. */ + if (slice->onheap_size > 0) { + if (!enif_alloc_binary(slice->onheap_size, &onheap_data)) { + return 0; + } + } + + sublist_iterator = slice->sublist_start; + onheap_offset = 0; + iovec_idx = 0; + + while (sublist_iterator != slice->sublist_end) { + ErlNifBinary raw_data; + Eterm *cell; + + cell = list_val(sublist_iterator); + inspect_raw_binary_data(CAR(cell), &raw_data); + + /* If this isn't a refc binary, copy its contents to the onheap buffer + * and reference that instead. */ + if (raw_data.ref_bin == NULL) { + ASSERT(onheap_offset < onheap_data.size); + ASSERT(slice->onheap_size > 0); + + sys_memcpy(&onheap_data.data[onheap_offset], + raw_data.data, raw_data.size); + + raw_data.data = &onheap_data.data[onheap_offset]; + raw_data.ref_bin = onheap_data.ref_bin; + } + + ASSERT(raw_data.ref_bin != NULL); + + while (raw_data.size > 0) { + UWord chunk_len = MIN(raw_data.size, MAX_SYSIOVEC_IOVLEN); + + ASSERT(iovec_idx < iovec->iovcnt); + + iovec->iov[iovec_idx].iov_base = raw_data.data; + iovec->iov[iovec_idx].iov_len = chunk_len; + + iovec->ref_bins[iovec_idx] = raw_data.ref_bin; + + raw_data.data += chunk_len; + raw_data.size -= chunk_len; + + iovec_idx += 1; + } + + sublist_iterator = CDR(cell); + } + + ASSERT(iovec_idx == iovec->iovcnt); + + if (env == NULL) { + int i; + for (i = 0; i < iovec->iovcnt; i++) { + Binary *refc_binary = (Binary*)(iovec->ref_bins[i]); + erts_refc_inc(&refc_binary->intern.refc, 1); + } + + if (slice->onheap_size > 0) { + /* Transfer ownership to the iovec; we've taken references to it in + * the above loop. */ + enif_release_binary(&onheap_data); + } + } else { + if (slice->onheap_size > 0) { + /* Attach the binary to our environment and let the GC take care of + * it after returning. */ + enif_make_binary(env, &onheap_data); + } + } + + return 1; +} + +static int create_iovec_from_slice(ErlNifEnv *env, + iovec_slice_t *slice, + ErlNifIOVec **result) { + ErlNifIOVec *iovec = *result; + + if (iovec && slice->iovec_len < ERL_NIF_IOVEC_SIZE) { + iovec->iov = iovec->small_iov; + iovec->ref_bins = iovec->small_ref_bin; + iovec->flags = ERL_NIF_IOVEC_FLAGS_PREALLOC; + } else { + UWord iov_offset, binv_offset, alloc_size; + char *alloc_base; + + iov_offset = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErlNifIOVec)); + binv_offset = iov_offset; + binv_offset += ERTS_ALC_DATA_ALIGN_SIZE(slice->iovec_len * sizeof(SysIOVec)); + alloc_size = binv_offset; + alloc_size += slice->iovec_len * sizeof(Binary*); + + /* If we have an environment we'll attach the allocated data to it. The + * GC will take care of releasing it later on. */ + if (env != NULL) { + ErlNifBinary gc_bin; + + if (!enif_alloc_binary(alloc_size, &gc_bin)) { + return 0; + } + + alloc_base = (char*)gc_bin.data; + enif_make_binary(env, &gc_bin); + } else { + alloc_base = enif_alloc(alloc_size); + } + + iovec = (ErlNifIOVec*)alloc_base; + iovec->iov = (SysIOVec*)(alloc_base + iov_offset); + iovec->ref_bins = (void**)(alloc_base + binv_offset); + iovec->flags = 0; + } + + iovec->size = slice->offheap_size + slice->onheap_size; + iovec->iovcnt = slice->iovec_len; + + if(!fill_iovec_with_slice(env, slice, iovec)) { + if (env == NULL && !(iovec->flags & ERL_NIF_IOVEC_FLAGS_PREALLOC)) { + enif_free(iovec); + } + + return 0; + } + + *result = iovec; + + return 1; +} + +int enif_inspect_iovec(ErlNifEnv *env, size_t max_elements, + ERL_NIF_TERM list, ERL_NIF_TERM *tail, + ErlNifIOVec **iov) { + iovec_slice_t slice; + + if(!examine_iovec_term(list, max_elements, &slice)) { + return 0; + } else if(!create_iovec_from_slice(env, &slice, iov)) { + return 0; + } + + (*tail) = slice.sublist_end; + + return 1; +} + +/* */ +int enif_ioq_enqv(ErlNifIOQueue *q, ErlNifIOVec *iov, size_t skip) +{ + if(skip <= iov->size) { + return !erts_ioq_enqv(q, (ErtsIOVec*)iov, skip); + } + + return 0; +} + +int enif_ioq_enq_binary(ErlNifIOQueue *q, ErlNifBinary *bin, size_t skip) +{ + ErlNifIOVec vec = {1, bin->size, NULL, NULL, ERL_NIF_IOVEC_FLAGS_PREALLOC }; + Binary *ref_bin = (Binary*)bin->ref_bin; + int res; + vec.iov = vec.small_iov; + vec.ref_bins = vec.small_ref_bin; + vec.iov[0].iov_base = bin->data; + vec.iov[0].iov_len = bin->size; + ((Binary**)(vec.ref_bins))[0] = ref_bin; + + res = enif_ioq_enqv(q, &vec, skip); + enif_release_binary(bin); + return res; +} + +size_t enif_ioq_size(ErlNifIOQueue *q) +{ + return erts_ioq_size(q); +} + +int enif_ioq_deq(ErlNifIOQueue *q, size_t elems, size_t *size) +{ + if (erts_ioq_deq(q, elems) == -1) + return 0; + if (size) + *size = erts_ioq_size(q); + return 1; +} + +SysIOVec *enif_ioq_peek(ErlNifIOQueue *q, int *iovlen) +{ + return erts_ioq_peekq(q, iovlen); +} + /*************************************************************************** ** load_nif/2 ** ***************************************************************************/ @@ -3270,7 +3613,7 @@ static ErtsCodeInfo** get_func_pp(BeamCodeHeader* mod_code, Eterm f_atom, unsign int j; for (j = 0; j < n; ++j) { ErtsCodeInfo* ci = mod_code->functions[j]; - ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI)); if (f_atom == ci->mfa.function && arity == ci->mfa.arity) { return mod_code->functions+j; @@ -3572,14 +3915,9 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) * dirty scheduler support, treat a non-zero flags field as * a load error. */ -#ifdef ERTS_DIRTY_SCHEDULERS if (f->flags != ERL_NIF_DIRTY_JOB_IO_BOUND && f->flags != ERL_NIF_DIRTY_JOB_CPU_BOUND) ret = load_nif_error(BIF_P, bad_lib, "Illegal flags field value %d for NIF %T:%s/%u", f->flags, mod_atom, f->name, f->arity); -#else - ret = load_nif_error(BIF_P, bad_lib, "NIF %T:%s/%u requires a runtime with dirty scheduler support.", - mod_atom, f->name, f->arity); -#endif } else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) < BEAM_NIF_MIN_FUNC_SZ) @@ -3644,15 +3982,13 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) code_ptr = erts_codeinfo_to_code(ci); if (ci->u.gen_bp == NULL) { - code_ptr[0] = (BeamInstr) BeamOp(op_call_nif); + code_ptr[0] = BeamOpCodeAddr(op_call_nif); } else { /* Function traced, patch the original instruction word */ GenericBp* g = ci->u.gen_bp; - ASSERT(code_ptr[0] == - (BeamInstr) BeamOp(op_i_generic_breakpoint)); - g->orig_instr = (BeamInstr) BeamOp(op_call_nif); + ASSERT(BeamIsOpCode(code_ptr[0], op_i_generic_breakpoint)); + g->orig_instr = BeamOpCodeAddr(op_call_nif); } -#ifdef ERTS_DIRTY_SCHEDULERS if (f->flags) { code_ptr[3] = (BeamInstr) f->fptr; code_ptr[1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ? @@ -3660,7 +3996,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) (BeamInstr) static_schedule_dirty_cpu_nif; } else -#endif code_ptr[1] = (BeamInstr) f->fptr; code_ptr[2] = (BeamInstr) lib; } diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index b0d5c39798..d195721054 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -50,6 +50,7 @@ ** 2.9: 18.2 enif_getenv ** 2.10: Time API ** 2.11: 19.0 enif_snprintf +** 2.12: 20.0 add enif_queue */ #define ERL_NIF_MAJOR_VERSION 2 #define ERL_NIF_MINOR_VERSION 12 @@ -241,6 +242,28 @@ typedef enum { ERL_NIF_PHASH2 = 2 } ErlNifHash; +#define ERL_NIF_IOVEC_SIZE 16 + +typedef struct erl_nif_io_vec { + int iovcnt; /* length of vectors */ + size_t size; /* total size in bytes */ + SysIOVec *iov; + + /* internals (avert your eyes) */ + void **ref_bins; /* Binary[] */ + int flags; + + /* Used when stack allocating the io vec */ + SysIOVec small_iov[ERL_NIF_IOVEC_SIZE]; + void *small_ref_bin[ERL_NIF_IOVEC_SIZE]; +} ErlNifIOVec; + +typedef struct erts_io_queue ErlNifIOQueue; + +typedef enum { + ERL_NIF_IOQ_NORMAL = 1 +} ErlNifIOQueueOpts; + /* * Return values from enif_thread_type(). Negative values * reserved for specific types of non-scheduler threads. diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index 94c04cd126..9e573307d8 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -184,6 +184,21 @@ ERL_NIF_API_FUNC_DECL(ErlNifUInt64,enif_hash,(ErlNifHash type, ERL_NIF_TERM term ERL_NIF_API_FUNC_DECL(int, enif_whereis_pid, (ErlNifEnv *env, ERL_NIF_TERM name, ErlNifPid *pid)); ERL_NIF_API_FUNC_DECL(int, enif_whereis_port, (ErlNifEnv *env, ERL_NIF_TERM name, ErlNifPort *port)); +ERL_NIF_API_FUNC_DECL(ErlNifIOQueue *,enif_ioq_create,(ErlNifIOQueueOpts opts)); +ERL_NIF_API_FUNC_DECL(void,enif_ioq_destroy,(ErlNifIOQueue *q)); + +ERL_NIF_API_FUNC_DECL(int,enif_ioq_enq_binary,(ErlNifIOQueue *q, ErlNifBinary *bin, size_t skip)); +ERL_NIF_API_FUNC_DECL(int,enif_ioq_enqv,(ErlNifIOQueue *q, ErlNifIOVec *iov, size_t skip)); + +ERL_NIF_API_FUNC_DECL(size_t,enif_ioq_size,(ErlNifIOQueue *q)); +ERL_NIF_API_FUNC_DECL(int,enif_ioq_deq,(ErlNifIOQueue *q, size_t count, size_t *size)); + +ERL_NIF_API_FUNC_DECL(SysIOVec*,enif_ioq_peek,(ErlNifIOQueue *q, int *iovlen)); + +ERL_NIF_API_FUNC_DECL(int,enif_inspect_iovec,(ErlNifEnv *env, size_t max_length, ERL_NIF_TERM iovec_term, ERL_NIF_TERM *tail, ErlNifIOVec **iovec)); +ERL_NIF_API_FUNC_DECL(void,enif_free_iovec,(ErlNifIOVec *iov)); + + /* ** ADD NEW ENTRIES HERE (before this comment) !!! */ @@ -348,6 +363,16 @@ ERL_NIF_API_FUNC_DECL(int, enif_whereis_port, (ErlNifEnv *env, ERL_NIF_TERM name # define enif_hash ERL_NIF_API_FUNC_MACRO(enif_hash) # define enif_whereis_pid ERL_NIF_API_FUNC_MACRO(enif_whereis_pid) # define enif_whereis_port ERL_NIF_API_FUNC_MACRO(enif_whereis_port) +# define enif_ioq_create ERL_NIF_API_FUNC_MACRO(enif_ioq_create) +# define enif_ioq_destroy ERL_NIF_API_FUNC_MACRO(enif_ioq_destroy) +# define enif_ioq_enq ERL_NIF_API_FUNC_MACRO(enif_ioq_enq) +# define enif_ioq_enq_binary ERL_NIF_API_FUNC_MACRO(enif_ioq_enq_binary) +# define enif_ioq_enqv ERL_NIF_API_FUNC_MACRO(enif_ioq_enqv) +# define enif_ioq_size ERL_NIF_API_FUNC_MACRO(enif_ioq_size) +# define enif_ioq_deq ERL_NIF_API_FUNC_MACRO(enif_ioq_deq) +# define enif_ioq_peek ERL_NIF_API_FUNC_MACRO(enif_ioq_peek) +# define enif_inspect_iovec ERL_NIF_API_FUNC_MACRO(enif_inspect_iovec) +# define enif_free_iovec ERL_NIF_API_FUNC_MACRO(enif_free_iovec) /* ** ADD NEW ENTRIES HERE (before this comment) diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index f8e9fec27a..0f3dfa797c 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -29,6 +29,8 @@ #include "error.h" #include "erl_thr_progress.h" #include "dtrace-wrapper.h" +#include "erl_binary.h" +#include "erl_bif_unique.h" Hash erts_dist_table; Hash erts_node_table; @@ -57,6 +59,58 @@ static ErtsMonotonicTime node_tab_delete_delay; /* -- The distribution table ---------------------------------------------- */ +#define ErtsBin2DistEntry(B) \ + ((DistEntry *) ERTS_MAGIC_BIN_DATA((B))) +#define ErtsDistEntry2Bin(DEP) \ + ((Binary *) ERTS_MAGIC_BIN_FROM_DATA((DEP))) + +static ERTS_INLINE erts_aint_t +de_refc_read(DistEntry *dep, erts_aint_t min) +{ + return erts_refc_read(&ErtsDistEntry2Bin(dep)->intern.refc, min); +} + +static ERTS_INLINE erts_aint_t +de_refc_inc_read(DistEntry *dep, erts_aint_t min) +{ + return erts_refc_inctest(&ErtsDistEntry2Bin(dep)->intern.refc, min); +} + +static ERTS_INLINE void +de_refc_inc(DistEntry *dep, erts_aint_t min) +{ + erts_refc_inc(&ErtsDistEntry2Bin(dep)->intern.refc, min); +} + +static ERTS_INLINE void +de_refc_dec(DistEntry *dep, erts_aint_t min) +{ +#ifdef DEBUG + (void) erts_refc_read(&ErtsDistEntry2Bin(dep)->intern.refc, min+1); +#endif + erts_bin_release(ErtsDistEntry2Bin(dep)); +} + +static ERTS_INLINE erts_aint_t +de_refc_dec_read(DistEntry *dep, erts_aint_t min) +{ + return erts_refc_dectest(&ErtsDistEntry2Bin(dep)->intern.refc, min); +} + +void +erts_ref_dist_entry(DistEntry *dep) +{ + ASSERT(dep); + de_refc_inc(dep, 1); +} + +void +erts_deref_dist_entry(DistEntry *dep) +{ + ASSERT(dep); + de_refc_dec(dep, 0); +} + #ifdef DEBUG static int is_in_de_list(DistEntry *dep, DistEntry *dep_list) @@ -85,22 +139,39 @@ dist_table_cmp(void *dep1, void *dep2) static void* dist_table_alloc(void *dep_tmpl) { +#ifdef DEBUG + erts_aint_t refc; +#endif Eterm sysname; + Binary *bin; DistEntry *dep; erts_rwmtx_opt_t rwmtx_opt = ERTS_RWMTX_OPT_DEFAULT_INITER; rwmtx_opt.type = ERTS_RWMTX_TYPE_FREQUENT_READ; sysname = ((DistEntry *) dep_tmpl)->sysname; - dep = (DistEntry *) erts_alloc(ERTS_ALC_T_DIST_ENTRY, sizeof(DistEntry)); + + bin = erts_create_magic_binary_x(sizeof(DistEntry), + erts_dist_entry_destructor, + ERTS_ALC_T_DIST_ENTRY, + 0); + dep = ErtsBin2DistEntry(bin); dist_entries++; +#ifdef DEBUG + refc = +#else + (void) +#endif + de_refc_dec_read(dep, -1); + ASSERT(refc == -1); + dep->prev = NULL; - erts_refc_init(&dep->refc, -1); erts_rwmtx_init_opt(&dep->rwmtx, &rwmtx_opt, "dist_entry", sysname, ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); dep->sysname = sysname; dep->cid = NIL; + erts_atomic_init_nob(&dep->input_handler, (erts_aint_t) NIL); dep->connection_id = 0; dep->status = 0; dep->flags = 0; @@ -114,12 +185,16 @@ dist_table_alloc(void *dep_tmpl) erts_mtx_init(&dep->qlock, "dist_entry_out_queue", sysname, ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); - dep->qflgs = 0; - dep->qsize = 0; + erts_atomic32_init_nob(&dep->qflgs, 0); + erts_atomic_init_nob(&dep->qsize, 0); + erts_atomic64_init_nob(&dep->in, 0); + erts_atomic64_init_nob(&dep->out, 0); dep->out_queue.first = NULL; dep->out_queue.last = NULL; dep->suspended = NULL; + dep->tmp_out_queue.first = NULL; + dep->tmp_out_queue.last = NULL; dep->finalized_out_queue.first = NULL; dep->finalized_out_queue.last = NULL; @@ -181,7 +256,7 @@ dist_table_free(void *vdep) #ifdef DEBUG sys_memset(vdep, 0x77, sizeof(DistEntry)); #endif - erts_free(ERTS_ALC_T_DIST_ENTRY, (void *) dep); + erts_bin_free(ErtsDistEntry2Bin(dep)); ASSERT(dist_entries > 0); dist_entries--; @@ -199,19 +274,52 @@ erts_dist_table_info(fmtfn_t to, void *to_arg) erts_rwmtx_runlock(&erts_dist_table_rwmtx); } +static ERTS_INLINE DistEntry *find_dist_entry(Eterm sysname, + int inc_refc, + int connected_only) +{ + DistEntry *res; + DistEntry de; + de.sysname = sysname; + erts_rwmtx_rlock(&erts_dist_table_rwmtx); + res = hash_get(&erts_dist_table, (void *) &de); + if (res) { + if (connected_only && is_nil(res->cid)) + res = NULL; + else { + int pend_delete; + erts_aint_t refc; + if (inc_refc) { + refc = de_refc_inc_read(res, 1); + pend_delete = refc < 2; + } + else { + refc = de_refc_read(res, 0); + pend_delete = refc < 1; + } + if (pend_delete) /* Pending delete */ + de_refc_inc(res, 1); + } + } + erts_rwmtx_runlock(&erts_dist_table_rwmtx); + return res; +} + DistEntry * erts_channel_no_to_dist_entry(Uint cno) { + /* + * Does NOT increase reference count! + */ + /* * For this node (and previous incarnations of this node), * ERST_INTERNAL_CHANNEL_NO (will always be 0 I guess) is used as * channel no. For other nodes, the atom index of the atom corresponding * to the node name is used as channel no. */ - if(cno == ERST_INTERNAL_CHANNEL_NO) { - erts_refc_inc(&erts_this_dist_entry->refc, 2); + if (cno == ERST_INTERNAL_CHANNEL_NO) return erts_this_dist_entry; - } if((cno > MAX_ATOM_INDEX) || (cno >= atom_table_size()) @@ -220,80 +328,97 @@ erts_channel_no_to_dist_entry(Uint cno) /* cno is a valid atom index; find corresponding dist entry (if there is one) */ - return erts_find_dist_entry(make_atom(cno)); + return find_dist_entry(make_atom(cno), 0, 0); } - DistEntry * erts_sysname_to_connected_dist_entry(Eterm sysname) { - DistEntry de; - DistEntry *res_dep; - de.sysname = sysname; - - if(erts_this_dist_entry->sysname == sysname) { - erts_refc_inc(&erts_this_dist_entry->refc, 2); + /* + * Does NOT increase reference count! + */ + if(erts_this_dist_entry->sysname == sysname) return erts_this_dist_entry; - } - - erts_rwmtx_rlock(&erts_dist_table_rwmtx); - res_dep = (DistEntry *) hash_get(&erts_dist_table, (void *) &de); - if (res_dep) { - erts_aint_t refc = erts_refc_inctest(&res_dep->refc, 1); - if (refc < 2) /* Pending delete */ - erts_refc_inc(&res_dep->refc, 1); - } - erts_rwmtx_runlock(&erts_dist_table_rwmtx); - if (res_dep) { - int deref; - erts_rwmtx_rlock(&res_dep->rwmtx); - deref = is_nil(res_dep->cid); - erts_rwmtx_runlock(&res_dep->rwmtx); - if (deref) { - erts_deref_dist_entry(res_dep); - res_dep = NULL; - } - } - return res_dep; + return find_dist_entry(sysname, 0, 1); } DistEntry *erts_find_or_insert_dist_entry(Eterm sysname) { + /* + * This function DOES increase reference count! + */ DistEntry *res; DistEntry de; erts_aint_t refc; - res = erts_find_dist_entry(sysname); + res = find_dist_entry(sysname, 1, 0); if (res) return res; de.sysname = sysname; erts_rwmtx_rwlock(&erts_dist_table_rwmtx); res = hash_put(&erts_dist_table, (void *) &de); - refc = erts_refc_inctest(&res->refc, 0); + refc = de_refc_inc_read(res, 0); if (refc < 2) /* New or pending delete */ - erts_refc_inc(&res->refc, 1); + de_refc_inc(res, 1); erts_rwmtx_rwunlock(&erts_dist_table_rwmtx); return res; } DistEntry *erts_find_dist_entry(Eterm sysname) { - DistEntry *res; - DistEntry de; - de.sysname = sysname; - erts_rwmtx_rlock(&erts_dist_table_rwmtx); - res = hash_get(&erts_dist_table, (void *) &de); - if (res) { - erts_aint_t refc = erts_refc_inctest(&res->refc, 1); - if (refc < 2) /* Pending delete */ - erts_refc_inc(&res->refc, 1); - } - erts_rwmtx_runlock(&erts_dist_table_rwmtx); - return res; + /* + * Does NOT increase reference count! + */ + return find_dist_entry(sysname, 0, 0); } -static void try_delete_dist_entry(void *vdep) +DistEntry * +erts_dhandle_to_dist_entry(Eterm dhandle) { - DistEntry *dep = (DistEntry *) vdep; + Binary *bin; + if (!is_internal_magic_ref(dhandle)) + return NULL; + bin = erts_magic_ref2bin(dhandle); + if (ERTS_MAGIC_BIN_DESTRUCTOR(bin) != erts_dist_entry_destructor) + return NULL; + return ErtsBin2DistEntry(bin); +} + +Eterm +erts_make_dhandle(Process *c_p, DistEntry *dep) +{ + Binary *bin; + Eterm *hp; + + bin = ErtsDistEntry2Bin(dep); + ASSERT(bin); + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == erts_dist_entry_destructor); + hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE); + return erts_mk_magic_ref(&hp, &c_p->off_heap, bin); +} + +static void try_delete_dist_entry(void *vbin); + +static void +prepare_try_delete_dist_entry(void *vbin) +{ + Binary *bin = (Binary *) vbin; + DistEntry *dep = ErtsBin2DistEntry(bin); + Uint size; + erts_aint_t refc; + + refc = de_refc_read(dep, 0); + if (refc > 0) + return; + + size = ERTS_MAGIC_BIN_SIZE(sizeof(DistEntry)); + erts_schedule_thr_prgr_later_cleanup_op(try_delete_dist_entry, + vbin, &dep->later_op, size); +} + +static void try_delete_dist_entry(void *vbin) +{ + Binary *bin = (Binary *) vbin; + DistEntry *dep = ErtsBin2DistEntry(bin); erts_aint_t refc; erts_rwmtx_rwlock(&erts_dist_table_rwmtx); @@ -312,26 +437,39 @@ static void try_delete_dist_entry(void *vdep) * * If refc > 0, the entry is in use. Keep the entry. */ - refc = erts_refc_dectest(&dep->refc, -1); + refc = de_refc_dec_read(dep, -1); if (refc == -1) (void) hash_erase(&erts_dist_table, (void *) dep); erts_rwmtx_rwunlock(&erts_dist_table_rwmtx); - if (refc == 0) - erts_schedule_delete_dist_entry(dep); + if (refc == 0) { + if (node_tab_delete_delay == 0) + prepare_try_delete_dist_entry(vbin); + else if (node_tab_delete_delay > 0) + erts_start_timer_callback(node_tab_delete_delay, + prepare_try_delete_dist_entry, + vbin); + } } -void erts_schedule_delete_dist_entry(DistEntry *dep) +int erts_dist_entry_destructor(Binary *bin) { - ASSERT(dep != erts_this_dist_entry); - if (dep != erts_this_dist_entry) { - if (node_tab_delete_delay == 0) - try_delete_dist_entry((void *) dep); - else if (node_tab_delete_delay > 0) - erts_start_timer_callback(node_tab_delete_delay, - try_delete_dist_entry, - (void *) dep); - } + DistEntry *dep = ErtsBin2DistEntry(bin); + erts_aint_t refc; + + refc = de_refc_read(dep, -1); + + if (refc == -1) + return 1; /* Allow deallocation of structure... */ + + if (node_tab_delete_delay == 0) + prepare_try_delete_dist_entry((void *) bin); + else if (node_tab_delete_delay > 0) + erts_start_timer_callback(node_tab_delete_delay, + prepare_try_delete_dist_entry, + (void *) bin); + + return 0; } Uint @@ -384,7 +522,7 @@ erts_set_dist_entry_not_connected(DistEntry *dep) erts_rwmtx_rwlock(&erts_dist_table_rwmtx); ASSERT(dep != erts_this_dist_entry); - ASSERT(is_internal_port(dep->cid)); + ASSERT(is_internal_port(dep->cid) || is_internal_pid(dep->cid)); if(dep->flags & DFLAG_PUBLISHED) { if(dep->prev) { @@ -439,7 +577,7 @@ erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags) ASSERT(dep != erts_this_dist_entry); ASSERT(is_nil(dep->cid)); - ASSERT(is_internal_port(cid)); + ASSERT(is_internal_port(cid) || is_internal_pid(cid)); if(dep->prev) { ASSERT(is_in_de_list(dep, erts_not_connected_dist_entries)); @@ -459,10 +597,19 @@ erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags) dep->status |= ERTS_DE_SFLG_CONNECTED; dep->flags = flags; dep->cid = cid; + erts_atomic_set_nob(&dep->input_handler, + (erts_aint_t) cid); + dep->connection_id++; dep->connection_id &= ERTS_DIST_EXT_CON_ID_MASK; dep->prev = NULL; + erts_atomic64_set_nob(&dep->in, 0); + erts_atomic64_set_nob(&dep->out, 0); + erts_atomic32_set_nob(&dep->qflgs, + (is_internal_port(cid) + ? ERTS_DE_QFLG_PORT_CTRL + : ERTS_DE_QFLG_PROC_CTRL)); if(flags & DFLAG_PUBLISHED) { dep->next = erts_visible_dist_entries; if(erts_visible_dist_entries) { @@ -716,19 +863,18 @@ void erts_set_this_node(Eterm sysname, Uint creation) { ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); - ASSERT(erts_refc_read(&erts_this_dist_entry->refc, 2)); + ASSERT(2 <= de_refc_read(erts_this_dist_entry, 2)); if (erts_refc_dectest(&erts_this_node->refc, 0) == 0) try_delete_node(erts_this_node); - if (erts_refc_dectest(&erts_this_dist_entry->refc, 0) == 0) - try_delete_dist_entry(erts_this_dist_entry); + erts_deref_dist_entry(erts_this_dist_entry); erts_this_node = NULL; /* to make sure refc is bumped for this node */ erts_this_node = erts_find_or_insert_node(sysname, creation); erts_this_dist_entry = erts_this_node->dist_entry; - erts_refc_inc(&erts_this_dist_entry->refc, 2); + erts_ref_dist_entry(erts_this_dist_entry); erts_this_node_sysname = erts_this_node_sysname_BUFFER; erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER), @@ -797,9 +943,9 @@ void erts_init_node_tables(int dd_sec) ASSERT(erts_this_node->dist_entry != NULL); erts_this_dist_entry = erts_this_node->dist_entry; /* +1 for erts_this_dist_entry */ - /* +1 for erts_this_node->dist_entry */ - erts_refc_init(&erts_this_dist_entry->refc, 2); + erts_ref_dist_entry(erts_this_dist_entry); + ASSERT(2 == de_refc_read(erts_this_dist_entry, 2)); erts_this_node_sysname = erts_this_node_sysname_BUFFER; erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER), @@ -876,6 +1022,7 @@ static Eterm AM_node_references; static Eterm AM_system; static Eterm AM_timer; static Eterm AM_delayed_delete_timer; +static Eterm AM_thread_progress_delete_timer; static void setup_reference_table(void); static Eterm reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp); @@ -965,6 +1112,7 @@ erts_get_node_and_dist_references(struct process *proc) INIT_AM(timer); INIT_AM(system); INIT_AM(delayed_delete_timer); + INIT_AM(thread_progress_delete_timer); references_atoms_need_init = 0; } @@ -1148,6 +1296,10 @@ insert_offheap2(ErlOffHeap *oh, void *arg) insert_offheap(oh, a->type, a->id); } +#define ErtsIsDistEntryBinary(Bin) \ + (((Bin)->intern.flags & BIN_FLAG_MAGIC) \ + && ERTS_MAGIC_BIN_DESTRUCTOR((Bin)) == erts_dist_entry_destructor) + static void insert_offheap(ErlOffHeap *oh, int type, Eterm id) { @@ -1158,7 +1310,10 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) for (u.hdr = oh->first; u.hdr; u.hdr = u.hdr->next) { switch (thing_subtag(u.hdr->thing_word)) { case REF_SUBTAG: - if(IsMatchProgBinary(u.mref->mb)) { + if (ErtsIsDistEntryBinary(u.mref->mb)) + insert_dist_entry(ErtsBin2DistEntry(u.mref->mb), + type, id, 0); + else if(IsMatchProgBinary(u.mref->mb)) { InsertedBin *ib; int insert_bin = 1; for (ib = inserted_bins; ib; ib = ib->next) @@ -1301,26 +1456,34 @@ insert_delayed_delete_node(void *state, ErtsMonotonicTime timeout_pos, void *vnp) { - DeclareTmpHeapNoproc(heap,3); - UseTmpHeapNoproc(3); + Eterm heap[3]; insert_node((ErlNode *) vnp, SYSTEM_REF, TUPLE2(&heap[0], AM_system, AM_delayed_delete_timer)); - UnUseTmpHeapNoproc(3); +} + +static void +insert_thr_prgr_delete_dist_entry(void *arg, ErtsThrPrgrVal thr_prgr, void *vbin) +{ + DistEntry *dep = ErtsBin2DistEntry(vbin); + Eterm heap[3]; + insert_dist_entry(dep, + SYSTEM_REF, + TUPLE2(&heap[0], AM_system, AM_thread_progress_delete_timer), + 0); } static void insert_delayed_delete_dist_entry(void *state, ErtsMonotonicTime timeout_pos, - void *vdep) + void *vbin) { - DeclareTmpHeapNoproc(heap,3); - UseTmpHeapNoproc(3); - insert_dist_entry((DistEntry *) vdep, + DistEntry *dep = ErtsBin2DistEntry(vbin); + Eterm heap[3]; + insert_dist_entry(dep, SYSTEM_REF, TUPLE2(&heap[0], AM_system, AM_delayed_delete_timer), 0); - UnUseTmpHeapNoproc(3); } static void @@ -1354,9 +1517,12 @@ setup_reference_table(void) erts_debug_callback_timer_foreach(try_delete_node, insert_delayed_delete_node, NULL); - erts_debug_callback_timer_foreach(try_delete_dist_entry, + erts_debug_callback_timer_foreach(prepare_try_delete_dist_entry, insert_delayed_delete_dist_entry, NULL); + erts_debug_later_op_foreach(try_delete_dist_entry, + insert_thr_prgr_delete_dist_entry, + NULL); UseTmpHeapNoproc(3); insert_node(erts_this_node, @@ -1421,6 +1587,14 @@ setup_reference_table(void) insert_links(ERTS_P_LINKS(proc), proc->common.id); if (ERTS_P_MONITORS(proc)) insert_monitors(ERTS_P_MONITORS(proc), proc->common.id); + { + DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(proc); + if (dep) + insert_dist_entry(dep, + CTRL_REF, + proc->common.id, + 0); + } } } @@ -1719,7 +1893,7 @@ reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp) /* DistList = [{Dist, Refc, ReferenceIdList}] */ tup = MK_3TUP(referred_dists[i].dist->sysname, - MK_UINT(erts_refc_read(&referred_dists[i].dist->refc, 0)), + MK_UINT(de_refc_read(referred_dists[i].dist, 0)), dril); dl = MK_CONS(tup, dl); } diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index 7974b25444..3bba673435 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -47,6 +47,9 @@ #define ERTS_PORT_TASK_ONLY_BASIC_TYPES__ #include "erl_port_task.h" #undef ERTS_PORT_TASK_ONLY_BASIC_TYPES__ +#define ERTS_BINARY_TYPES_ONLY__ +#include "erl_binary.h" +#undef ERTS_BINARY_TYPES_ONLY__ #define ERTS_NODE_TAB_DELAY_GC_DEFAULT (60) #define ERTS_NODE_TAB_DELAY_GC_MAX (100*1000*1000) @@ -60,11 +63,17 @@ #define ERTS_DE_SFLGS_ALL (ERTS_DE_SFLG_CONNECTED \ | ERTS_DE_SFLG_EXITING) -#define ERTS_DE_QFLG_BUSY (((Uint32) 1) << 0) -#define ERTS_DE_QFLG_EXIT (((Uint32) 1) << 1) +#define ERTS_DE_QFLG_BUSY (((erts_aint32_t) 1) << 0) +#define ERTS_DE_QFLG_EXIT (((erts_aint32_t) 1) << 1) +#define ERTS_DE_QFLG_REQ_INFO (((erts_aint32_t) 1) << 2) +#define ERTS_DE_QFLG_PORT_CTRL (((erts_aint32_t) 1) << 3) +#define ERTS_DE_QFLG_PROC_CTRL (((erts_aint32_t) 1) << 4) #define ERTS_DE_QFLGS_ALL (ERTS_DE_QFLG_BUSY \ - | ERTS_DE_QFLG_EXIT) + | ERTS_DE_QFLG_EXIT \ + | ERTS_DE_QFLG_REQ_INFO \ + | ERTS_DE_QFLG_PORT_CTRL \ + | ERTS_DE_QFLG_PROC_CTRL) #if defined(ARCH_64) #define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713f713f713UL) @@ -106,12 +115,13 @@ typedef struct dist_entry_ { HashBucket hash_bucket; /* Hash bucket */ struct dist_entry_ *next; /* Next entry in dist_table (not sorted) */ struct dist_entry_ *prev; /* Previous entry in dist_table (not sorted) */ - erts_refc_t refc; /* Reference count */ - erts_rwmtx_t rwmtx; /* Protects all fields below until lck_mtx. */ + erts_rwmtx_t rwmtx; /* Protects all fields below until lck_mtx. */ Eterm sysname; /* name@host atom for efficiency */ Uint32 creation; /* creation of connected node */ - Eterm cid; /* connection handler (pid or port), NIL == free */ + erts_atomic_t input_handler; /* Input handler */ + Eterm cid; /* connection handler (pid or port), + NIL == free */ Uint32 connection_id; /* Connection id incremented on connect */ Uint32 status; /* Slot status, like exiting reserved etc */ Uint32 flags; /* Distribution flags, like hidden, @@ -119,7 +129,7 @@ typedef struct dist_entry_ { unsigned long version; /* Protocol version */ - erts_mtx_t lnk_mtx; /* Protects node_links, nlinks, and + erts_mtx_t lnk_mtx; /* Protects node_links, nlinks, and monitors. */ ErtsLink *node_links; /* In a dist entry, node links are kept in a separate tree, while they are @@ -131,12 +141,15 @@ typedef struct dist_entry_ { ErtsLink *nlinks; /* Link tree with subtrees */ ErtsMonitor *monitors; /* Monitor tree */ - erts_mtx_t qlock; /* Protects qflgs and out_queue */ - Uint32 qflgs; - Sint qsize; + erts_mtx_t qlock; /* Protects qflgs and out_queue */ + erts_atomic32_t qflgs; + erts_atomic_t qsize; + erts_atomic64_t in; + erts_atomic64_t out; ErtsDistOutputQueue out_queue; struct ErtsProcList_ *suspended; + ErtsDistOutputQueue tmp_out_queue; ErtsDistOutputQueue finalized_out_queue; erts_atomic_t dist_cmd_scheduled; ErtsPortTaskHandle dist_cmd; @@ -144,6 +157,8 @@ typedef struct dist_entry_ { Uint (*send)(Port *prt, ErtsDistOutputBuf *obuf); struct cache* cache; /* The atom cache */ + + ErtsThrPrgrLaterOp later_op; } DistEntry; typedef struct erl_node_ { @@ -193,12 +208,12 @@ Eterm erts_get_node_and_dist_references(struct process *); int erts_lc_is_de_rwlocked(DistEntry *); int erts_lc_is_de_rlocked(DistEntry *); #endif +int erts_dist_entry_destructor(Binary *bin); +DistEntry *erts_dhandle_to_dist_entry(Eterm dhandle); +Eterm erts_make_dhandle(Process *c_p, DistEntry *dep); +void erts_ref_dist_entry(DistEntry *dep); +void erts_deref_dist_entry(DistEntry *dep); -#ifdef ERTS_ENABLE_LOCK_COUNT -void erts_lcnt_update_distribution_locks(int enable); -#endif - -ERTS_GLB_INLINE void erts_deref_dist_entry(DistEntry *dep); ERTS_GLB_INLINE void erts_deref_node_entry(ErlNode *np); ERTS_GLB_INLINE void erts_de_rlock(DistEntry *dep); ERTS_GLB_INLINE void erts_de_runlock(DistEntry *dep); @@ -210,14 +225,6 @@ ERTS_GLB_INLINE void erts_de_links_unlock(DistEntry *dep); #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void -erts_deref_dist_entry(DistEntry *dep) -{ - ASSERT(dep); - if (erts_refc_dectest(&dep->refc, 0) == 0) - erts_schedule_delete_dist_entry(dep); -} - -ERTS_GLB_INLINE void erts_deref_node_entry(ErlNode *np) { ASSERT(np); diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index 98e9b9ccaf..9117eb1f72 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -31,6 +31,9 @@ typedef struct ErtsProc2PortSigData_ ErtsProc2PortSigData; #include "erl_ptab.h" #include "erl_thr_progress.h" #include "erl_trace.h" +#define ERTS_IO_QUEUE_TYPES_ONLY__ +#include "erl_io_queue.h" +#undef ERTS_IO_QUEUE_TYPES_ONLY__ #ifndef __WIN32__ #define ERTS_DEFAULT_MAX_PORTS (1 << 16) @@ -75,23 +78,8 @@ typedef struct erts_driver_t_ erts_driver_t; #define ERTS_Port2ErlDrvPort(PH) ((ErlDrvPort) (PH)) #endif -#define SMALL_IO_QUEUE 5 /* Number of fixed elements */ +typedef ErtsIOQueue ErlPortIOQueue; -typedef struct { - ErlDrvSizeT size; /* total size in bytes */ - - SysIOVec* v_start; - SysIOVec* v_end; - SysIOVec* v_head; - SysIOVec* v_tail; - SysIOVec v_small[SMALL_IO_QUEUE]; - - ErlDrvBinary** b_start; - ErlDrvBinary** b_end; - ErlDrvBinary** b_head; - ErlDrvBinary** b_tail; - ErlDrvBinary* b_small[SMALL_IO_QUEUE]; -} ErlIOQueue; typedef struct line_buf { /* Buffer used in line oriented I/O */ ErlDrvSizeT bufsiz; /* Size of character buffer */ @@ -165,7 +153,7 @@ struct _erl_drv_port { Uint bytes_in; /* Number of bytes read */ Uint bytes_out; /* Number of bytes written */ - ErlIOQueue ioq; /* driver accessible i/o queue */ + ErlPortIOQueue ioq; /* driver accessible i/o queue */ DistEntry *dist_entry; /* Dist entry used in DISTRIBUTION */ char *name; /* String used in the open */ erts_driver_t* drv_ptr; diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 1420fb9c06..a588477320 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -36,6 +36,7 @@ #include "erl_check_io.h" #include "dtrace-wrapper.h" #include "lttng-wrapper.h" +#include "erl_check_io.h" #include <stdarg.h> /* @@ -90,8 +91,6 @@ static void chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_q erts_atomic_read_nob(&(PP)->run_queue))); \ } while (0) -erts_atomic_t erts_port_task_outstanding_io_tasks; - #define ERTS_PT_STATE_SCHEDULED 0 #define ERTS_PT_STATE_ABORTED 1 #define ERTS_PT_STATE_EXECUTING 2 @@ -99,7 +98,6 @@ erts_atomic_t erts_port_task_outstanding_io_tasks; typedef union { struct { /* I/O tasks */ ErlDrvEvent event; - ErlDrvEventData event_data; } io; struct { ErtsProc2PortSigCallback callback; @@ -149,10 +147,10 @@ static void begin_port_cleanup(Port *pp, ErtsPortTask **execq, int *processing_busy_q_p); -ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(port_task, - ErtsPortTask, - 1000, - ERTS_ALC_T_PORT_TASK) +ERTS_THR_PREF_QUICK_ALLOC_IMPL(port_task, + ErtsPortTask, + 1000, + ERTS_ALC_T_PORT_TASK) ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(busy_caller_table, ErtsPortTaskBusyCallerTable, @@ -585,8 +583,9 @@ reset_executed_io_task_handle(ErtsPortTask *ptp) { if (ptp->u.alive.handle) { ASSERT(ptp == handle2task(ptp->u.alive.handle)); - erts_io_notify_port_task_executed(ptp->u.alive.handle); - reset_port_task_handle(ptp->u.alive.handle); + /* The port task handle is reset inside task_executed */ + erts_io_notify_port_task_executed(ptp->type, ptp->u.alive.handle, + reset_port_task_handle); } } @@ -1308,21 +1307,7 @@ erts_port_task_abort(ErtsPortTaskHandle *pthp) if (old_state != ERTS_PT_STATE_SCHEDULED) res = - 1; /* Task already aborted, executing, or executed */ else { - reset_port_task_handle(pthp); - - switch (ptp->type) { - case ERTS_PORT_TASK_INPUT: - case ERTS_PORT_TASK_OUTPUT: - case ERTS_PORT_TASK_EVENT: - ASSERT(erts_atomic_read_nob( - &erts_port_task_outstanding_io_tasks) > 0); - erts_atomic_dec_relb(&erts_port_task_outstanding_io_tasks); - break; - default: - break; - } - res = 0; } } @@ -1438,10 +1423,10 @@ erts_port_task_schedule(Eterm id, erts_thr_progress_unmanaged_continue(dhndl); } - if (!pp) - goto fail; - if (type != ERTS_PORT_TASK_PROC_SIG) { + if (!pp) + goto fail; + ptp = port_task_alloc(); ptp->type = type; @@ -1459,16 +1444,6 @@ erts_port_task_schedule(Eterm id, va_start(argp, type); ptp->u.alive.td.io.event = va_arg(argp, ErlDrvEvent); va_end(argp); - erts_atomic_inc_relb(&erts_port_task_outstanding_io_tasks); - break; - } - case ERTS_PORT_TASK_EVENT: { - va_list argp; - va_start(argp, type); - ptp->u.alive.td.io.event = va_arg(argp, ErlDrvEvent); - ptp->u.alive.td.io.event_data = va_arg(argp, ErlDrvEventData); - va_end(argp); - erts_atomic_inc_relb(&erts_port_task_outstanding_io_tasks); break; } case ERTS_PORT_TASK_PROC_SIG: { @@ -1479,6 +1454,9 @@ erts_port_task_schedule(Eterm id, ptp->u.alive.td.psig.callback = va_arg(argp, ErtsProc2PortSigCallback); ptp->u.alive.flags |= va_arg(argp, int); va_end(argp); + if (!pp) + goto fail; + if (!(ptp->u.alive.flags & ERTS_PT_FLG_NOSUSPEND)) set_tmp_handle(ptp, pthp); else { @@ -1641,13 +1619,12 @@ erts_port_task_free_port(Port *pp) * scheduling of processes. Run-queue lock should be held by caller. */ -int +void erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) { Port *pp; ErtsPortTask *execq; int processing_busy_q; - int res = 0; int vreds = 0; int reds = 0; erts_aint_t io_tasks_executed = 0; @@ -1662,7 +1639,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) pp = pop_port(runq); if (!pp) { - res = 0; goto done; } @@ -1765,17 +1741,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) reset_executed_io_task_handle(ptp); io_tasks_executed++; break; - case ERTS_PORT_TASK_EVENT: - reds = ERTS_PORT_REDS_EVENT; - ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0); - DTRACE_DRIVER(driver_event, pp); - LTTNG_DRIVER(driver_event, pp); - (*pp->drv_ptr->event)((ErlDrvData) pp->drv_data, - ptp->u.alive.td.io.event, - ptp->u.alive.td.io.event_data); - reset_executed_io_task_handle(ptp); - io_tasks_executed++; - break; case ERTS_PORT_TASK_PROC_SIG: { ErtsProc2PortSigData *sigdp = &ptp->u.alive.td.psig.data; reset_handle(ptp); @@ -1840,14 +1805,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) erts_unblock_fpe(fpe_was_unmasked); ERTS_MSACC_POP_STATE_M(); - - if (io_tasks_executed) { - ASSERT(erts_atomic_read_nob(&erts_port_task_outstanding_io_tasks) - >= io_tasks_executed); - erts_atomic_add_relb(&erts_port_task_outstanding_io_tasks, - -1*io_tasks_executed); - } - ASSERT(runq == (ErtsRunQueue *) erts_atomic_read_nob(&pp->run_queue)); active = finalize_exec(pp, &execq, processing_busy_q); @@ -1888,15 +1845,11 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) } done: - res = (erts_atomic_read_nob(&erts_port_task_outstanding_io_tasks) - != (erts_aint_t) 0); runq->scheduler->reductions += reds; ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq)); ERTS_PORT_REDUCTIONS_EXECUTED(esdp, runq, reds); - - return res; } static void @@ -2028,13 +1981,6 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) DO_WRITE, 1); break; - case ERTS_PORT_TASK_EVENT: - erts_stale_drv_select(pp->common.id, - ERTS_Port2ErlDrvPort(pp), - ptp->u.alive.td.io.event, - 0, - 1); - break; case ERTS_PORT_TASK_DIST_CMD: break; case ERTS_PORT_TASK_PROC_SIG: { @@ -2148,8 +2094,7 @@ erts_dequeue_port(ErtsRunQueue *rq) void erts_port_task_init(void) { - erts_atomic_init_nob(&erts_port_task_outstanding_io_tasks, - (erts_aint_t) 0); - init_port_task_alloc(); + init_port_task_alloc(erts_no_schedulers + erts_no_poll_threads + + 1); /* aux_thread */ init_busy_caller_table_alloc(); } diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index 561f4ca936..ae78a7d8a3 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -56,17 +56,11 @@ typedef erts_atomic_t ErtsPortTaskHandle; typedef enum { ERTS_PORT_TASK_INPUT, ERTS_PORT_TASK_OUTPUT, - ERTS_PORT_TASK_EVENT, ERTS_PORT_TASK_TIMEOUT, ERTS_PORT_TASK_DIST_CMD, ERTS_PORT_TASK_PROC_SIG } ErtsPortTaskType; -#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS -/* NOTE: Do not access any of the exported variables directly */ -extern erts_atomic_t erts_port_task_outstanding_io_tasks; -#endif - #define ERTS_PTS_FLG_IN_RUNQ (((erts_aint32_t) 1) << 0) #define ERTS_PTS_FLG_EXEC (((erts_aint32_t) 1) << 1) #define ERTS_PTS_FLG_HAVE_TASKS (((erts_aint32_t) 1) << 2) @@ -140,10 +134,6 @@ ERTS_GLB_INLINE void erts_port_task_sched_unlock(ErtsPortTaskSched *ptsp); ERTS_GLB_INLINE int erts_port_task_sched_lock_is_locked(ErtsPortTaskSched *ptsp); ERTS_GLB_INLINE void erts_port_task_sched_enter_exiting_state(ErtsPortTaskSched *ptsp); -#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS -ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void); -#endif - #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void @@ -221,24 +211,16 @@ erts_port_task_sched_enter_exiting_state(ErtsPortTaskSched *ptsp) erts_atomic32_read_bor_nob(&ptsp->flags, ERTS_PTS_FLG_EXITING); } -#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS - -ERTS_GLB_INLINE int -erts_port_task_have_outstanding_io_tasks(void) -{ - return (erts_atomic_read_acqb(&erts_port_task_outstanding_io_tasks) - != 0); -} - -#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */ - #endif #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS -int erts_port_task_execute(ErtsRunQueue *, Port **); +void erts_port_task_execute(ErtsRunQueue *, Port **); void erts_port_task_init(void); #endif +/* generated for 'port_task' quick allocator */ +void erts_port_task_pre_alloc_init_thread(void); + void erts_port_task_tmp_handle_detach(ErtsPortTaskHandle *); int erts_port_task_abort(ErtsPortTaskHandle *); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 0d4f7305d0..3d12b3bc21 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -24,6 +24,8 @@ # include "config.h" #endif +#define ERTS_WANT_BREAK_HANDLING + #include <stddef.h> /* offsetof() */ #include "sys.h" #include "erl_vm.h" @@ -49,6 +51,8 @@ #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" #include "erl_nfunc_sched.h" +#include "erl_check_io.h" +#include "erl_poll.h" #define ERTS_CHECK_TIME_REDS CONTEXT_REDS #define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0) @@ -170,7 +174,6 @@ extern BeamInstr beam_exit[]; extern BeamInstr beam_continue_exit[]; int ERTS_WRITE_UNLIKELY(erts_default_spo_flags) = SPO_ON_HEAP_MSGQ; -int ERTS_WRITE_UNLIKELY(erts_eager_check_io) = 1; int ERTS_WRITE_UNLIKELY(erts_sched_compact_load); int ERTS_WRITE_UNLIKELY(erts_sched_balance_util) = 0; Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers); @@ -192,15 +195,13 @@ static UWord thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_O ErtsPTab erts_proc erts_align_attribute(ERTS_CACHE_LINE_SIZE); int erts_sched_thread_suggested_stack_size = -1; -#ifdef ERTS_DIRTY_SCHEDULERS int erts_dcpu_sched_thread_suggested_stack_size = -1; int erts_dio_sched_thread_suggested_stack_size = -1; -#endif #ifdef ERTS_ENABLE_LOCK_CHECK ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE]; #endif -static struct { +static struct ErtsSchedBusyWait_ { int aux_work; int tse; int sys_schedule; @@ -209,27 +210,26 @@ static struct { int erts_disable_proc_not_running_opt; static ErtsAuxWorkData *aux_thread_aux_work_data; +static ErtsAuxWorkData *poll_thread_aux_work_data; #define ERTS_SCHDLR_SSPND_CHNG_NMSB (((erts_aint32_t) 1) << 0) #define ERTS_SCHDLR_SSPND_CHNG_MSB (((erts_aint32_t) 1) << 1) #define ERTS_SCHDLR_SSPND_CHNG_ONLN (((erts_aint32_t) 1) << 2) #define ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN (((erts_aint32_t) 1) << 3) -typedef struct { +typedef struct ErtsMultiSchedulingBlock_ { int ongoing; ErtsProcList *blckrs; ErtsProcList *chngq; } ErtsMultiSchedulingBlock; -typedef struct { +typedef struct ErtsSchedTypeCounters_ { Uint32 normal; -#ifdef ERTS_DIRTY_SCHEDULERS Uint32 dirty_cpu; Uint32 dirty_io; -#endif } ErtsSchedTypeCounters; -static struct { +static struct ErtsSchedSuspend_ { erts_mtx_t mtx; ErtsSchedTypeCounters online; ErtsSchedTypeCounters curr_online; @@ -239,9 +239,7 @@ static struct { Eterm changer; ErtsMultiSchedulingBlock nmsb; /* Normal multi Scheduling Block */ ErtsMultiSchedulingBlock msb; /* Multi Scheduling Block */ -#ifdef ERTS_DIRTY_SCHEDULERS ErtsSchedType last_msb_dirty_type; -#endif } schdlr_sspnd; static void init_scheduler_suspend(void); @@ -250,10 +248,8 @@ static ERTS_INLINE Uint32 schdlr_sspnd_eq_nscheds(ErtsSchedTypeCounters *val1p, ErtsSchedTypeCounters *val2p) { int res = val1p->normal == val2p->normal; -#ifdef ERTS_DIRTY_SCHEDULERS res &= val1p->dirty_cpu == val2p->dirty_cpu; res &= val1p->dirty_io == val2p->dirty_io; -#endif return res; } @@ -264,16 +260,10 @@ schdlr_sspnd_get_nscheds(ErtsSchedTypeCounters *valp, switch (type) { case ERTS_SCHED_NORMAL: return valp->normal; -#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: return valp->dirty_cpu; case ERTS_SCHED_DIRTY_IO: return valp->dirty_io; -#else - case ERTS_SCHED_DIRTY_CPU: - case ERTS_SCHED_DIRTY_IO: - return 0; -#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); return 0; @@ -285,10 +275,8 @@ static ERTS_INLINE Uint32 schdlr_sspnd_get_nscheds_tot(ErtsSchedTypeCounters *valp) { Uint32 res = valp->normal; -#ifdef ERTS_DIRTY_SCHEDULERS res += valp->dirty_cpu; res += valp->dirty_io; -#endif return res; } #endif @@ -303,14 +291,12 @@ schdlr_sspnd_dec_nscheds(ErtsSchedTypeCounters *valp, case ERTS_SCHED_NORMAL: valp->normal--; break; -#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: valp->dirty_cpu--; break; case ERTS_SCHED_DIRTY_IO: valp->dirty_io--; break; -#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); } @@ -324,14 +310,12 @@ schdlr_sspnd_inc_nscheds(ErtsSchedTypeCounters *valp, case ERTS_SCHED_NORMAL: valp->normal++; break; -#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: valp->dirty_cpu++; break; case ERTS_SCHED_DIRTY_IO: valp->dirty_io++; break; -#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); } @@ -345,14 +329,12 @@ schdlr_sspnd_set_nscheds(ErtsSchedTypeCounters *valp, case ERTS_SCHED_NORMAL: valp->normal = no; break; -#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: valp->dirty_cpu = no; break; case ERTS_SCHED_DIRTY_IO: valp->dirty_io = no; break; -#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); } @@ -386,9 +368,6 @@ erts_sched_stat_t erts_sched_stat; static erts_tsd_key_t ERTS_WRITE_UNLIKELY(sched_data_key); -static erts_atomic32_t function_calls; - -static erts_atomic32_t doing_sys_schedule; static erts_atomic32_t no_empty_run_queues; long erts_runq_supervision_interval = 0; static ethr_event runq_supervision_event; @@ -398,7 +377,6 @@ static erts_atomic_t runq_supervisor_sleeping; ErtsAlignedRunQueue * ERTS_WRITE_UNLIKELY(erts_aligned_run_queues); Uint ERTS_WRITE_UNLIKELY(erts_no_run_queues); -#ifdef ERTS_DIRTY_SCHEDULERS struct { union { @@ -411,12 +389,10 @@ struct { } io; } dirty_count erts_align_attribute(ERTS_CACHE_LINE_SIZE); -#endif static ERTS_INLINE void dirty_active(ErtsSchedulerData *esdp, erts_aint32_t add) { -#ifdef ERTS_DIRTY_SCHEDULERS erts_aint32_t val; erts_atomic32_t *ap; switch (esdp->type) { @@ -441,18 +417,15 @@ dirty_active(ErtsSchedulerData *esdp, erts_aint32_t add) val = erts_atomic32_read_nob(ap); val += add; erts_atomic32_set_nob(ap, val); -#endif } ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_scheduler_data); -#ifdef ERTS_DIRTY_SCHEDULERS ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_cpu_scheduler_data); ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_io_scheduler_data); typedef union { Process dsp; char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(Process))]; } ErtsAlignedDirtyShadowProcess; -#endif typedef union { ErtsSchedulerSleepInfo ssi; @@ -460,10 +433,9 @@ typedef union { } ErtsAlignedSchedulerSleepInfo; static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info; -#ifdef ERTS_DIRTY_SCHEDULERS static ErtsAlignedSchedulerSleepInfo *aligned_dirty_cpu_sched_sleep_info; static ErtsAlignedSchedulerSleepInfo *aligned_dirty_io_sched_sleep_info; -#endif +static ErtsAlignedSchedulerSleepInfo *aligned_poll_thread_sleep_info; static Uint last_reductions; static Uint last_exact_reductions; @@ -531,11 +503,14 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist, 200, ERTS_ALC_T_PROC_LIST) +#define ERTS_POLL_THREAD_SLEEP_INFO_IX(IX) \ + (ASSERT(0 <= ((int) (IX)) \ + && ((int) (IX)) < ((int) erts_no_poll_threads)), \ + &aligned_poll_thread_sleep_info[(IX)].ssi) #define ERTS_SCHED_SLEEP_INFO_IX(IX) \ - (ASSERT(-1 <= ((int) (IX)) \ - && ((int) (IX)) < ((int) erts_no_schedulers)), \ + (ASSERT(((int)-1) <= ((int) (IX)) \ + && ((int) (IX)) < ((int) erts_no_schedulers)), \ &aligned_sched_sleep_info[(IX)].ssi) -#ifdef ERTS_DIRTY_SCHEDULERS #define ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(IX) \ (ASSERT(0 <= ((int) (IX)) \ && ((int) (IX)) < ((int) erts_no_dirty_cpu_schedulers)), \ @@ -544,7 +519,6 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist, (ASSERT(0 <= ((int) (IX)) \ && ((int) (IX)) < ((int) erts_no_dirty_io_schedulers)), \ &aligned_dirty_io_sched_sleep_info[(IX)].ssi) -#endif #define ERTS_FOREACH_RUNQ(RQVAR, DO) \ do { \ @@ -776,6 +750,11 @@ erts_pre_init_process(void) = ERTS_PSD_ETS_FIXED_TABLES_GET_LOCKS; erts_psd_required_locks[ERTS_PSD_ETS_FIXED_TABLES].set_locks = ERTS_PSD_ETS_FIXED_TABLES_SET_LOCKS; + + erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].get_locks + = ERTS_PSD_DIST_ENTRY_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].set_locks + = ERTS_PSD_DIST_ENTRY_SET_LOCKS; #endif } @@ -857,7 +836,6 @@ erts_late_init_process(void) static void init_sched_wall_time(ErtsSchedulerData *esdp, Uint64 time_stamp) { -#ifdef ERTS_DIRTY_SCHEDULERS if (esdp->type != ERTS_SCHED_NORMAL) { erts_atomic32_init_nob(&esdp->sched_wall_time.u.mod, 0); esdp->sched_wall_time.enabled = 1; @@ -866,7 +844,6 @@ init_sched_wall_time(ErtsSchedulerData *esdp, Uint64 time_stamp) esdp->sched_wall_time.working.start = ERTS_SCHED_WTIME_IDLE; } else -#endif { esdp->sched_wall_time.u.need = erts_sched_balance_util; esdp->sched_wall_time.enabled = 0; @@ -1064,7 +1041,6 @@ init_runq_sched_util(ErtsRunQueueSchedUtil *rqsu, int enabled) #endif /* ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT */ -#ifdef ERTS_DIRTY_SCHEDULERS typedef struct { Uint64 working; @@ -1117,7 +1093,6 @@ read_dirty_sched_wall_time(ErtsSchedulerData *esdp, ErtsDirtySchedWallTime *info info->working = info->total; } -#endif static void @@ -1166,13 +1141,11 @@ dirty_sched_wall_time_change(ErtsSchedulerData *esdp, int working) mod++; erts_atomic32_set_nob(&esdp->sched_wall_time.u.mod, mod); -#if 0 if (!working) { - ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_BUSY_WAIT); + ERTS_MSACC_SET_STATE_X(ERTS_MSACC_STATE_BUSY_WAIT); } else { - ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_OTHER); + ERTS_MSACC_SET_STATE_X(ERTS_MSACC_STATE_OTHER); } -#endif } @@ -1220,10 +1193,8 @@ typedef struct { Eterm ref_heap[ERTS_REF_THING_SIZE]; Uint req_sched; erts_atomic32_t refc; -#ifdef ERTS_DIRTY_SCHEDULERS int want_dirty_cpu; int want_dirty_io; -#endif } ErtsSchedWallTimeReq; typedef struct { @@ -1294,7 +1265,6 @@ reply_sched_wall_time(void *vswtrp) hpp = NULL; szp = &sz; -#ifdef ERTS_DIRTY_SCHEDULERS if (esdp->sched_wall_time.enabled && swtrp->req_sched == esdp->no && (swtrp->want_dirty_cpu || swtrp->want_dirty_io)) { @@ -1376,7 +1346,6 @@ reply_sched_wall_time(void *vswtrp) erts_free(ERTS_ALC_T_TMP, dswt); } else -#endif { /* Reply with info about this scheduler only... */ @@ -1444,10 +1413,8 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable, swtrp->proc = c_p; swtrp->ref = STORE_NC(&hp, NULL, ref); swtrp->req_sched = esdp->no; -#ifdef ERTS_DIRTY_SCHEDULERS swtrp->want_dirty_cpu = want_dirty_cpu; swtrp->want_dirty_io = want_dirty_io; -#endif erts_atomic32_init_nob(&swtrp->refc, (erts_aint32_t) erts_no_schedulers); @@ -1559,6 +1526,12 @@ erts_proclist_create(Process *p) return proclist_create(p); } +ErtsProcList * +erts_proclist_copy(ErtsProcList *plp) +{ + return proclist_copy(plp); +} + void erts_proclist_destroy(ErtsProcList *plp) { @@ -1590,18 +1563,19 @@ erts_psd_set_init(Process *p, int ix, void *data) void -erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags) +erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, + erts_aint32_t flags) { switch (flags & ERTS_SSI_FLGS_SLEEP_TYPE) { case ERTS_SSI_FLG_POLL_SLEEPING: - erts_sys_schedule_interrupt(1); + erts_check_io_interrupt(ssi->psi, 1); break; case ERTS_SSI_FLG_POLL_SLEEPING|ERTS_SSI_FLG_TSE_SLEEPING: /* * Thread progress blocking while poll sleeping; need * to signal on both... */ - erts_sys_schedule_interrupt(1); + erts_check_io_interrupt(ssi->psi, 1); /* fall through */ case ERTS_SSI_FLG_TSE_SLEEPING: erts_tse_set(ssi->event); @@ -2861,36 +2835,6 @@ erts_set_aux_work_timeout(int ix, erts_aint32_t type, int enable) return old; } - - -static ERTS_INLINE void -sched_waiting_sys(Uint no, ErtsRunQueue *rq) -{ - ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); - ASSERT(rq->waiting >= 0); - (void) ERTS_RUNQ_FLGS_SET(rq, (ERTS_RUNQ_FLG_OUT_OF_WORK - | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); - rq->waiting++; - rq->waiting *= -1; - rq->woken = 0; - if (erts_system_profile_flags.scheduler) - profile_scheduler(make_small(no), am_inactive); -} - -static ERTS_INLINE void -sched_active_sys(Uint no, ErtsRunQueue *rq) -{ - ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); - - ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); - - ASSERT(rq->waiting < 0); - rq->waiting *= -1; - rq->waiting--; - if (erts_system_profile_flags.scheduler) - profile_scheduler(make_small(no), am_active); -} - Uint erts_active_schedulers(void) { @@ -2901,49 +2845,6 @@ erts_active_schedulers(void) return as; } - -static ERTS_INLINE void -clear_sys_scheduling(void) -{ - erts_atomic32_set_mb(&doing_sys_schedule, 0); -} - -static ERTS_INLINE int -try_set_sys_scheduling(void) -{ - return 0 == erts_atomic32_cmpxchg_acqb(&doing_sys_schedule, 1, 0); -} - - -static ERTS_INLINE int -prepare_for_sys_schedule(int non_blocking) -{ - if (non_blocking && erts_eager_check_io) { - return try_set_sys_scheduling(); - } - else { - while (!erts_port_task_have_outstanding_io_tasks() - && try_set_sys_scheduling()) { - if (!erts_port_task_have_outstanding_io_tasks()) - return 1; - clear_sys_scheduling(); - } - return 0; - } -} - - -static ERTS_INLINE void -sched_change_waiting_sys_to_waiting(Uint no, ErtsRunQueue *rq) -{ - ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); - - ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); - - ASSERT(rq->waiting < 0); - rq->waiting *= -1; -} - static ERTS_INLINE void sched_waiting(Uint no, ErtsRunQueue *rq) { @@ -3115,7 +3016,7 @@ sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type) erts_tse_reset(ssi->event); else { ASSERT(sleep_type == ERTS_SSI_FLG_POLL_SLEEPING); - erts_sys_schedule_interrupt(0); + erts_check_io_interrupt(ssi->psi, 0); } while (1) { @@ -3183,6 +3084,12 @@ thr_prgr_fin_wait(void *vssi) static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp); +void +erts_aux_thread_poke() +{ + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(-1)); +} + static void * aux_thread(void *unused) { @@ -3191,6 +3098,7 @@ aux_thread(void *unused) erts_aint32_t aux_work; ErtsThrPrgrCallbacks callbacks; int thr_prgr_active = 1; + ERTS_MSACC_DECLARE_CACHE(); #ifdef ERTS_ENABLE_LOCK_CHECK { @@ -3199,6 +3107,7 @@ aux_thread(void *unused) } #endif + erts_port_task_pre_alloc_init_thread(); ssi->event = erts_tse_fetch(); erts_msacc_init_thread("aux", 1, 1); @@ -3213,9 +3122,14 @@ aux_thread(void *unused) init_aux_work_data(awdp, NULL, NULL); awdp->ssi = ssi; +#if ERTS_POLL_USE_FALLBACK + ssi->psi = erts_create_pollset_thread(-1); +#endif sched_prep_spin_wait(ssi); + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER); + while (1) { erts_aint32_t flgs; @@ -3224,30 +3138,54 @@ aux_thread(void *unused) if (!thr_prgr_active) erts_thr_progress_active(NULL, thr_prgr_active = 1); aux_work = handle_aux_work(awdp, aux_work, 1); + ERTS_MSACC_UPDATE_CACHE(); if (aux_work && erts_thr_progress_update(NULL)) erts_thr_progress_leader_update(NULL); } if (!aux_work) { + +#ifdef ERTS_BREAK_REQUESTED + if (ERTS_BREAK_REQUESTED) + erts_do_break_handling(); +#endif + if (thr_prgr_active) erts_thr_progress_active(NULL, thr_prgr_active = 0); - erts_thr_progress_prepare_wait(NULL); + +#if ERTS_POLL_USE_FALLBACK flgs = sched_spin_wait(ssi, 0); if (flgs & ERTS_SSI_FLG_SLEEPING) { ASSERT(flgs & ERTS_SSI_FLG_WAITING); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + erts_check_io(ssi->psi); + } + } +#else + erts_thr_progress_prepare_wait(NULL); + + flgs = sched_spin_wait(ssi, 0); + + if (flgs & ERTS_SSI_FLG_SLEEPING) { flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); if (flgs & ERTS_SSI_FLG_SLEEPING) { - int res; + int res; ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); ASSERT(flgs & ERTS_SSI_FLG_WAITING); - do { - res = erts_tse_wait(ssi->event); - } while (res == EINTR); + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_SLEEP); + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER); } - } - erts_thr_progress_finalize_wait(NULL); + } + erts_thr_progress_finalize_wait(NULL); +#endif } flgs = sched_prep_spin_wait(ssi); @@ -3255,8 +3193,79 @@ aux_thread(void *unused) return NULL; } -static void suspend_scheduler(ErtsSchedulerData *esdp); +static void * +poll_thread(void *arg) +{ + int id = (int)(UWord)arg; + ErtsAuxWorkData *awdp = poll_thread_aux_work_data+id; + ErtsSchedulerSleepInfo *ssi = ERTS_POLL_THREAD_SLEEP_INFO_IX(id); + erts_aint32_t aux_work; + ErtsThrPrgrCallbacks callbacks; + int thr_prgr_active = 1; + struct erts_poll_thread *psi = erts_create_pollset_thread(id); + ERTS_MSACC_DECLARE_CACHE(); + +#ifdef ERTS_ENABLE_LOCK_CHECK + { + char buf[] = "poll_thread"; + erts_lc_set_thread_name(buf); + } +#endif + + erts_port_task_pre_alloc_init_thread(); + ssi->event = erts_tse_fetch(); + + erts_msacc_init_thread("poll", id, 0); + + callbacks.arg = (void *) ssi; + callbacks.wakeup = thr_prgr_wakeup; + callbacks.prepare_wait = thr_prgr_prep_wait; + callbacks.wait = thr_prgr_wait; + callbacks.finalize_wait = thr_prgr_fin_wait; + erts_thr_progress_register_managed_thread(NULL, &callbacks, 0); + init_aux_work_data(awdp, NULL, NULL); + awdp->ssi = ssi; + ssi->psi = psi; + + sched_prep_spin_wait(ssi); + + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER); + + while (1) { + erts_aint32_t flgs; + + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + if (aux_work) { + if (!thr_prgr_active) + erts_thr_progress_active(NULL, thr_prgr_active = 1); + aux_work = handle_aux_work(awdp, aux_work, 1); + ERTS_MSACC_UPDATE_CACHE(); + if (aux_work && erts_thr_progress_update(NULL)) + erts_thr_progress_leader_update(NULL); + } + + if (!aux_work) { + if (thr_prgr_active) + erts_thr_progress_active(NULL, thr_prgr_active = 0); + + flgs = sched_spin_wait(ssi, 0); + + if (flgs & ERTS_SSI_FLG_SLEEPING) { + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + erts_check_io(psi); + } + } + } + + flgs = sched_prep_spin_wait(ssi); + } + return NULL; +} static void scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) @@ -3271,21 +3280,16 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) erts_spin_lock(&rq->sleepers.lock); -#endif flgs = sched_prep_spin_wait(ssi); if (flgs & ERTS_SSI_FLG_SUSPENDED) { /* Go suspend instead... */ -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) erts_spin_unlock(&rq->sleepers.lock); -#endif return; } -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) { ssi->prev = NULL; ssi->next = rq->sleepers.list; @@ -3295,320 +3299,137 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_spin_unlock(&rq->sleepers.lock); dirty_active(esdp, -1); } -#endif - - /* - * If all schedulers are waiting, one of them *should* - * be waiting in erl_sys_schedule() - */ - - if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule(0)) { - sched_waiting(esdp->no, rq); - - erts_runq_unlock(rq); + sched_waiting(esdp->no, rq); - spincount = sched_busy_wait.tse; - - tse_wait: - - if (ERTS_SCHEDULER_IS_DIRTY(esdp)) - dirty_sched_wall_time_change(esdp, working = 0); - else if (thr_prgr_active != working) - sched_wall_time_change(esdp, working = thr_prgr_active); - - while (1) { - ErtsMonotonicTime current_time = 0; + erts_runq_unlock(rq); - aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); - } - aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1); - ERTS_MSACC_UPDATE_CACHE(); - if (aux_work && erts_thr_progress_update(esdp)) - erts_thr_progress_leader_update(esdp); - } + spincount = sched_busy_wait.tse; - if (aux_work) { - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - flgs = erts_atomic32_read_acqb(&ssi->flags); - current_time = erts_get_monotonic_time(esdp); - if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); - } - erts_bump_timers(esdp->timer_wheel, current_time); - } - } - } - else { - ErtsMonotonicTime timeout_time; - int do_timeout = 0; - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - timeout_time = erts_check_next_timeout_time(esdp); - current_time = erts_get_monotonic_time(esdp); - do_timeout = (current_time >= timeout_time); - } else { - current_time = 0; - timeout_time = ERTS_MONOTONIC_TIME_MAX; - } - if (do_timeout) { - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); - } - } - else { - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - if (thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 0); - sched_wall_time_change(esdp, 0); - } - erts_thr_progress_prepare_wait(esdp); - } + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) + dirty_sched_wall_time_change(esdp, working = 0); + else if (thr_prgr_active != working) + sched_wall_time_change(esdp, working = thr_prgr_active); - flgs = sched_spin_wait(ssi, spincount); - if (flgs & ERTS_SSI_FLG_SLEEPING) { - ASSERT(flgs & ERTS_SSI_FLG_WAITING); - flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); - if (flgs & ERTS_SSI_FLG_SLEEPING) { - int res; - ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); - ASSERT(flgs & ERTS_SSI_FLG_WAITING); - current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : - erts_get_monotonic_time(esdp); - do { - Sint64 timeout; - if (current_time >= timeout_time) - break; - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time - - current_time - - 1) + 1; - } else - timeout = -1; - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); - res = erts_tse_twait(ssi->event, timeout); - ERTS_MSACC_POP_STATE_M(); - current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : - erts_get_monotonic_time(esdp); - } while (res == EINTR); - } - } - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) - erts_thr_progress_finalize_wait(esdp); - } - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && current_time >= timeout_time) - erts_bump_timers(esdp->timer_wheel, current_time); - } + while (1) { + ErtsMonotonicTime current_time = 0; - if (!(flgs & ERTS_SSI_FLG_WAITING)) { - ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); - break; - } + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1); + ERTS_MSACC_UPDATE_CACHE(); + if (aux_work && erts_thr_progress_update(esdp)) + erts_thr_progress_leader_update(esdp); + } - flgs = sched_prep_cont_spin_wait(ssi); - spincount = sched_busy_wait.aux_work; + if (aux_work) { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + flgs = erts_atomic32_read_acqb(&ssi->flags); + current_time = erts_get_monotonic_time(esdp); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + erts_bump_timers(esdp->timer_wheel, current_time); + } + } + } + else { + ErtsMonotonicTime timeout_time; + int do_timeout = 0; + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + timeout_time = erts_check_next_timeout_time(esdp); + current_time = erts_get_monotonic_time(esdp); + do_timeout = (current_time >= timeout_time); + } else { + current_time = 0; + timeout_time = ERTS_MONOTONIC_TIME_MAX; + } + if (do_timeout) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + } + else { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 0); + sched_wall_time_change(esdp, 0); + } + erts_thr_progress_prepare_wait(esdp); + } - if (!(flgs & ERTS_SSI_FLG_WAITING)) { - ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); - break; - } + flgs = sched_spin_wait(ssi, spincount); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + int res; + ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : + erts_get_monotonic_time(esdp); + do { + Sint64 timeout; + if (current_time >= timeout_time) + break; + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time + - current_time + - 1) + 1; + } else + timeout = -1; + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); + res = erts_tse_twait(ssi->event, timeout); + ERTS_MSACC_POP_STATE_M(); + current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : + erts_get_monotonic_time(esdp); + } while (res == EINTR); + } + } + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) + erts_thr_progress_finalize_wait(esdp); + } + if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && current_time >= timeout_time) + erts_bump_timers(esdp->timer_wheel, current_time); + } - } + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + break; + } - if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) - erts_atomic32_read_band_nob(&ssi->flags, - (ERTS_SSI_FLG_SUSPENDED - | ERTS_SSI_FLG_MSB_EXEC)); + flgs = sched_prep_cont_spin_wait(ssi); + spincount = sched_busy_wait.aux_work; - if (ERTS_SCHEDULER_IS_DIRTY(esdp)) - dirty_sched_wall_time_change(esdp, working = 1); - else if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + break; } - erts_runq_lock(rq); - sched_active(esdp->no, rq); - } - else - { - - erts_atomic32_set_relb(&function_calls, 0); - *fcalls = 0; - - ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); - - sched_waiting_sys(esdp->no, rq); - - erts_runq_unlock(rq); - - ASSERT(working); - sched_wall_time_change(esdp, working = 0); - - spincount = sched_busy_wait.sys_schedule; - if (spincount == 0) - goto sys_aux_work; - - while (spincount-- > 0) { - ErtsMonotonicTime current_time; - - sys_poll_aux_work: - - if (working) - sched_wall_time_change(esdp, working = 0); - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO); + if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) + erts_atomic32_read_band_nob(&ssi->flags, + (ERTS_SSI_FLG_SUSPENDED + | ERTS_SSI_FLG_MSB_EXEC)); - ASSERT(!erts_port_task_have_outstanding_io_tasks()); - LTTNG2(scheduler_poll, esdp->no, 1); - erl_sys_schedule(1); /* Might give us something to do */ - - ERTS_MSACC_POP_STATE_M(); - - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - current_time = erts_get_monotonic_time(esdp); - if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) - erts_bump_timers(esdp->timer_wheel, current_time); - } - - sys_aux_work: - - aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { - if (!working) - sched_wall_time_change(esdp, working = 1); - if (!thr_prgr_active) - erts_thr_progress_active(esdp, thr_prgr_active = 1); - aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1); - ERTS_MSACC_UPDATE_CACHE(); - if (aux_work && erts_thr_progress_update(esdp)) - erts_thr_progress_leader_update(esdp); - } - - flgs = erts_atomic32_read_acqb(&ssi->flags); - if (!(flgs & ERTS_SSI_FLG_WAITING)) { - ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); - goto sys_woken; - } - - /* - * If we got new I/O tasks we aren't allowed to - * call erl_sys_schedule() until it is handled. - */ - if (erts_port_task_have_outstanding_io_tasks()) { - clear_sys_scheduling(); - /* - * Got to check that we still got I/O tasks; otherwise - * we have to continue checking for I/O... - */ - if (!prepare_for_sys_schedule(0)) { - spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT; - goto tse_wait; - } - } - } - - erts_runq_lock(rq); - - /* - * If we got new I/O tasks we aren't allowed to - * sleep in erl_sys_schedule(). - */ - if (erts_port_task_have_outstanding_io_tasks()) { - clear_sys_scheduling(); - - /* - * Got to check that we still got I/O tasks; otherwise - * we have to wait in erl_sys_schedule() after all... - */ - if (!prepare_for_sys_schedule(0)) { - /* - * Not allowed to wait in erl_sys_schedule; - * do tse wait instead... - */ - sched_change_waiting_sys_to_waiting(esdp->no, rq); - erts_runq_unlock(rq); - spincount = 0; - goto tse_wait; - } - } - if (aux_work) { - erts_runq_unlock(rq); - goto sys_poll_aux_work; - } - flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING); - if (!(flgs & ERTS_SSI_FLG_SLEEPING)) { - if (!(flgs & ERTS_SSI_FLG_WAITING)) { - ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); - goto sys_locked_woken; - } - erts_runq_unlock(rq); - flgs = sched_prep_cont_spin_wait(ssi); - if (!(flgs & ERTS_SSI_FLG_WAITING)) { - ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); - goto sys_woken; - } - ASSERT(!erts_port_task_have_outstanding_io_tasks()); - goto sys_poll_aux_work; - } - - ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING); - ASSERT(flgs & ERTS_SSI_FLG_WAITING); - - erts_runq_unlock(rq); - - if (working) - sched_wall_time_change(esdp, working = 0); - - if (thr_prgr_active) - erts_thr_progress_active(esdp, thr_prgr_active = 0); - - ASSERT(!erts_port_task_have_outstanding_io_tasks()); - - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO); - LTTNG2(scheduler_poll, esdp->no, 0); - - erl_sys_schedule(0); - - ERTS_MSACC_POP_STATE_M(); - - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - ErtsMonotonicTime current_time = erts_get_monotonic_time(esdp); - if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) - erts_bump_timers(esdp->timer_wheel, current_time); - } - - flgs = sched_prep_cont_spin_wait(ssi); - if (flgs & ERTS_SSI_FLG_WAITING) - goto sys_aux_work; - - sys_woken: - if (!thr_prgr_active) - erts_thr_progress_active(esdp, thr_prgr_active = 1); - erts_runq_lock(rq); - sys_locked_woken: - if (!thr_prgr_active) { - erts_runq_unlock(rq); - erts_thr_progress_active(esdp, thr_prgr_active = 1); - erts_runq_lock(rq); - } - clear_sys_scheduling(); - if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) - erts_atomic32_read_band_nob(&ssi->flags, - (ERTS_SSI_FLG_SUSPENDED - | ERTS_SSI_FLG_MSB_EXEC)); - if (!working) - sched_wall_time_change(esdp, working = 1); - sched_active_sys(esdp->no, rq); + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) + dirty_sched_wall_time_change(esdp, working = 1); + else if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); } + erts_runq_lock(rq); + sched_active(esdp->no, rq); + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) dirty_active(esdp, 1); @@ -3638,7 +3459,6 @@ ssi_wake(ErtsSchedulerSleepInfo *ssi) erts_sched_finish_poke(ssi, ssi_flags_set_wake(ssi)); } -#ifdef ERTS_DIRTY_SCHEDULERS static void dcpu_sched_ix_suspend_wake(Uint ix) @@ -3670,7 +3490,6 @@ dio_sched_ix_wake(Uint ix) } #endif -#endif static void wake_scheduler(ErtsRunQueue *rq) @@ -3689,7 +3508,6 @@ wake_scheduler(ErtsRunQueue *rq) ssi_wake(rq->scheduler->ssi); } -#ifdef ERTS_DIRTY_SCHEDULERS static void wake_dirty_schedulers(ErtsRunQueue *rq, int one) { @@ -3740,7 +3558,6 @@ wake_dirty_scheduler(ErtsRunQueue *rq) wake_dirty_schedulers(rq, 1); } -#endif #define ERTS_NO_USED_RUNQS_SHIFT 16 #define ERTS_NO_RUNQS_MASK 0xffffU @@ -3876,11 +3693,9 @@ static ERTS_INLINE void smp_notify_inc_runq(ErtsRunQueue *runq) { if (runq) { -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) wake_dirty_scheduler(runq); else -#endif wake_scheduler(runq); } } @@ -4239,7 +4054,6 @@ schedule_bound_processes(ErtsRunQueue *rq, } } -#ifdef ERTS_DIRTY_SCHEDULERS static ERTS_INLINE void clear_proc_dirty_queue_bit(Process *p, ErtsRunQueue *rq, int prio_bit) @@ -4263,7 +4077,6 @@ clear_proc_dirty_queue_bit(Process *p, ErtsRunQueue *rq, int prio_bit) ASSERT(old & qb); } -#endif /* ERTS_DIRTY_SCHEDULERS */ static void @@ -5534,13 +5347,11 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags) rq->wakeup_other += (left_len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC); if (rq->wakeup_other > wakeup_other.limit) { -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) { if (rq->waiting) { wake_dirty_scheduler(rq); } } else -#endif { int empty_rqs = erts_atomic32_read_acqb(&no_empty_run_queues); @@ -5817,7 +5628,6 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp) case ERTS_SCHED_NORMAL: id = (int) esdp->no; break; -#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: id = (int) erts_no_schedulers; id += (int) esdp->dirty_no; @@ -5827,7 +5637,6 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp) id += (int) erts_no_dirty_cpu_schedulers; id += (int) esdp->dirty_no; break; -#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); break; @@ -5887,7 +5696,6 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, esdp->f_reg_array = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER, MAX_REG * sizeof(FloatDef)); -#ifdef ERTS_DIRTY_SCHEDULERS esdp->run_queue = runq; if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) { esdp->no = 0; @@ -5921,13 +5729,8 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, | ERTS_PSFLG_PROXY)); shadow_proc->static_flags = ERTS_STC_FLG_SHADOW_PROC; } -#else - runq->scheduler = esdp; - esdp->run_queue = runq; - esdp->no = (Uint) num; - esdp->type = ERTS_SCHED_NORMAL; -#endif + ssi->esdp = esdp; esdp->ssi = ssi; esdp->current_process = NULL; esdp->current_port = NULL; @@ -5958,11 +5761,9 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, } void -erts_init_scheduling(int no_schedulers, int no_schedulers_online -#ifdef ERTS_DIRTY_SCHEDULERS - , int no_dirty_cpu_schedulers, int no_dirty_cpu_schedulers_online, +erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_threads, + int no_dirty_cpu_schedulers, int no_dirty_cpu_schedulers_online, int no_dirty_io_schedulers -#endif ) { int ix, n, no_ssi, tot_rqs; @@ -5984,12 +5785,11 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online ASSERT(no_schedulers_online <= no_schedulers); ASSERT(no_schedulers_online >= 1); ASSERT(no_schedulers >= 1); -#ifdef ERTS_DIRTY_SCHEDULERS ASSERT(no_dirty_cpu_schedulers <= no_schedulers); ASSERT(no_dirty_cpu_schedulers >= 1); ASSERT(no_dirty_cpu_schedulers_online <= no_schedulers_online); ASSERT(no_dirty_cpu_schedulers_online >= 1); -#endif + ASSERT(erts_no_poll_threads == no_poll_threads); /* Create and initialize run queues */ @@ -6016,14 +5816,12 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER); erts_cnd_init(&rq->cnd); -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(ix)) { erts_spinlock_init(&rq->sleepers.lock, "dirty_run_queue_sleep_list", make_small(ix + 1), ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER); } rq->sleepers.list = NULL; -#endif rq->waiting = 0; rq->woken = 0; @@ -6083,15 +5881,13 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online n = (int) no_schedulers; erts_no_schedulers = n; erts_no_total_schedulers = n; -#ifdef ERTS_DIRTY_SCHEDULERS erts_no_dirty_cpu_schedulers = no_dirty_cpu_schedulers; erts_no_total_schedulers += no_dirty_cpu_schedulers; erts_no_dirty_io_schedulers = no_dirty_io_schedulers; erts_no_total_schedulers += no_dirty_io_schedulers; -#endif /* Create and initialize scheduler sleep info */ - no_ssi = n+1; + no_ssi = n + 1 /* aux thread */; aligned_sched_sleep_info = erts_alloc_permanent_cache_aligned( ERTS_ALC_T_SCHDLR_SLP_INFO, @@ -6102,14 +5898,14 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online ssi->next = NULL; ssi->prev = NULL; #endif + ssi->esdp = NULL; erts_atomic32_init_nob(&ssi->flags, 0); ssi->event = NULL; /* initialized in sched_thread_func */ erts_atomic32_init_nob(&ssi->aux_work, 0); } - aligned_sched_sleep_info++; + aligned_sched_sleep_info += 1 /* aux thread */; -#ifdef ERTS_DIRTY_SCHEDULERS aligned_dirty_cpu_sched_sleep_info = erts_alloc_permanent_cache_aligned( ERTS_ALC_T_SCHDLR_SLP_INFO, @@ -6130,7 +5926,18 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online ssi->event = NULL; /* initialized in sched_dirty_io_thread_func */ erts_atomic32_init_nob(&ssi->aux_work, 0); } -#endif + + aligned_poll_thread_sleep_info = + erts_alloc_permanent_cache_aligned( + ERTS_ALC_T_SCHDLR_SLP_INFO, + no_poll_threads*sizeof(ErtsAlignedSchedulerSleepInfo)); + for (ix = 0; ix < no_poll_threads; ix++) { + ErtsSchedulerSleepInfo *ssi = &aligned_poll_thread_sleep_info[ix].ssi; + ssi->esdp = NULL; + erts_atomic32_init_nob(&ssi->flags, 0); + ssi->event = NULL; /* initialized in poll_thread */ + erts_atomic32_init_nob(&ssi->aux_work, 0); + } /* Create and initialize scheduler specific data */ @@ -6150,7 +5957,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online NULL, 0); } -#ifdef ERTS_DIRTY_SCHEDULERS { Uint64 ts = sched_wall_time_ts(); int dirty_scheds = no_dirty_cpu_schedulers + no_dirty_io_schedulers; @@ -6181,7 +5987,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online &adsp[adspix++].dsp, ts); } } -#endif init_misc_aux_work(); init_swtreq_alloc(); @@ -6190,11 +5995,14 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online erts_atomic32_init_nob(&debug_wait_completed_count, 0); /* debug only */ debug_wait_completed_flags = 0; - aux_thread_aux_work_data = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, sizeof(ErtsAuxWorkData)); + poll_thread_aux_work_data = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, + no_poll_threads * sizeof(ErtsAuxWorkData)); + init_no_runqs(no_schedulers_online, no_schedulers_online); balance_info.last_active_runqs = no_schedulers; erts_mtx_init(&balance_info.update_mtx, "migration_info_update", NIL, @@ -6233,7 +6041,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online suspend_run_queue(ERTS_RUNQ_IX(ix)); } -#ifdef ERTS_DIRTY_SCHEDULERS schdlr_sspnd_set_nscheds(&schdlr_sspnd.online, ERTS_SCHED_DIRTY_CPU, @@ -6269,19 +6076,14 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online erts_atomic32_init_nob(&dirty_count.io.active, (erts_aint32_t) no_dirty_io_schedulers); -#endif if (set_schdlr_sspnd_change_flags) erts_atomic32_set_nob(&schdlr_sspnd.changing, set_schdlr_sspnd_change_flags); - erts_atomic32_init_nob(&doing_sys_schedule, 0); - init_misc_aux_work(); - erts_atomic32_init_nob(&function_calls, 0); - /* init port tasks */ erts_port_task_init(); @@ -6354,7 +6156,6 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio) #define ERTS_ENQUEUE_DIRTY_CPU_QUEUE 2 #define ERTS_ENQUEUE_DIRTY_IO_QUEUE 3 -#ifdef ERTS_DIRTY_SCHEDULERS static int check_dirty_enqueue_in_prio_queue(Process *c_p, @@ -6448,7 +6249,6 @@ fin_dirty_enq_s_change(Process *p, return !0; } -#endif /* ERTS_DIRTY_SCHEDULERS */ static ERTS_INLINE int check_enqueue_in_prio_queue(Process *c_p, @@ -6463,14 +6263,12 @@ check_enqueue_in_prio_queue(Process *c_p, *prq_prio_p = aprio; -#ifdef ERTS_DIRTY_SCHEDULERS if (actual & ERTS_PSFLGS_DIRTY_WORK) { int res = check_dirty_enqueue_in_prio_queue(c_p, newp, actual, aprio, qbit); if (res != ERTS_ENQUEUE_NORMAL_QUEUE) return res; } -#endif max_qbit = (actual >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET) & ERTS_PSFLGS_QMASK; max_qbit |= 1 << ERTS_PSFLGS_QMASK_BITS; @@ -6513,7 +6311,6 @@ select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t st return NULL; -#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_ENQUEUE_DIRTY_CPU_QUEUE: case -ERTS_ENQUEUE_DIRTY_CPU_QUEUE: @@ -6534,7 +6331,6 @@ select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t st return NULL; -#endif default: { ErtsRunQueue* runq; @@ -6578,7 +6374,6 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, running_flgs = ERTS_PSFLG_DIRTY_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS; else { running_flgs = ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS; -#ifdef ERTS_DIRTY_SCHEDULERS if (state & ERTS_PSFLG_DIRTY_ACTIVE_SYS && (p->flags & (F_DELAY_GC|F_DISABLE_GC))) { /* @@ -6596,7 +6391,6 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, ~ERTS_PSFLG_DIRTY_ACTIVE_SYS); state &= ~ERTS_PSFLG_DIRTY_ACTIVE_SYS; } -#endif } a = state; @@ -6791,11 +6585,9 @@ change_proc_schedule_state(Process *p, | ERTS_PSFLG_DIRTY_RUNNING_SYS | ERTS_PSFLG_IN_RUNQ | ERTS_PSFLG_ACTIVE)) == ERTS_PSFLG_ACTIVE -#ifdef ERTS_DIRTY_SCHEDULERS || (n & (ERTS_PSFLG_RUNNING | ERTS_PSFLG_RUNNING_SYS | ERTS_PSFLG_EXITING)) == ERTS_PSFLG_EXITING -#endif ) { /* * Active and seemingly need to be enqueued, but @@ -7127,7 +6919,6 @@ nrml_sched_ix_resume_wake(Uint ix) sched_resume_wake__(ERTS_SCHED_SLEEP_INFO_IX(ix)); } -#ifdef ERTS_DIRTY_SCHEDULERS static void dcpu_sched_ix_resume_wake(Uint ix) @@ -7141,7 +6932,6 @@ dio_sched_ix_resume_wake(Uint ix) sched_resume_wake__(ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix)); } -#endif static erts_aint32_t sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, erts_aint32_t xpct) @@ -7189,17 +6979,19 @@ sched_spin_suspended(ErtsSchedulerSleepInfo *ssi, int spincount) } static erts_aint32_t -sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi) +sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi, + erts_aint32_t sleep_type) { erts_aint32_t oflgs; - erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_TSE_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED); + erts_aint32_t nflgs = ((ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED) + | sleep_type); erts_aint32_t xflgs = (ERTS_SSI_FLG_SLEEPING | ERTS_SSI_FLG_WAITING | ERTS_SSI_FLG_SUSPENDED); + ASSERT(sleep_type == ERTS_SSI_FLG_TSE_SLEEPING); erts_tse_reset(ssi->event); while (1) { @@ -7226,7 +7018,6 @@ init_scheduler_suspend(void) schdlr_sspnd.online.normal = 1; schdlr_sspnd.curr_online.normal = 1; schdlr_sspnd.active.normal = 1; -#ifdef ERTS_DIRTY_SCHEDULERS schdlr_sspnd.online.dirty_cpu = 0; schdlr_sspnd.curr_online.dirty_cpu = 0; schdlr_sspnd.active.dirty_cpu = 0; @@ -7234,7 +7025,6 @@ init_scheduler_suspend(void) schdlr_sspnd.curr_online.dirty_io = 0; schdlr_sspnd.active.dirty_io = 0; schdlr_sspnd.last_msb_dirty_type = ERTS_SCHED_DIRTY_IO; -#endif erts_atomic32_init_nob(&schdlr_sspnd.changing, 0); schdlr_sspnd.chngq = NULL; schdlr_sspnd.changer = am_false; @@ -7295,7 +7085,6 @@ schdlr_sspnd_resume_procs(ErtsSchedType sched_type, } } -#ifdef ERTS_DIRTY_SCHEDULERS static ERTS_INLINE int have_dirty_work(void) @@ -7455,17 +7244,6 @@ msb_scheduler_type_switch(ErtsSchedType sched_type, if (exec_type != ERTS_SCHED_NORMAL) schdlr_sspnd.last_msb_dirty_type = exec_type; else { - erts_aint32_t calls; - /* - * Going back to normal scheduler after - * dirty execution; make sure it will check - * for I/O... - */ - if (ERTS_USE_MODIFIED_TIMING()) - calls = ERTS_MODIFIED_TIMING_INPUT_REDS + 1; - else - calls = INPUT_REDUCTIONS + 1; - erts_atomic32_set_nob(&function_calls, calls); if ((nrml_prio == ERTS_MSB_NONE_PRIO_BIT) & ((dcpu_prio != ERTS_MSB_NONE_PRIO_BIT) @@ -7539,7 +7317,34 @@ msb_scheduler_type_switch(ErtsSchedType sched_type, } -#endif +static ERTS_INLINE void +suspend_normal_scheduler_sleep(ErtsSchedulerData *esdp) +{ + ErtsSchedulerSleepInfo *ssi = esdp->ssi; + erts_aint32_t flgs = sched_spin_suspended(ssi, + ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + flgs = sched_set_suspended_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + int res; + + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + } + } +} + +static ERTS_INLINE void +suspend_dirty_scheduler_sleep(ErtsSchedulerData *esdp) +{ + suspend_normal_scheduler_sleep(esdp); +} static void suspend_scheduler(ErtsSchedulerData *esdp) @@ -7567,14 +7372,6 @@ suspend_scheduler(ErtsSchedulerData *esdp) */ -#if !defined(ERTS_DIRTY_SCHEDULERS) - - sched_type = ERTS_SCHED_NORMAL; - online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN; - no = esdp->no; - ASSERT(no != 1); - -#else sched_type = esdp->type; switch (sched_type) { @@ -7602,7 +7399,6 @@ suspend_scheduler(ErtsSchedulerData *esdp) /* Suspend and let scheduler 1 of another type execute... */ } -#endif if (sched_type != ERTS_SCHED_NORMAL) { dirty_active(esdp, -1); @@ -7737,14 +7533,15 @@ suspend_scheduler(ErtsSchedulerData *esdp) schdlr_sspnd_resume_procs(sched_type, &resume); while (1) { - ErtsMonotonicTime current_time; - erts_aint32_t flgs; - if (sched_type != ERTS_SCHED_NORMAL) - aux_work = 0; - else { + suspend_dirty_scheduler_sleep(esdp); + else + { + ErtsMonotonicTime current_time, timeout_time; int evacuate = no == 1 ? 0 : !ERTS_EMPTY_RUNQ(esdp->run_queue); + ASSERT(sched_type == ERTS_SCHED_NORMAL); + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); if (aux_work|evacuate) { @@ -7766,95 +7563,34 @@ suspend_scheduler(ErtsSchedulerData *esdp) } } - } - if (aux_work) { - ASSERT(sched_type == ERTS_SCHED_NORMAL); - current_time = erts_get_monotonic_time(esdp); - if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); - } - erts_bump_timers(esdp->timer_wheel, current_time); - } - } - else { - ErtsMonotonicTime timeout_time; - int do_timeout; + if (aux_work) + timeout_time = erts_next_timeout_time(esdp->next_tmo_ref); + else + timeout_time = erts_check_next_timeout_time(esdp); - if (sched_type == ERTS_SCHED_NORMAL) { - timeout_time = erts_check_next_timeout_time(esdp); - current_time = erts_get_monotonic_time(esdp); - do_timeout = (current_time >= timeout_time); - } - else { - timeout_time = ERTS_MONOTONIC_TIME_MAX; - current_time = 0; - do_timeout = 0; - } + current_time = erts_get_monotonic_time(esdp); - if (do_timeout) { - ASSERT(sched_type == ERTS_SCHED_NORMAL); - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); - } - } - else { - if (sched_type == ERTS_SCHED_NORMAL) { - if (thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 0); - sched_wall_time_change(esdp, 0); - } - erts_thr_progress_prepare_wait(esdp); - } - flgs = sched_spin_suspended(ssi, - ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); - if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED)) { - flgs = sched_set_suspended_sleeptype(ssi); - if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_TSE_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED)) { - int res; - - if (sched_type == ERTS_SCHED_NORMAL) - current_time = erts_get_monotonic_time(esdp); - else - current_time = 0; - - do { - Sint64 timeout; - if (current_time >= timeout_time) - break; - if (sched_type != ERTS_SCHED_NORMAL) - timeout = -1; - else - timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time - - current_time - - 1) + 1; - res = erts_tse_twait(ssi->event, timeout); - - if (sched_type == ERTS_SCHED_NORMAL) - current_time = erts_get_monotonic_time(esdp); - else - current_time = 0; - - } while (res == EINTR); - } - } - if (sched_type == ERTS_SCHED_NORMAL) - erts_thr_progress_finalize_wait(esdp); - } + if (!aux_work && current_time < timeout_time) { + /* go to sleep... */ + if (thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 0); + sched_wall_time_change(esdp, 0); + } + erts_thr_progress_prepare_wait(NULL); + suspend_normal_scheduler_sleep(esdp); + erts_thr_progress_finalize_wait(NULL); + current_time = erts_get_monotonic_time(esdp); + } - if (current_time >= timeout_time) { - ASSERT(sched_type == ERTS_SCHED_NORMAL); - erts_bump_timers(esdp->timer_wheel, current_time); - } - } + if (current_time >= timeout_time) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + erts_bump_timers(esdp->timer_wheel, current_time); + } + } flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING | ERTS_SSI_FLG_SUSPENDED)); @@ -8034,11 +7770,7 @@ erts_set_schedulers_online(Process *p, erts_aint32_t changing = 0, change_flags; int online, increase; ErtsProcList *plp; -#ifdef ERTS_DIRTY_SCHEDULERS int dirty_no, change_dirty, dirty_online; -#else - ASSERT(!dirty_only); -#endif if (new_no < 1) return ERTS_SCHDLR_SSPND_EINVAL; @@ -8047,11 +7779,9 @@ erts_set_schedulers_online(Process *p, else if (erts_no_schedulers < new_no) return ERTS_SCHDLR_SSPND_EINVAL; -#ifdef ERTS_DIRTY_SCHEDULERS if (dirty_only) resume_proc = 0; else -#endif { resume_proc = 1; /* @@ -8072,9 +7802,7 @@ erts_set_schedulers_online(Process *p, have_unlocked_plocks = 0; no = (int) new_no; -#ifdef ERTS_DIRTY_SCHEDULERS if (!dirty_only) -#endif { changing = erts_atomic32_read_nob(&schdlr_sspnd.changing); if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) { @@ -8102,12 +7830,6 @@ erts_set_schedulers_online(Process *p, *old_no = online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, ERTS_SCHED_NORMAL); -#ifndef ERTS_DIRTY_SCHEDULERS - if (no == online) { - res = ERTS_SCHDLR_SSPND_DONE; - goto done; - } -#else /* ERTS_DIRTY_SCHEDULERS */ dirty_online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, ERTS_SCHED_DIRTY_CPU); if (dirty_only) @@ -8159,7 +7881,6 @@ erts_set_schedulers_online(Process *p, if (dirty_only) increase = (dirty_no > dirty_online); else -#endif /* ERTS_DIRTY_SCHEDULERS */ { change_flags |= ERTS_SCHDLR_SSPND_CHNG_ONLN; schdlr_sspnd_set_nscheds(&schdlr_sspnd.online, @@ -8173,7 +7894,6 @@ erts_set_schedulers_online(Process *p, res = ERTS_SCHDLR_SSPND_DONE; if (increase) { int ix; -#ifdef ERTS_DIRTY_SCHEDULERS if (change_dirty) { ErtsSchedulerSleepInfo* ssi; if (schdlr_sspnd.msb.ongoing) { @@ -8187,7 +7907,6 @@ erts_set_schedulers_online(Process *p, } } if (!dirty_only) -#endif { if (schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) { for (ix = online; ix < no; ix++) @@ -8209,7 +7928,6 @@ erts_set_schedulers_online(Process *p, } } else /* if decrease */ { -#ifdef ERTS_DIRTY_SCHEDULERS if (change_dirty) { if (schdlr_sspnd.msb.ongoing) { for (ix = dirty_no; ix < dirty_online; ix++) @@ -8227,7 +7945,6 @@ erts_set_schedulers_online(Process *p, } } if (!dirty_only) -#endif { if (schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) { for (ix = no; ix < online; ix++) @@ -8365,7 +8082,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal wake_scheduler(rq); } -#ifdef ERTS_DIRTY_SCHEDULERS if (!normal) { ERTS_RUNQ_FLGS_SET_NOB(ERTS_RUNQ_IX(0), ERTS_RUNQ_FLG_MSB_EXEC); erts_atomic32_read_bor_nob(&ERTS_RUNQ_IX(0)->scheduler->ssi->flags, @@ -8375,7 +8091,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) dio_sched_ix_suspend_wake(ix); } -#endif wait_until_msb: @@ -8441,7 +8156,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal for (ix = online; ix < erts_no_run_queues; ix++) suspend_run_queue(ERTS_RUNQ_IX(ix)); } -#ifdef ERTS_DIRTY_SCHEDULERS if (!schdlr_sspnd.msb.ongoing) { /* Get rid of msb-exec flag in run-queue of scheduler 1 */ resume_run_queue(ERTS_RUNQ_IX(0)); @@ -8452,7 +8166,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) dio_sched_ix_resume_wake(ix); } -#endif } unblock_res: @@ -8543,6 +8256,7 @@ sched_thread_func(void *vesdp) Uint no = esdp->no; erts_tse_t *tse; + erts_port_task_pre_alloc_init_thread(); erts_sched_init_time_sup(esdp); if (no == 1) @@ -8604,7 +8318,6 @@ sched_thread_func(void *vesdp) return NULL; } -#ifdef ERTS_DIRTY_SCHEDULERS static void* sched_dirty_cpu_thread_func(void *vesdp) { @@ -8696,19 +8409,18 @@ sched_dirty_io_thread_func(void *vesdp) no); return NULL; } -#endif - -static ethr_tid aux_tid; void erts_start_schedulers(void) { + ethr_tid tid; int res = 0; Uint actual; Uint wanted = erts_no_schedulers; Uint wanted_no_schedulers = erts_no_schedulers; char name[16]; ethr_thr_opts opts = ETHR_THR_OPTS_DEFAULT_INITER; + int ix; opts.detached = 1; @@ -8720,11 +8432,13 @@ erts_start_schedulers(void) erts_atomic_init_nob(&runq_supervisor_sleeping, 0); if (0 != ethr_event_init(&runq_supervision_event)) erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision event\n"); - if (0 != ethr_thr_create(&runq_supervisor_tid, - runq_supervisor, - NULL, - &opts)) - erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision thread\n"); + res = ethr_thr_create(&runq_supervisor_tid, + runq_supervisor, + NULL, + &opts); + if (0 != res) + erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision thread, " + "error = %d\n", res); } @@ -8752,16 +8466,14 @@ erts_start_schedulers(void) } erts_no_schedulers = actual; -#ifdef ERTS_DIRTY_SCHEDULERS { - int ix; for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) { ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); erts_snprintf(opts.name, 16, "%d_dirty_cpu_scheduler", ix + 1); opts.suggested_stack_size = erts_dcpu_sched_thread_suggested_stack_size; res = ethr_thr_create(&esdp->tid,sched_dirty_cpu_thread_func,(void*)esdp,&opts); if (res != 0) - erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty cpu scheduler thread %d\n", ix); + erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty cpu scheduler thread %d, error = %d\n", ix, res); } for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix); @@ -8769,18 +8481,25 @@ erts_start_schedulers(void) opts.suggested_stack_size = erts_dio_sched_thread_suggested_stack_size; res = ethr_thr_create(&esdp->tid,sched_dirty_io_thread_func,(void*)esdp,&opts); if (res != 0) - erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty io scheduler thread %d\n", ix); + erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty io scheduler thread %d, error = %d\n", ix, res); } } -#endif ERTS_THR_MEMORY_BARRIER; erts_snprintf(opts.name, 16, "aux"); - res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts); + res = ethr_thr_create(&tid, aux_thread, NULL, &opts); if (res != 0) - erts_exit(ERTS_ERROR_EXIT, "Failed to create aux thread\n"); + erts_exit(ERTS_ERROR_EXIT, "Failed to create aux thread, error = %d\n", res); + + for (ix = 0; ix < erts_no_poll_threads; ix++) { + erts_snprintf(opts.name, 16, "%d_poller", ix); + + res = ethr_thr_create(&tid, poll_thread, (void*)(UWord)ix, &opts); + if (res != 0) + erts_exit(ERTS_ERROR_EXIT, "Failed to create poll thread\n"); + } if (actual < 1) erts_exit(ERTS_ERROR_EXIT, @@ -8962,7 +8681,6 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, ASSERT((ERTS_PSFLG_RUNNING | ERTS_PSFLG_DIRTY_RUNNING) & erts_atomic32_read_nob(&rp->state)); -#ifdef ERTS_DIRTY_SCHEDULERS if (!suspend && (erts_atomic32_read_nob(&rp->state) & ERTS_PSFLG_DIRTY_RUNNING)) { @@ -8974,7 +8692,6 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, } goto done; } -#endif running: @@ -9490,7 +9207,6 @@ erts_internal_is_process_executing_dirty_1(BIF_ALIST_1) { if (is_not_internal_pid(BIF_ARG_1)) BIF_ERROR(BIF_P, BADARG); -#ifdef ERTS_DIRTY_SCHEDULERS else { Process *rp = erts_proc_lookup(BIF_ARG_1); if (rp) { @@ -9501,7 +9217,6 @@ erts_internal_is_process_executing_dirty_1(BIF_ALIST_1) } } } -#endif BIF_RET(am_false); } @@ -9517,7 +9232,6 @@ run_queues_len_aux(ErtsRunQueue *rq, Uint *tot_len, Uint *qlen, int *ip, int inc ASSERT(rq_len >= 0); if (incl_active_sched) { -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) { erts_aint32_t dcnt; if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(rq)) { @@ -9532,7 +9246,6 @@ run_queues_len_aux(ErtsRunQueue *rq, Uint *tot_len, Uint *qlen, int *ip, int inc rq_len += (Sint) dcnt; } else -#endif { if (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC) rq_len++; @@ -9551,12 +9264,10 @@ erts_run_queues_len(Uint *qlen, int atomic_queues_read, int incl_active_sched, Uint len = 0; int no_rqs = erts_no_run_queues; -#ifdef ERTS_DIRTY_SCHEDULERS if (incl_dirty_io) no_rqs += ERTS_NUM_DIRTY_RUNQS; else no_rqs += ERTS_NUM_DIRTY_CPU_RUNQS; -#endif if (atomic_queues_read) { ERTS_ATOMIC_FOREACH_RUNQ_X(rq, no_rqs, @@ -9856,7 +9567,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) ErtsRunQueue *rq; int context_reds; int fcalls; - int input_reductions; int actual_reds; int reds; Uint32 flags; @@ -9876,11 +9586,9 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) if (ERTS_USE_MODIFIED_TIMING()) { context_reds = ERTS_MODIFIED_TIMING_CONTEXT_REDS; - input_reductions = ERTS_MODIFIED_TIMING_INPUT_REDS; } else { context_reds = CONTEXT_REDS; - input_reductions = INPUT_REDUCTIONS; } ERTS_LC_ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()) @@ -9890,7 +9598,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) * Clean up after the process being scheduled out. */ if (!p) { /* NULL in the very first schedule() call */ -#ifdef ERTS_DIRTY_SCHEDULERS is_normal_sched = !esdp; if (is_normal_sched) { esdp = erts_get_scheduler_data(); @@ -9899,17 +9606,11 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) else { ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)); } -#else - esdp = erts_get_scheduler_data(); - is_normal_sched = 1; -#endif rq = erts_get_runq_current(esdp); ASSERT(esdp); - fcalls = (int) erts_atomic32_read_acqb(&function_calls); actual_reds = reds = 0; erts_runq_lock(rq); } else { -#ifdef ERTS_DIRTY_SCHEDULERS is_normal_sched = !esdp; if (is_normal_sched) { esdp = p->scheduler_data; @@ -9918,10 +9619,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) else { ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)); } -#else - esdp = p->scheduler_data; - is_normal_sched = 1; -#endif ASSERT(esdp->current_process == p || esdp->free_process == p); @@ -9936,7 +9633,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST; esdp->virtual_reds = 0; - fcalls = (int) erts_atomic32_add_read_acqb(&function_calls, reds); ASSERT(esdp && esdp == erts_get_scheduler_data()); rq = erts_get_runq_current(esdp); @@ -10026,16 +9722,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) ERTS_CHK_NO_PROC_LOCKS; - if (is_normal_sched) { - if (esdp->check_time_reds >= ERTS_CHECK_TIME_REDS) - (void) erts_get_monotonic_time(esdp); - - if (esdp->last_monotonic_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { - erts_runq_unlock(rq); - erts_bump_timers(esdp->timer_wheel, esdp->last_monotonic_time); - erts_runq_lock(rq); - } - } } ERTS_LC_ASSERT(!is_normal_sched || !erts_thr_progress_is_blocking()); @@ -10046,6 +9732,16 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) ErtsMigrationPath *mp; if (is_normal_sched) { + + if (esdp->check_time_reds >= ERTS_CHECK_TIME_REDS) + (void) erts_get_monotonic_time(esdp); + + if (esdp->last_monotonic_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { + erts_runq_unlock(rq); + erts_bump_timers(esdp->timer_wheel, esdp->last_monotonic_time); + erts_runq_lock(rq); + } + if (rq->check_balance_reds <= 0) check_balance(rq); @@ -10147,7 +9843,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) */ flags = ERTS_RUNQ_FLGS_GET_NOB(rq); if ((flags & ERTS_RUNQ_FLG_SUSPENDED) -#ifdef ERTS_DIRTY_SCHEDULERS /* If multi scheduling block and we have * dirty work, suspend and let dirty * scheduler handle work... */ @@ -10155,7 +9850,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) | ERTS_RUNQ_FLG_MSB_EXEC)) == ERTS_RUNQ_FLG_MSB_EXEC)) && have_dirty_work()) -#endif ) { non_empty_runq(rq); flags |= ERTS_RUNQ_FLG_NONEMPTY; @@ -10177,38 +9871,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) goto check_activities_to_run; } - else if (is_normal_sched - && (fcalls > input_reductions - && prepare_for_sys_schedule(!0))) { - ErtsMonotonicTime current_time; - /* - * Schedule system-level activities. - */ - - ERTS_MSACC_PUSH_STATE_CACHED_M(); - - erts_atomic32_set_relb(&function_calls, 0); - fcalls = 0; - -#if 0 /* Not needed since we wont wait in sys schedule */ - erts_sys_schedule_interrupt(0); -#endif - erts_runq_unlock(rq); - - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO); - LTTNG2(scheduler_poll, esdp->no, 1); - - erl_sys_schedule(1); - ERTS_MSACC_POP_STATE_M(); - - current_time = erts_get_monotonic_time(esdp); - if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) - erts_bump_timers(esdp->timer_wheel, current_time); - - erts_runq_lock(rq); - clear_sys_scheduling(); - goto continue_check_activities_to_run; - } if (flags & ERTS_RUNQ_FLG_MISC_OP) exec_misc_ops(rq); @@ -10222,29 +9884,9 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) flags = ERTS_RUNQ_FLGS_GET_NOB(rq); if (flags & PORT_BIT) { - int have_outstanding_io; - have_outstanding_io = erts_port_task_execute(rq, &esdp->current_port); - if ((!erts_eager_check_io - && have_outstanding_io - && fcalls > 2*input_reductions) - || (flags & ERTS_RUNQ_FLG_HALTING)) { - /* - * If we have performed more than 2*INPUT_REDUCTIONS since - * last call to erl_sys_schedule() and we still haven't - * handled all I/O tasks we stop running processes and - * focus completely on ports. - * - * One could argue that this is a strange behavior. The - * reason for doing it this way is that it is similar - * to the behavior before port tasks were introduced. - * We don't want to change the behavior too much, at - * least not at the time of writing. This behavior - * might change in the future. - * - * /rickard - */ - goto check_activities_to_run; - } + erts_port_task_execute(rq, &esdp->current_port); + if (flags & ERTS_RUNQ_FLG_HALTING) + goto check_activities_to_run; } /* @@ -10314,10 +9956,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) state = erts_atomic32_read_nob(&p->state); } -#ifdef ERTS_DIRTY_SCHEDULERS if (!is_normal_sched) clear_proc_dirty_queue_bit(p, rq, qbit); -#endif while (1) { erts_aint32_t exp, new; @@ -10335,7 +9975,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) | ERTS_PSFLG_DIRTY_RUNNING | ERTS_PSFLG_DIRTY_RUNNING_SYS | ERTS_PSFLG_FREE))) -#ifdef ERTS_DIRTY_SCHEDULERS | (((state & (ERTS_PSFLG_RUNNING | ERTS_PSFLG_FREE @@ -10344,7 +9983,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) | ERTS_PSFLG_EXITING)) == ERTS_PSFLG_EXITING) & (!!is_normal_sched)) -#endif ) & ((state & (ERTS_PSFLG_SUSPENDED | ERTS_PSFLG_EXITING @@ -10353,11 +9991,9 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) | ERTS_PSFLG_ACTIVE_SYS | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) != ERTS_PSFLG_SUSPENDED) -#ifdef ERTS_DIRTY_SCHEDULERS & (!(state & (ERTS_PSFLG_EXITING | ERTS_PSFLG_PENDING_EXIT)) | (!!is_normal_sched)) -#endif ); if (run_process) { @@ -10431,10 +10067,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) state = erts_atomic32_read_nob(&p->state); -#ifndef ERTS_DIRTY_SCHEDULERS - ASSERT(!p->scheduler_data); - p->scheduler_data = esdp; -#else /* ERTS_DIRTY_SCHEDULERS */ if (is_normal_sched) { if ((!!(state & ERTS_PSFLGS_DIRTY_WORK)) & (!(state & ERTS_PSFLG_ACTIVE_SYS))) { @@ -10470,7 +10102,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) : (rq == ERTS_DIRTY_IO_RUNQ && (state & ERTS_PSFLG_DIRTY_IO_PROC))); } -#endif if (state & ERTS_PSFLG_PENDING_EXIT) { erts_handle_pending_exit(p, @@ -10515,10 +10146,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) reds -= cost; if (reds <= 0) goto sched_out_proc; -#ifdef ERTS_DIRTY_SCHEDULERS if (state & ERTS_PSFLGS_DIRTY_WORK) goto sched_out_proc; -#endif } ASSERT(state & psflg_running_sys); @@ -10556,10 +10185,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) reds -= cost; if (reds <= 0) goto sched_out_proc; -#ifdef ERTS_DIRTY_SCHEDULERS if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) goto sched_out_proc; -#endif } } } @@ -10603,13 +10230,11 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result, int normal_sched) { Process *rp; -#ifdef ERTS_DIRTY_SCHEDULERS if (!normal_sched) rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, st->requester, 0, ERTS_P2P_FLG_INC_REFC); else -#endif rp = erts_proc_lookup(st->requester); if (rp) { ErtsProcLocks rp_locks; @@ -10659,10 +10284,8 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, if (rp_locks) erts_proc_unlock(rp, rp_locks); -#ifdef ERTS_DIRTY_SCHEDULERS if (!normal_sched) erts_proc_dec_refc(rp); -#endif } erts_cleanup_offheap(&st->off_heap); @@ -10818,9 +10441,7 @@ done: } static void save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio); -#ifdef ERTS_DIRTY_SCHEDULERS static void save_dirty_task(Process *c_p, ErtsProcSysTask *st); -#endif static int execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) @@ -10868,13 +10489,11 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) FLAGS(c_p) |= F_NEED_FULLSWEEP; } reds -= scheduler_gc_proc(c_p, reds); -#ifdef ERTS_DIRTY_SCHEDULERS if (c_p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) { save_dirty_task(c_p, st); st = NULL; break; } -#endif if (type == ERTS_PSTT_GC_MAJOR) minor_gc = major_gc = 1; else @@ -10916,13 +10535,11 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) fcalls, do_gc); reds -= cla_reds; if (is_non_value(st_res)) { -#ifdef ERTS_DIRTY_SCHEDULERS if (c_p->flags & F_DIRTY_CLA) { save_dirty_task(c_p, st); st = NULL; break; } -#endif /* Needed gc, but gc was disabled */ save_gc_task(c_p, st, st_prio); st = NULL; @@ -10980,13 +10597,11 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) Eterm st_res; int st_prio; -#ifdef ERTS_DIRTY_SCHEDULERS if (c_p->dirty_sys_tasks) { st = c_p->dirty_sys_tasks; c_p->dirty_sys_tasks = st->next; } else -#endif { st = fetch_sys_task(c_p, state, &qmask, &st_prio); if (!st) @@ -11022,7 +10637,6 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) return reds; } -#ifdef ERTS_DIRTY_SCHEDULERS void erts_execute_dirty_system_task(Process *c_p) @@ -11165,7 +10779,6 @@ dispatch_system_task(Process *c_p, erts_aint_t fail_state, return ret; } -#endif static BIF_RETTYPE request_system_task(Process *c_p, Eterm requester, Eterm target, @@ -11270,7 +10883,6 @@ request_system_task(Process *c_p, Eterm requester, Eterm target, st->type = ERTS_PSTT_CPC; if (!rp) goto noproc; -#ifdef ERTS_DIRTY_SCHEDULERS /* * If the process should start executing dirty * code it is important that this task is @@ -11278,7 +10890,6 @@ request_system_task(Process *c_p, Eterm requester, Eterm target, */ fail_state |= (ERTS_PSFLG_DIRTY_RUNNING | ERTS_PSFLG_DIRTY_RUNNING_SYS); -#endif break; case am_copy_literals: @@ -11300,14 +10911,12 @@ request_system_task(Process *c_p, Eterm requester, Eterm target, noproc: failure = noproc_res; } -#ifdef ERTS_DIRTY_SCHEDULERS else if (fail_state & (ERTS_PSFLG_DIRTY_RUNNING | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { ret = dispatch_system_task(c_p, fail_state, st, target, priority, operation); goto cleanup_return; } -#endif else { ERTS_INTERNAL_ERROR("Unknown failure schedule_process_sys_task()"); failure = am_internal_error; @@ -11323,9 +10932,7 @@ badarg: ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); -#ifdef ERTS_DIRTY_SCHEDULERS cleanup_return: -#endif if (st) { erts_cleanup_offheap(&st->off_heap); @@ -11389,7 +10996,6 @@ erts_schedule_ets_free_fixation(Eterm pid, DbFixation* fix) erts_schedule_generic_sys_task(pid, ERTS_PSTT_ETS_FREE_FIXATION, fix); } -#ifdef ERTS_DIRTY_SCHEDULERS static void flush_dirty_trace_messages(void *vpid) @@ -11410,7 +11016,6 @@ flush_dirty_trace_messages(void *vpid) } } -#endif /* ERTS_DIRTY_SCHEDULERS */ void erts_schedule_flush_trace_messages(Process *proc, int force_on_proc) @@ -11418,7 +11023,6 @@ erts_schedule_flush_trace_messages(Process *proc, int force_on_proc) ErtsThrPrgrDelayHandle dhndl; Eterm pid = proc->common.id; -#ifdef ERTS_DIRTY_SCHEDULERS erts_aint32_t state; if (!force_on_proc) { @@ -11428,7 +11032,6 @@ erts_schedule_flush_trace_messages(Process *proc, int force_on_proc) goto sched_flush_dirty; } } -#endif dhndl = erts_thr_progress_unmanaged_delay(); @@ -11436,7 +11039,6 @@ erts_schedule_flush_trace_messages(Process *proc, int force_on_proc) erts_thr_progress_unmanaged_continue(dhndl); -#ifdef ERTS_DIRTY_SCHEDULERS if (!force_on_proc) { state = erts_atomic32_read_mb(&proc->state); if (state & (ERTS_PSFLG_DIRTY_RUNNING @@ -11466,7 +11068,6 @@ erts_schedule_flush_trace_messages(Process *proc, int force_on_proc) erts_schedule_misc_aux_work(1, flush_dirty_trace_messages, vargp); } } -#endif } static void @@ -11527,14 +11128,12 @@ save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio) } } -#ifdef ERTS_DIRTY_SCHEDULERS static void save_dirty_task(Process *c_p, ErtsProcSysTask *st) { st->next = c_p->dirty_sys_tasks; c_p->dirty_sys_tasks = st; } -#endif int erts_set_gc_state(Process *c_p, int enable) @@ -11871,10 +11470,8 @@ static void early_init_process_struct(void *varg, Eterm data) Process *proc = arg->proc; proc->common.id = make_internal_pid(data); -#ifdef ERTS_DIRTY_SCHEDULERS erts_atomic32_init_nob(&proc->dirty_state, 0); proc->dirty_sys_tasks = NULL; -#endif erts_atomic32_init_relb(&proc->state, arg->state); RUNQ_SET_RQ(&proc->run_queue, arg->run_queue); @@ -12122,6 +11719,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->msg.first = NULL; p->msg.last = &p->msg.first; p->msg.save = &p->msg.first; + p->msg.saved_last = &p->msg.first; p->msg.len = 0; p->msg_inq.first = NULL; p->msg_inq.last = &p->msg_inq.first; @@ -12364,10 +11962,8 @@ void erts_init_empty_process(Process *p) p->last_old_htop = NULL; #endif -#ifdef ERTS_DIRTY_SCHEDULERS erts_atomic32_init_nob(&p->dirty_state, 0); p->dirty_sys_tasks = NULL; -#endif erts_atomic32_init_nob(&p->state, (erts_aint32_t) PRIORITY_NORMAL); p->scheduler_data = NULL; @@ -12880,9 +12476,9 @@ send_exit_signal(Process *c_p, /* current process if and only if ((state & ERTS_PSFLG_TRAP_EXIT) && (reason != am_kill || (flags & ERTS_XSIG_FLG_IGN_KILL))) { - /* have to release the status lock in order to send the exit message */ - erts_proc_unlock(rp, *rp_locks & ERTS_PROC_LOCKS_XSIG_SEND); - *rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND; + /* have to release the status and trace lock in order to send the exit message */ + erts_proc_unlock(rp, *rp_locks & (ERTS_PROC_LOCKS_XSIG_SEND|ERTS_PROC_LOCK_TRACE)); + *rp_locks &= ~(ERTS_PROC_LOCKS_XSIG_SEND|ERTS_PROC_LOCK_TRACE); if (have_seqtrace(token) && token_update) seq_trace_update_send(token_update); if (is_value(exit_tuple)) @@ -12938,14 +12534,12 @@ send_exit_signal(Process *c_p, /* current process if and only Eterm *hp; ErlOffHeap *ohp; Uint rsn_sz = size_object(rsn); -#ifdef ERTS_DIRTY_SCHEDULERS if (state & ERTS_PSFLG_DIRTY_RUNNING) { bp = new_message_buffer(rsn_sz); ohp = &bp->off_heap; hp = &bp->mem[0]; } else -#endif { hp = HAlloc(rp, rsn_sz); ohp = &rp->off_heap; @@ -12985,10 +12579,6 @@ send_exit_signal(Process *c_p, /* current process if and only * has been scheduled, we may need to add it to a normal run * queue... */ -#ifndef ERTS_DIRTY_SCHEDULERS - (void) erts_atomic32_read_bor_relb(&rp->state, - ERTS_PSFLG_PENDING_EXIT); -#else { erts_aint32_t a = erts_atomic32_read_nob(&rp->state); while (1) { @@ -13006,7 +12596,6 @@ send_exit_signal(Process *c_p, /* current process if and only } } } -#endif } } /* else: @@ -13082,7 +12671,6 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) } erts_destroy_monitor(rmon); } - erts_deref_dist_entry(dep); } } else { ASSERT(is_pid(mon->u.pid) || is_port(mon->u.pid)); @@ -13322,7 +12910,6 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) erts_de_links_unlock(dep); if (rlnk) erts_destroy_link(rlnk); - erts_deref_dist_entry(dep); } break; @@ -13430,7 +13017,7 @@ erts_continue_exit_process(Process *p) ErtsMonitor *mon; ErtsProcLocks curr_locks = ERTS_PROC_LOCK_MAIN; Eterm reason = p->fvalue; - DistEntry *dep; + DistEntry *dep = NULL; erts_aint32_t state; int delay_del_proc = 0; @@ -13505,9 +13092,7 @@ erts_continue_exit_process(Process *p) erts_set_gc_state(p, 1); state = erts_atomic32_read_acqb(&p->state); if (state & ERTS_PSFLG_ACTIVE_SYS -#ifdef ERTS_DIRTY_SCHEDULERS || p->dirty_sys_tasks -#endif ) { if (cleanup_sys_tasks(p, state, CONTEXT_REDS) >= CONTEXT_REDS/2) goto yield; @@ -13517,9 +13102,7 @@ erts_continue_exit_process(Process *p) erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); ASSERT(p->sys_task_qs == NULL); ASSERT(ERTS_PROC_GET_DELAYED_GC_TASK_QS(p) == NULL); -#ifdef ERTS_DIRTY_SCHEDULERS ASSERT(p->dirty_sys_tasks == NULL); -#endif erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); #endif @@ -13616,7 +13199,6 @@ erts_continue_exit_process(Process *p) break; } -#ifdef ERTS_DIRTY_SCHEDULERS if (a & (ERTS_PSFLG_DIRTY_RUNNING | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { p->flags |= F_DELAYED_DEL_PROC; @@ -13626,18 +13208,20 @@ erts_continue_exit_process(Process *p) * when done with the process... */ } -#endif if (refc_inced && !(n & ERTS_PSFLG_IN_RUNQ)) erts_proc_dec_refc(p); } - - dep = (p->flags & F_DISTRIBUTION) ? erts_this_dist_entry : NULL; + + dep = ((p->flags & F_DISTRIBUTION) + ? ERTS_PROC_SET_DIST_ENTRY(p, NULL) + : NULL); erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL); if (dep) { - erts_do_net_exits(dep, reason); + erts_do_net_exits(dep, (reason == am_kill) ? am_killed : reason); + erts_deref_dist_entry(dep); } /* @@ -13672,9 +13256,7 @@ erts_continue_exit_process(Process *p) erts_proc_lock(p, ERTS_PROC_LOCK_MAIN); ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); -#ifdef ERTS_SMP erts_flush_trace_messages(p, ERTS_PROC_LOCK_MAIN); -#endif ERTS_TRACER_CLEAR(&ERTS_TRACER(p)); @@ -14018,10 +13600,8 @@ void erts_halt(int code) if (-1 == erts_atomic32_cmpxchg_acqb(&erts_halt_progress, erts_no_schedulers, -1)) { -#ifdef ERTS_DIRTY_SCHEDULERS ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_CPU_RUNQ, ERTS_RUNQ_FLG_HALTING); ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_IO_RUNQ, ERTS_RUNQ_FLG_HALTING); -#endif erts_halt_code = code; notify_reap_ports_relb(); } @@ -14047,3 +13627,24 @@ erts_dbg_check_halloc_lock(Process *p) return 0; } #endif + +void +erts_debug_later_op_foreach(void (*callback)(void*), + void (*func)(void *, ErtsThrPrgrVal, void *), + void *arg) +{ + int six; + if (!erts_thr_progress_is_blocking()) + ERTS_INTERNAL_ERROR("Not blocking thread progress"); + + for (six = 0; six < erts_no_schedulers; six++) { + ErtsSchedulerData *esdp = &erts_aligned_scheduler_data[six].esd; + ErtsThrPrgrLaterOp *lop = esdp->aux_work_data.later_op.first; + + while (lop) { + if (lop->func == callback) + func(arg, lop->later, lop->data); + lop = lop->next; + } + } +} diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 7ca37882c2..66d7848f89 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -105,21 +105,16 @@ struct saved_calls { }; extern Export exp_send, exp_receive, exp_timeout; -extern int erts_eager_check_io; extern int erts_sched_compact_load; extern int erts_sched_balance_util; extern Uint erts_no_schedulers; extern Uint erts_no_total_schedulers; -#ifdef ERTS_DIRTY_SCHEDULERS extern Uint erts_no_dirty_cpu_schedulers; extern Uint erts_no_dirty_io_schedulers; -#endif extern Uint erts_no_run_queues; extern int erts_sched_thread_suggested_stack_size; -#ifdef ERTS_DIRTY_SCHEDULERS extern int erts_dcpu_sched_thread_suggested_stack_size; extern int erts_dio_sched_thread_suggested_stack_size; -#endif #define ERTS_SCHED_THREAD_MIN_STACK_SIZE 20 /* Kilo words */ #define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192 /* Kilo words */ @@ -362,18 +357,18 @@ typedef enum { typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo; -#ifdef ERTS_DIRTY_SCHEDULERS typedef struct { erts_spinlock_t lock; ErtsSchedulerSleepInfo *list; } ErtsSchedulerSleepList; -#endif struct ErtsSchedulerSleepInfo_ { + struct ErtsSchedulerData_ *esdp; ErtsSchedulerSleepInfo *next; ErtsSchedulerSleepInfo *prev; erts_atomic32_t flags; erts_tse_t *event; + struct erts_poll_thread *psi; erts_atomic32_t aux_work; }; @@ -477,9 +472,7 @@ struct ErtsRunQueue_ { erts_mtx_t mtx; erts_cnd_t cnd; -#ifdef ERTS_DIRTY_SCHEDULERS ErtsSchedulerSleepList sleepers; -#endif ErtsSchedulerData *scheduler; int waiting; /* < 0 in sys schedule; > 0 on cnd variable */ @@ -616,13 +609,11 @@ typedef struct { (&(ESDP)->aux_work_data.yield.NAME) void erts_notify_new_aux_yield_work(ErtsSchedulerData *esdp); -#ifdef ERTS_DIRTY_SCHEDULERS typedef enum { ERTS_DIRTY_CPU_SCHEDULER, ERTS_DIRTY_IO_SCHEDULER } ErtsDirtySchedulerType; -#endif struct ErtsSchedulerData_ { /* @@ -645,10 +636,8 @@ struct ErtsSchedulerData_ { Process *current_process; ErtsSchedType type; Uint no; /* Scheduler number for normal schedulers */ -#ifdef ERTS_DIRTY_SCHEDULERS Uint dirty_no; /* Scheduler number for dirty schedulers */ Process *dirty_shadow_process; -#endif Port *current_port; ErtsRunQueue *run_queue; int virtual_reds; @@ -687,16 +676,19 @@ typedef union { } ErtsAlignedSchedulerData; extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data; -#ifdef ERTS_DIRTY_SCHEDULERS extern ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data; extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data; -#endif #if defined(ERTS_ENABLE_LOCK_CHECK) int erts_lc_runq_is_locked(ErtsRunQueue *); #endif +void +erts_debug_later_op_foreach(void (*callback)(void*), + void (*func)(void *, ErtsThrPrgrVal, void *), + void *arg); + #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS void erts_empty_runq(ErtsRunQueue *rq); @@ -803,14 +795,15 @@ erts_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) #define ERTS_PSD_NIF_TRAP_EXPORT 5 #define ERTS_PSD_ETS_OWNED_TABLES 6 #define ERTS_PSD_ETS_FIXED_TABLES 7 -#define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF 8 +#define ERTS_PSD_DIST_ENTRY 8 +#define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF 9 /* keep last... */ -#define ERTS_PSD_SIZE 9 +#define ERTS_PSD_SIZE 10 #if !defined(HIPE) # undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF # undef ERTS_PSD_SIZE -# define ERTS_PSD_SIZE 8 +# define ERTS_PSD_SIZE 9 #endif typedef struct { @@ -844,6 +837,9 @@ typedef struct { #define ERTS_PSD_ETS_FIXED_TABLES_GET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_ETS_FIXED_TABLES_SET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_DIST_ENTRY_GET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_DIST_ENTRY_SET_LOCKS ERTS_PROC_LOCK_MAIN + typedef struct { ErtsProcLocks get_locks; ErtsProcLocks set_locks; @@ -1049,14 +1045,10 @@ struct process { Uint64 bin_old_vheap; /* Virtual old heap size for binaries */ ErtsProcSysTaskQs *sys_task_qs; -#ifdef ERTS_DIRTY_SCHEDULERS ErtsProcSysTask *dirty_sys_tasks; -#endif erts_atomic32_t state; /* Process state flags (see ERTS_PSFLG_*) */ -#ifdef ERTS_DIRTY_SCHEDULERS erts_atomic32_t dirty_state; /* Process dirty state flags (see ERTS_PDSFLG_*) */ -#endif ErlMessageInQueue msg_inq; ErlTraceMessageQueue *trace_msg_q; @@ -1213,7 +1205,6 @@ void erts_check_for_holes(Process* p); #define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \ (((PSFLGS) >> ERTS_PSFLGS_PRQ_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK) -#ifdef ERTS_DIRTY_SCHEDULERS /* * Flags in the dirty_state field. @@ -1240,7 +1231,6 @@ void erts_check_for_holes(Process* p); | ERTS_PDSFLG_IN_CPU_PRQ_HIGH \ | ERTS_PDSFLG_IN_CPU_PRQ_NORMAL\ | ERTS_PDSFLG_IN_CPU_PRQ_LOW) -#endif /* @@ -1505,20 +1495,14 @@ extern int erts_system_profile_ts_type; } \ } while (0) -#if defined(ERTS_DIRTY_SCHEDULERS) #define ERTS_NUM_DIRTY_CPU_RUNQS 1 #define ERTS_NUM_DIRTY_IO_RUNQS 1 -#else -#define ERTS_NUM_DIRTY_CPU_RUNQS 0 -#define ERTS_NUM_DIRTY_IO_RUNQS 0 -#endif #define ERTS_NUM_DIRTY_RUNQS (ERTS_NUM_DIRTY_CPU_RUNQS+ERTS_NUM_DIRTY_IO_RUNQS) #define ERTS_RUNQ_IX(IX) \ (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues+ERTS_NUM_DIRTY_RUNQS), \ &erts_aligned_run_queues[(IX)].runq) -#ifdef ERTS_DIRTY_SCHEDULERS #define ERTS_RUNQ_IX_IS_DIRTY(IX) \ (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues+ERTS_NUM_DIRTY_RUNQS), \ (erts_no_run_queues <= (IX))) @@ -1529,13 +1513,9 @@ extern int erts_system_profile_ts_type; #define ERTS_DIRTY_IO_RUNQ (&erts_aligned_run_queues[erts_no_run_queues+1].runq) #define ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(RQ) ((RQ) == ERTS_DIRTY_CPU_RUNQ) #define ERTS_RUNQ_IS_DIRTY_IO_RUNQ(RQ) ((RQ) == ERTS_DIRTY_IO_RUNQ) -#else -#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0 -#endif #define ERTS_SCHEDULER_IX(IX) \ (ASSERT(0 <= (IX) && (IX) < erts_no_schedulers), \ &erts_aligned_scheduler_data[(IX)].esd) -#ifdef ERTS_DIRTY_SCHEDULERS #define ERTS_DIRTY_CPU_SCHEDULER_IX(IX) \ (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_cpu_schedulers), \ &erts_aligned_dirty_cpu_scheduler_data[(IX)].esd) @@ -1548,24 +1528,12 @@ extern int erts_system_profile_ts_type; ((ESDP)->type == ERTS_SCHED_DIRTY_CPU) #define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) \ ((ESDP)->type == ERTS_SCHED_DIRTY_IO) -#else /* !ERTS_DIRTY_SCHEDULERS */ -#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0 -#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0 -#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0 -#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0 -#endif void erts_pre_init_process(void); void erts_late_init_process(void); void erts_early_init_scheduling(int); -void erts_init_scheduling(int, int -#ifdef ERTS_DIRTY_SCHEDULERS - , int, int, int -#endif - ); -#ifdef ERTS_DIRTY_SCHEDULERS +void erts_init_scheduling(int, int, int, int, int, int); void erts_execute_dirty_system_task(Process *c_p); -#endif int erts_set_gc_state(Process *c_p, int enable); Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable, int dirty_cpu, int want_dirty_io); @@ -1576,6 +1544,7 @@ Uint64 erts_ensure_later_proc_interval(Uint64); Uint64 erts_step_proc_interval(void); ErtsProcList *erts_proclist_create(Process *); +ErtsProcList *erts_proclist_copy(ErtsProcList *); void erts_proclist_destroy(ErtsProcList *); ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *); @@ -2041,6 +2010,11 @@ erts_psd_set(Process *p, int ix, void *data) #define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, NTE) \ erts_psd_set((P), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE)) +#define ERTS_PROC_GET_DIST_ENTRY(P) \ + ((DistEntry *) erts_psd_get((P), ERTS_PSD_DIST_ENTRY)) +#define ERTS_PROC_SET_DIST_ENTRY(P, DE) \ + ((DistEntry *) erts_psd_set((P), ERTS_PSD_DIST_ENTRY, (void *) (DE))) + #ifdef HIPE #define ERTS_PROC_GET_SUSPENDED_SAVED_CALLS_BUF(P) \ ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF)) @@ -2227,7 +2201,6 @@ ErtsSchedulerData *erts_proc_sched_data(Process *c_p) ErtsSchedulerData *esdp; ASSERT(c_p); esdp = c_p->scheduler_data; -# if defined(ERTS_DIRTY_SCHEDULERS) if (esdp) { ASSERT(esdp == erts_get_scheduler_data()); ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); @@ -2237,7 +2210,6 @@ ErtsSchedulerData *erts_proc_sched_data(Process *c_p) ASSERT(esdp); ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)); } -# endif ASSERT(esdp); return esdp; } @@ -2269,11 +2241,9 @@ ERTS_GLB_INLINE Uint erts_get_scheduler_id(void) { ErtsSchedulerData *esdp = erts_get_scheduler_data(); -#ifdef ERTS_DIRTY_SCHEDULERS if (esdp && ERTS_SCHEDULER_IS_DIRTY(esdp)) return 0; else -#endif return esdp ? esdp->no : (Uint) 0; } @@ -2499,6 +2469,7 @@ void erts_notify_inc_runq(ErtsRunQueue *runq); void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t); ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi); +void erts_aux_thread_poke(void); #if ERTS_GLB_INLINE_INCL_FUNC_DEF diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c index 6cb7ccab8d..ab204303d7 100644 --- a/erts/emulator/beam/erl_sched_spec_pre_alloc.c +++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c @@ -37,7 +37,7 @@ #include "erl_thr_progress.h" erts_sspa_data_t * -erts_sspa_create(size_t blk_sz, int pa_size) +erts_sspa_create(size_t blk_sz, int pa_size, int nthreads, const char* name) { erts_sspa_data_t *data; size_t tot_size; @@ -48,22 +48,30 @@ erts_sspa_create(size_t blk_sz, int pa_size) int no_blocks = pa_size; int no_blocks_per_chunk; - if (erts_no_schedulers == 1) + if (!name) { /* schedulers only variant */ + ASSERT(!nthreads); + nthreads = erts_no_schedulers; + } + else { + ASSERT(nthreads > 0); + } + + if (nthreads == 1) no_blocks_per_chunk = no_blocks; else { int extra = (no_blocks - 1)/4 + 1; if (extra == 0) extra = 1; no_blocks_per_chunk = no_blocks; - no_blocks_per_chunk += extra*erts_no_schedulers; - no_blocks_per_chunk /= erts_no_schedulers; + no_blocks_per_chunk += extra * nthreads; + no_blocks_per_chunk /= nthreads; } - no_blocks = no_blocks_per_chunk * erts_no_schedulers; + no_blocks = no_blocks_per_chunk * nthreads; chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_chunk_header_t)); chunk_mem_size += blk_sz * no_blocks_per_chunk; chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(chunk_mem_size); tot_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t)); - tot_size += chunk_mem_size*erts_no_schedulers; + tot_size += chunk_mem_size * nthreads; p = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_PRE_ALLOC_DATA, tot_size); data = (erts_sspa_data_t *) p; @@ -72,10 +80,16 @@ erts_sspa_create(size_t blk_sz, int pa_size) data->chunks_mem_size = chunk_mem_size; data->start = chunk_start; - data->end = chunk_start + chunk_mem_size*erts_no_schedulers; + data->end = chunk_start + chunk_mem_size * nthreads; + data->nthreads = nthreads; + + if (name) { /* thread variant */ + erts_tsd_key_create(&data->tsd_key, (char*)name); + erts_atomic_init_nob(&data->id_generator, 0); + } /* Initialize all chunks */ - for (cix = 0; cix < erts_no_schedulers; cix++) { + for (cix = 0; cix < nthreads; cix++) { erts_sspa_chunk_t *chnk = erts_sspa_cix2chunk(data, cix); erts_sspa_chunk_header_t *chdr = &chnk->aligned.header; erts_sspa_blk_t *blk; diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.h b/erts/emulator/beam/erl_sched_spec_pre_alloc.h index 1307e65962..d232db0e69 100644 --- a/erts/emulator/beam/erl_sched_spec_pre_alloc.h +++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.h @@ -59,6 +59,11 @@ typedef struct { char *start; char *end; int chunks_mem_size; + int nthreads; + + /* Used only by thread variant: */ + erts_tsd_key_t tsd_key; + erts_atomic_t id_generator; } erts_sspa_data_t; typedef union erts_sspa_blk_t_ erts_sspa_blk_t; @@ -140,7 +145,9 @@ check_local_list(erts_sspa_chunk_header_t *chdr) #endif erts_sspa_data_t *erts_sspa_create(size_t blk_sz, - int pa_size); + int pa_size, + int nthreads, + const char* name); void erts_sspa_remote_free(erts_sspa_chunk_header_t *chdr, erts_sspa_blk_t *blk, int cinit); @@ -158,7 +165,7 @@ ERTS_GLB_INLINE int erts_sspa_free(erts_sspa_data_t *data, int cix, char *blk); ERTS_GLB_INLINE erts_sspa_chunk_t * erts_sspa_cix2chunk(erts_sspa_data_t *data, int cix) { - ASSERT(0 <= cix && cix < erts_no_schedulers); + ASSERT(0 <= cix && cix < data->nthreads); return (erts_sspa_chunk_t *) (data->start + cix*data->chunks_mem_size); } @@ -171,7 +178,7 @@ erts_sspa_ptr2cix(erts_sspa_data_t *data, void *ptr) return -1; diff = ((char *) ptr) - data->start; cix = (int) diff / data->chunks_mem_size; - ASSERT(0 <= cix && cix < erts_no_schedulers); + ASSERT(0 <= cix && cix < data->nthreads); return cix; } diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index 842802f8d9..6daf043117 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -270,7 +270,6 @@ _ET_DECLARE_CHECKED(Eterm*,list_val,Wterm) #define is_byte(x) (((x) & ((~(Uint)0 << (_TAG_IMMED1_SIZE+8)) + _TAG_IMMED1_MASK)) == _TAG_IMMED1_SMALL) #define is_valid_bit_size(x) (((Sint)(x)) >= 0 && ((x) & 0x7F) == _TAG_IMMED1_SMALL) #define is_not_valid_bit_size(x) (!is_valid_bit_size((x))) -#define MY_IS_SSMALL(x) (((Uint) ((((x)) >> (SMALL_BITS-1)) + 1)) < 2) #define _unchecked_unsigned_val(x) ((x) >> _TAG_IMMED1_SIZE) _ET_DECLARE_CHECKED(Uint,unsigned_val,Eterm) #define unsigned_val(x) _ET_APPLY(unsigned_val,(x)) diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index e306df818d..aedceb6fc2 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -454,7 +454,6 @@ ERTS_GLB_INLINE void erts_rwmtx_runlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx); ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx); - ERTS_GLB_INLINE void erts_spinlock_init(erts_spinlock_t *lock, char *name, Eterm extra, diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index ccc5526664..27164d50a0 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -130,6 +130,13 @@ Eterm erts_get_monotonic_end_time(struct process *c_p); Eterm erts_monotonic_time_source(struct process*c_p); Eterm erts_system_time_source(struct process*c_p); +void erts_runtime_elapsed_both(ErtsMonotonicTime *ms_user, + ErtsMonotonicTime *ms_sys, + ErtsMonotonicTime *ms_user_diff, + ErtsMonotonicTime *ms_sys_diff); +void erts_wall_clock_elapsed_both(ErtsMonotonicTime *total, + ErtsMonotonicTime *diff); + #ifdef SYS_CLOCK_RESOLUTION #define ERTS_CLKTCK_RESOLUTION ((ErtsMonotonicTime) (SYS_CLOCK_RESOLUTION*1000)) #else diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index c06b464458..f2e0900fec 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -36,12 +36,29 @@ #include "erl_driver.h" #include "erl_nif.h" -static erts_mtx_t erts_timeofday_mtx; static erts_mtx_t erts_get_time_mtx; -static SysTimes t_start; /* Used in elapsed_time_both */ -static ErtsMonotonicTime prev_wall_clock_elapsed; /* Used in wall_clock_elapsed_time_both */ -static ErtsMonotonicTime previous_now; /* Used in get_now */ + /* used by erts_runtime_elapsed_both */ +typedef struct { + erts_mtx_t mtx; + ErtsMonotonicTime user; + ErtsMonotonicTime sys; +} ErtsRunTimePrevData; + +static union { + ErtsRunTimePrevData data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsRunTimePrevData))]; +} runtime_prev erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +static union { + erts_atomic64_t time; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_atomic64_t))]; +} wall_clock_prev erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +static union { + erts_atomic64_t time; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_atomic64_t))]; +} now_prev erts_align_attribute(ERTS_CACHE_LINE_SIZE); static ErtsMonitor *time_offset_monitors = NULL; static Uint no_time_offset_monitors = 0; @@ -954,10 +971,12 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode) ASSERT(ERTS_MONOTONIC_TIME_MIN < ERTS_MONOTONIC_TIME_MAX); - erts_mtx_init(&erts_timeofday_mtx, "timeofday", NIL, - ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); erts_mtx_init(&erts_get_time_mtx, "get_time", NIL, ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); + erts_mtx_init(&runtime_prev.data.mtx, "runtime", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); + runtime_prev.data.user = 0; + runtime_prev.data.sys = 0; time_sup.r.o.correction = time_correction; time_sup.r.o.warp_mode = time_warp_mode; @@ -1157,9 +1176,13 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode) time_sup.f.c.last_not_corrected_time = 0; } - prev_wall_clock_elapsed = 0; + erts_atomic64_init_nob(&wall_clock_prev.time, + (erts_aint64_t) 0); + + erts_atomic64_init_nob( + &now_prev.time, + (erts_aint64_t) ERTS_MONOTONIC_TO_USEC(get_time_offset())); - previous_now = ERTS_MONOTONIC_TO_USEC(get_time_offset()); #ifdef DEBUG time_sup_initialized = 1; @@ -1289,36 +1312,65 @@ erts_finalize_time_offset(void) /* info functions */ void -elapsed_time_both(ErtsMonotonicTime *ms_user, ErtsMonotonicTime *ms_sys, - ErtsMonotonicTime *ms_user_diff, ErtsMonotonicTime *ms_sys_diff) +erts_runtime_elapsed_both(ErtsMonotonicTime *ms_user, ErtsMonotonicTime *ms_sys, + ErtsMonotonicTime *ms_user_diff, ErtsMonotonicTime *ms_sys_diff) { - ErtsMonotonicTime prev_total_user, prev_total_sys; - ErtsMonotonicTime total_user, total_sys; + ErtsMonotonicTime prev_user, prev_sys, user, sys; + +#ifdef HAVE_GETRUSAGE + + struct rusage now; + + if (getrusage(RUSAGE_SELF, &now) != 0) { + erts_exit(ERTS_ABORT_EXIT, "getrusage(RUSAGE_SELF, _) failed: %d\n", errno); + return; + } + + user = (ErtsMonotonicTime) now.ru_utime.tv_sec; + user *= (ErtsMonotonicTime) 1000000; + user += (ErtsMonotonicTime) now.ru_utime.tv_usec; + user /= (ErtsMonotonicTime) 1000; + + sys = (ErtsMonotonicTime) now.ru_stime.tv_sec; + sys *= (ErtsMonotonicTime) 1000000; + sys += (ErtsMonotonicTime) now.ru_stime.tv_usec; + sys /= (ErtsMonotonicTime) 1000; + +#else + SysTimes now; sys_times(&now); - total_user = (ErtsMonotonicTime) ((now.tms_utime * 1000) / SYS_CLK_TCK); - total_sys = (ErtsMonotonicTime) ((now.tms_stime * 1000) / SYS_CLK_TCK); + user = (ErtsMonotonicTime) now.tms_utime; + user *= (ErtsMonotonicTime) 1000; + user /= (ErtsMonotonicTime) SYS_CLK_TCK; - if (ms_user != NULL) - *ms_user = total_user; - if (ms_sys != NULL) - *ms_sys = total_sys; + sys = (ErtsMonotonicTime) now.tms_stime; + sys *= (ErtsMonotonicTime) 1000; + sys /= (ErtsMonotonicTime) SYS_CLK_TCK; + +#endif + + if (ms_user) + *ms_user = user; + if (ms_sys) + *ms_sys = sys; if (ms_user_diff || ms_sys_diff) { - erts_mtx_lock(&erts_timeofday_mtx); - - prev_total_user = (ErtsMonotonicTime) ((t_start.tms_utime * 1000) / SYS_CLK_TCK); - prev_total_sys = (ErtsMonotonicTime) ((t_start.tms_stime * 1000) / SYS_CLK_TCK); - t_start = now; - - erts_mtx_unlock(&erts_timeofday_mtx); + + erts_mtx_lock(&runtime_prev.data.mtx); - if (ms_user_diff != NULL) - *ms_user_diff = total_user - prev_total_user; - - if (ms_sys_diff != NULL) - *ms_sys_diff = total_sys - prev_total_sys; + prev_user = runtime_prev.data.user; + prev_sys = runtime_prev.data.sys; + runtime_prev.data.user = user; + runtime_prev.data.sys = sys; + + erts_mtx_unlock(&runtime_prev.data.mtx); + + if (ms_user_diff) + *ms_user_diff = user - prev_user; + if (ms_sys_diff) + *ms_sys_diff = sys - prev_sys; } } @@ -1326,7 +1378,7 @@ elapsed_time_both(ErtsMonotonicTime *ms_user, ErtsMonotonicTime *ms_sys, /* wall clock routines */ void -wall_clock_elapsed_time_both(ErtsMonotonicTime *ms_total, ErtsMonotonicTime *ms_diff) +erts_wall_clock_elapsed_both(ErtsMonotonicTime *ms_total, ErtsMonotonicTime *ms_diff) { ErtsMonotonicTime now, elapsed; @@ -1334,16 +1386,18 @@ wall_clock_elapsed_time_both(ErtsMonotonicTime *ms_total, ErtsMonotonicTime *ms_ update_last_mtime(NULL, now); elapsed = ERTS_MONOTONIC_TO_MSEC(now); + elapsed -= ERTS_MONOTONIC_TO_MSEC(ERTS_MONOTONIC_BEGIN); *ms_total = elapsed; if (ms_diff) { - erts_mtx_lock(&erts_timeofday_mtx); + ErtsMonotonicTime prev; - *ms_diff = elapsed - prev_wall_clock_elapsed; - prev_wall_clock_elapsed = elapsed; + prev = ((ErtsMonotonicTime) + erts_atomic64_xchg_mb(&wall_clock_prev.time, + (erts_aint64_t) elapsed)); - erts_mtx_unlock(&erts_timeofday_mtx); + *ms_diff = elapsed - prev; } } @@ -1722,22 +1776,27 @@ univ_to_local(Sint *year, Sint *month, Sint *day, void get_now(Uint* megasec, Uint* sec, Uint* microsec) { - ErtsMonotonicTime now_megasec, now_sec, now, mtime, time_offset; + ErtsMonotonicTime now_megasec, now_sec, now, prev, mtime, time_offset; mtime = time_sup.r.o.get_time(); time_offset = get_time_offset(); update_last_mtime(NULL, mtime); now = ERTS_MONOTONIC_TO_USEC(mtime + time_offset); - erts_mtx_lock(&erts_timeofday_mtx); - /* Make sure now time is later than last time */ - if (now <= previous_now) - now = previous_now + 1; - - previous_now = now; - - erts_mtx_unlock(&erts_timeofday_mtx); + prev = erts_atomic64_read_nob(&now_prev.time); + while (1) { + ErtsMonotonicTime act; + if (now <= prev) + now = prev + 1; + act = ((ErtsMonotonicTime) + erts_atomic64_cmpxchg_mb(&now_prev.time, + (erts_aint64_t) now, + (erts_aint64_t) prev)); + if (act == prev) + break; + prev = act; + } now_megasec = now / ERTS_MONOTONIC_TIME_TERA; now_sec = now / ERTS_MONOTONIC_TIME_MEGA; diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index a07e3642f6..4b996d8fc2 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1449,7 +1449,6 @@ monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time) case ERTS_PORT_TASK_TIMEOUT: op = am_timeout; break; case ERTS_PORT_TASK_INPUT: op = am_input; break; case ERTS_PORT_TASK_OUTPUT: op = am_output; break; - case ERTS_PORT_TASK_EVENT: op = am_event; break; case ERTS_PORT_TASK_DIST_CMD: op = am_dist_cmd; break; default: op = am_undefined; break; } diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index 2d1d1443a7..b7a5c45fea 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -1988,7 +1988,7 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbu is_list(name) || (allow_empty && is_nil(name))) { Sint need; - if ((need = erts_native_filename_need(name,encoding)) < 0) { + if ((need = erts_native_filename_need(name, encoding)) < 0) { return NULL; } if (encoding == ERL_FILENAME_WIN_WCHAR) { @@ -2152,12 +2152,13 @@ Eterm erts_convert_native_to_filename(Process *p, byte *bytes) } -Sint erts_native_filename_need(Eterm ioterm, int encoding) +Sint erts_native_filename_need(Eterm ioterm, int encoding) { Eterm *objp; Eterm obj; DECLARE_ESTACK(stack); Sint need = 0; + int seen_null = 0; if (is_atom(ioterm)) { Atom* ap; @@ -2194,6 +2195,22 @@ Sint erts_native_filename_need(Eterm ioterm, int encoding) default: need = -1; } + /* + * Do not allow null in + * the middle of filenames + */ + if (need > 0) { + byte *name = ap->name; + int len = ap->len; + for (i = 0; i < len; i++) { + if (name[i] == 0) + seen_null = 1; + else if (seen_null) { + need = -1; + break; + } + } + } DESTROY_ESTACK(stack); return need; } @@ -2224,6 +2241,16 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */ if (is_small(obj)) { /* Always small */ for(;;) { Uint x = unsigned_val(obj); + /* + * Do not allow null in + * the middle of filenames + */ + if (x == 0) + seen_null = 1; + else if (seen_null) { + DESTROY_ESTACK(stack); + return ((Sint) -1); + } switch (encoding) { case ERL_FILENAME_LATIN1: if (x > 255) { @@ -2497,6 +2524,38 @@ void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars) } /* + * *** Requirements on Raw Filename Format *** + * + * These requirements are due to the 'filename' module + * in stdlib. This since it is documented that it + * should be able to operate on raw filenames as well + * as ordinary filenames. + * + * A raw filename *must* be a byte sequence where: + * 1. Codepoints 0-127 (7-bit ascii) *must* be encoded + * as a byte with the corresponding value. That is, + * the most significant bit in the byte encoding the + * codepoint is never set. + * 2. Codepoints greater than 127 *must* be encoded + * with the most significant bit set in *every* byte + * encoding it. + * + * Latin1 and UTF-8 meet these requirements while + * UTF-16 and UTF-32 don't. + * + * On Windows filenames are natively stored as malformed + * UTF-16LE (lonely surrogates may appear). A more correct + * description than UTF-16 would be an array of 16-bit + * words... In order to meet the requirements of the + * raw file format we convert the malformed UTF-16LE to + * malformed UTF-8 which meet the requirements. + * + * Note that these requirements are today only OTP + * internal (erts-stdlib internal) requirements that + * could be changed. + */ + +/* * This internal bif converts a filename to whatever format is suitable for the file driver * It also adds zero termination so that prim_file needn't bother with the character encoding * of the file driver @@ -2507,6 +2566,12 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1) Sint need; Eterm bin_term; byte* bin_p; + + /* + * See comment on "Requirements on Raw Filename Format" + * above. + */ + /* Prim file explicitly does not allow atoms, although we could very well cope with it. Instead of letting 'file' handle them, it would probably be more efficient to handle them here. Subject to @@ -2515,6 +2580,7 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1) BIF_ERROR(BIF_P,BADARG); } if (is_binary(BIF_ARG_1)) { + int seen_null = 0; byte *temp_alloc = NULL; byte *bytes; byte *err_pos; @@ -2524,10 +2590,18 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1) size = binary_size(BIF_ARG_1); bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc); if (encoding != ERL_FILENAME_WIN_WCHAR) { + Uint i; /*Add 0 termination only*/ bin_term = new_binary(BIF_P, NULL, size+1); bin_p = binary_bytes(bin_term); - memcpy(bin_p,bytes,size); + for (i = 0; i < size; i++) { + /* Don't allow null in the middle of filenames... */ + if (bytes[i] == 0) + seen_null = 1; + else if (seen_null) + goto bin_name_error; + bin_p[i] = bytes[i]; + } bin_p[size]=0; erts_free_aligned_binary_bytes(temp_alloc); BIF_RET(bin_term); @@ -2541,6 +2615,11 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1) bin_term = new_binary(BIF_P, 0, (size+1)*2); bin_p = binary_bytes(bin_term); while (size--) { + /* Don't allow null in the middle of filenames... */ + if (*bytes == 0) + seen_null = 1; + else if (seen_null) + goto bin_name_error; *bin_p++ = *bytes++; *bin_p++ = 0; } @@ -2558,11 +2637,14 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1) bin_p[num_chars*2+1] = 0; erts_free_aligned_binary_bytes(temp_alloc); BIF_RET(bin_term); + bin_name_error: + erts_free_aligned_binary_bytes(temp_alloc); + BIF_ERROR(BIF_P,BADARG); } /* binary */ - if ((need = erts_native_filename_need(BIF_ARG_1,encoding)) < 0) { - BIF_ERROR(BIF_P,BADARG); + if ((need = erts_native_filename_need(BIF_ARG_1, encoding)) < 0) { + BIF_ERROR(BIF_P,BADARG); } if (encoding == ERL_FILENAME_WIN_WCHAR) { need += 2; @@ -2596,6 +2678,11 @@ BIF_RETTYPE prim_file_internal_native2name_1(BIF_ALIST_1) Eterm ret; int mac = 0; + /* + * See comment on "Requirements on Raw Filename Format" + * above. + */ + if (is_not_binary(BIF_ARG_1)) { BIF_ERROR(BIF_P,BADARG); } diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index f2d0af64df..76980b5871 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -46,8 +46,6 @@ */ #define ERTS_X_REGS_ALLOCATED (MAX_REG+3) -#define INPUT_REDUCTIONS (2 * CONTEXT_REDS) - #define H_DEFAULT_SIZE 233 /* default (heap + stack) min size */ #define VH_DEFAULT_SIZE 32768 /* default virtual (bin) heap min size (words) */ #define H_DEFAULT_MAX_SIZE 0 /* default max heap size is off */ @@ -159,6 +157,7 @@ typedef struct op_entry { Uint32 mask[3]; /* Signature mask. */ unsigned involves_r; /* Needs special attention when matching. */ int sz; /* Number of loaded words. */ + int adjust; /* Adjustment for start of instruction. */ char* pack; /* Instructions for packing engine. */ char* sign; /* Signature string. */ } OpEntry; @@ -201,11 +200,24 @@ extern int erts_pd_initial_size;/* Initial Process dictionary table size */ #include "erl_term.h" -#ifdef NO_JUMP_TABLE -#define BeamOp(Op) (Op) +#if defined(NO_JUMP_TABLE) +# define BeamOpsAreInitialized() (1) +# define BeamOpCodeAddr(OpCode) ((BeamInstr)(OpCode)) #else extern void** beam_ops; -#define BeamOp(Op) beam_ops[(Op)] +# define BeamOpsAreInitialized() (beam_ops != 0) +# define BeamOpCodeAddr(OpCode) ((BeamInstr)beam_ops[(OpCode)]) #endif +#if defined(ARCH_64) && defined(CODE_MODEL_SMALL) +# define BeamCodeAddr(InstrWord) ((BeamInstr)(Uint32)(InstrWord)) +# define BeamSetCodeAddr(InstrWord, Addr) (((InstrWord) & ~((1ull << 32)-1)) | (Addr)) +# define BeamExtraData(InstrWord) ((InstrWord) >> 32) +#else +# define BeamCodeAddr(InstrWord) ((BeamInstr)(InstrWord)) +# define BeamSetCodeAddr(InstrWord, Addr) (Addr) +#endif + +#define BeamIsOpCode(InstrWord, OpCode) (BeamCodeAddr(InstrWord) == BeamOpCodeAddr(OpCode)) + #endif /* __ERL_VM_H__ */ diff --git a/erts/emulator/beam/erlang_lttng.h b/erts/emulator/beam/erlang_lttng.h index 4e869671f7..feb05f4f4c 100644 --- a/erts/emulator/beam/erlang_lttng.h +++ b/erts/emulator/beam/erlang_lttng.h @@ -159,21 +159,6 @@ TRACEPOINT_EVENT( TRACEPOINT_EVENT( org_erlang_otp, - driver_event, - TP_ARGS( - char*, pid, - char*, port, - char*, driver - ), - TP_FIELDS( - ctf_string(pid, pid) - ctf_string(port, port) - ctf_string(driver, driver) - ) -) - -TRACEPOINT_EVENT( - org_erlang_otp, driver_timeout, TP_ARGS( char*, pid, diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index c81503f722..946ffeffb8 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -48,7 +48,6 @@ static erts_atomic_t total_entries_bytes; */ erts_mtx_t export_staging_lock; -extern BeamInstr* em_call_error_handler; extern BeamInstr* em_call_traced_function; struct export_entry @@ -130,7 +129,10 @@ export_alloc(struct export_entry* tmpl_e) obj->info.mfa.module = tmpl->info.mfa.module; obj->info.mfa.function = tmpl->info.mfa.function; obj->info.mfa.arity = tmpl->info.mfa.arity; - obj->beam[0] = (BeamInstr) em_call_error_handler; + obj->beam[0] = 0; + if (BeamOpsAreInitialized()) { + obj->beam[0] = BeamOpCodeAddr(op_call_error_handler); + } obj->beam[1] = 0; for (ix=0; ix<ERTS_NUM_CODE_IX; ix++) { @@ -267,7 +269,7 @@ erts_find_function(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix) ee = hash_get(&export_tables[code_ix].htable, init_template(&templ, m, f, a)); if (ee == NULL || (ee->ep->addressv[code_ix] == ee->ep->beam && - ee->ep->beam[0] != (BeamInstr) BeamOp(op_i_generic_breakpoint))) { + ! BeamIsOpCode(ee->ep->beam[0], op_i_generic_breakpoint))) { return NULL; } return ee->ep; diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h index be6cce07bf..194e514b12 100644 --- a/erts/emulator/beam/export.h +++ b/erts/emulator/beam/export.h @@ -73,7 +73,7 @@ extern erts_mtx_t export_staging_lock; #include "beam_load.h" /* For em_* extern declarations */ #define ExportIsBuiltIn(EntryPtr) \ (((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->beam) && \ - ((EntryPtr)->beam[0] == (BeamInstr) em_apply_bif)) + (BeamIsOpCode((EntryPtr)->beam[0], op_apply_bif))) #if ERTS_GLB_INLINE_INCL_FUNC_DEF diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 0874be7250..970158933f 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -616,7 +616,7 @@ erts_make_dist_ext_copy(ErtsDistExternal *edep, Uint xsize) sys_memcpy((void *) ep, (void *) edep, dist_ext_sz); ep += dist_ext_sz; if (new_edep->dep) - erts_refc_inc(&new_edep->dep->refc, 1); + erts_ref_dist_entry(new_edep->dep); new_edep->extp = ep; new_edep->ext_endp = ep + ext_sz; new_edep->heap_size = -1; @@ -629,7 +629,8 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, byte *ext, Uint size, DistEntry *dep, - ErtsAtomCache *cache) + ErtsAtomCache *cache, + Uint32 *connection_id) { #undef ERTS_EXT_FAIL #undef ERTS_EXT_HDR_FAIL @@ -650,33 +651,36 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, if (size < 2) ERTS_EXT_FAIL; + if (!dep) + ERTS_INTERNAL_ERROR("Invalid use"); + if (ep[0] != VERSION_MAGIC) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - if (dep) - erts_dsprintf(dsbufp, - "** Got message from incompatible erlang on " - "channel %d\n", - dist_entry_channel_no(dep)); - else - erts_dsprintf(dsbufp, - "** Attempt to convert old incompatible " - "binary %d\n", - *ep); + erts_dsprintf(dsbufp, + "** Got message from incompatible erlang on " + "channel %d\n", + dist_entry_channel_no(dep)); erts_send_error_to_logger_nogl(dsbufp); ERTS_EXT_FAIL; } edep->flags = 0; edep->dep = dep; - if (dep) { - erts_de_rlock(dep); - if (dep->flags & DFLAG_DIST_HDR_ATOM_CACHE) - edep->flags |= ERTS_DIST_EXT_DFLAG_HDR; - - edep->flags |= (dep->connection_id & ERTS_DIST_EXT_CON_ID_MASK); - erts_de_runlock(dep); + + erts_de_rlock(dep); + + if ((dep->status & (ERTS_DE_SFLG_EXITING|ERTS_DE_SFLG_CONNECTED)) + != ERTS_DE_SFLG_CONNECTED) { + erts_de_runlock(dep); + return ERTS_PREP_DIST_EXT_CLOSED; } + if (dep->flags & DFLAG_DIST_HDR_ATOM_CACHE) + edep->flags |= ERTS_DIST_EXT_DFLAG_HDR; + + *connection_id = dep->connection_id; + edep->flags |= (dep->connection_id & ERTS_DIST_EXT_CON_ID_MASK); + if (ep[1] != DIST_HEADER) { if (edep->flags & ERTS_DIST_EXT_DFLAG_HDR) ERTS_EXT_HDR_FAIL; @@ -835,14 +839,15 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, ERTS_EXT_FAIL; #endif - return 0; + erts_de_runlock(dep); + + return ERTS_PREP_DIST_EXT_SUCCESS; #undef CHKSIZE #undef ERTS_EXT_FAIL #undef ERTS_EXT_HDR_FAIL - bad_hdr: - if (dep) { + bad_hdr: { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "%T got a corrupted distribution header from %T " @@ -855,10 +860,11 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, erts_dsprintf(dsbufp, ">>"); erts_send_warning_to_logger_nogl(dsbufp); } - fail: - if (dep) - erts_kill_dist_connection(dep, dep->connection_id); - return -1; + fail: { + erts_de_runlock(dep); + erts_kill_dist_connection(dep, *connection_id); + } + return ERTS_PREP_DIST_EXT_FAILED; } static void @@ -1923,7 +1929,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla } result_bin = erts_bin_nrml_alloc(size); - result_bin->orig_bytes[0] = VERSION_MAGIC; + result_bin->orig_bytes[0] = (byte)VERSION_MAGIC; /* Next state immediately, no need to export context */ context->state = TTBEncode; context->s.ec.flags = flags; @@ -1981,7 +1987,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla context->s.cc.result_bin = result_bin; result_bin = erts_bin_nrml_alloc(real_size); - result_bin->orig_bytes[0] = VERSION_MAGIC; + result_bin->orig_bytes[0] = (byte) VERSION_MAGIC; context->s.cc.destination_bin = result_bin; context->s.cc.dest_len = 0; @@ -3109,7 +3115,7 @@ dec_term(ErtsDistExternal *edep, #if defined(ARCH_64) *objp = make_small(sn); #else - if (MY_IS_SSMALL(sn)) { + if (IS_SSMALL(sn)) { *objp = make_small(sn); } else { *objp = small_to_big(sn, hp); diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index f00426cc16..3c61d013da 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -185,8 +185,13 @@ ERTS_GLB_INLINE void *erts_dist_ext_trailer(ErtsDistExternal *); ErtsDistExternal *erts_make_dist_ext_copy(ErtsDistExternal *, Uint); void *erts_dist_ext_trailer(ErtsDistExternal *); void erts_destroy_dist_ext_copy(ErtsDistExternal *); + +#define ERTS_PREP_DIST_EXT_FAILED (-1) +#define ERTS_PREP_DIST_EXT_SUCCESS (0) +#define ERTS_PREP_DIST_EXT_CLOSED (1) + int erts_prepare_dist_ext(ErtsDistExternal *, byte *, Uint, - DistEntry *, ErtsAtomCache *); + DistEntry *, ErtsAtomCache *, Uint32 *); Sint erts_decode_dist_ext_size(ErtsDistExternal *); Eterm erts_decode_dist_ext(ErtsHeapFactory* factory, ErtsDistExternal *); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 7cb94ba3d2..09aeba00fa 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -128,10 +128,8 @@ extern Eterm erts_nif_call_function(Process *p, Process *tracee, struct enif_func_t *, int argc, Eterm *argv); -#ifdef ERTS_DIRTY_SCHEDULERS int erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *reg); -#endif /* ERTS_DIRTY_SCHEDULERS */ /* Driver handle (wrapper for old plain handle) */ @@ -224,8 +222,6 @@ struct erts_driver_t_ { char *buf, ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen, /* Might be NULL */ unsigned int *flags); - void (*event)(ErlDrvData drv_data, ErlDrvEvent event, - ErlDrvEventData event_data); void (*ready_input)(ErlDrvData drv_data, ErlDrvEvent event); void (*ready_output)(ErlDrvData drv_data, ErlDrvEvent event); void (*timeout)(ErlDrvData drv_data); @@ -911,9 +907,7 @@ extern erts_atomic_t erts_copy_literal_area__; #define ERTS_COPY_LITERAL_AREA() \ ((ErtsLiteralArea *) erts_atomic_read_nob(&erts_copy_literal_area__)) extern Process *erts_literal_area_collector; -#ifdef ERTS_DIRTY_SCHEDULERS extern Process *erts_dirty_process_code_checker; -#endif extern Process *erts_code_purger; @@ -1107,7 +1101,6 @@ void erts_save_stacktrace(Process* p, struct StackTrace* s, int depth); typedef struct { Eterm delay_time; int context_reds; - int input_reds; } ErtsModifiedTimings; extern Export *erts_delay_trap; @@ -1280,7 +1273,8 @@ int erts_utf8_to_latin1(byte* dest, const byte* source, int slen); void bin_write(fmtfn_t, void*, byte*, size_t); Sint intlist_to_buf(Eterm, char*, Sint); /* most callers pass plain char*'s */ -Sint erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len); +int erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len, Sint* written); +Sint erts_unicode_list_to_buf_len(Eterm list); struct Sint_buf { #if defined(ARCH_64) @@ -1385,7 +1379,7 @@ Uint erts_current_reductions(Process* current, Process *p); int erts_print_system_version(fmtfn_t to, void *arg, Process *c_p); -int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg); +int erts_hibernate(Process* c_p, Eterm* reg); ERTS_GLB_FORCE_INLINE int erts_is_literal(Eterm tptr, Eterm *ptr); diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab index d45da62d03..07cc4bd527 100644 --- a/erts/emulator/beam/instrs.tab +++ b/erts/emulator/beam/instrs.tab @@ -78,7 +78,14 @@ move_deallocate_return(Src, Deallocate) { // Call instructions -DISPATCH(CallDest) { +DISPATCH_REL(CallDest) { + //| -no_next + $SET_I_REL($CallDest); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); + Dispatch(); +} + +DISPATCH_ABS(CallDest) { //| -no_next SET_I((BeamInstr *) $CallDest); DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); @@ -87,18 +94,18 @@ DISPATCH(CallDest) { i_call(CallDest) { SET_CP(c_p, $NEXT_INSTRUCTION); - $DISPATCH($CallDest); + $DISPATCH_REL($CallDest); } move_call(Src, CallDest) { x(0) = $Src; SET_CP(c_p, $NEXT_INSTRUCTION); - $DISPATCH($CallDest); + $DISPATCH_REL($CallDest); } i_call_last(CallDest, Deallocate) { $deallocate($Deallocate); - $DISPATCH($CallDest); + $DISPATCH_REL($CallDest); } move_call_last(Src, CallDest, Deallocate) { @@ -107,18 +114,14 @@ move_call_last(Src, CallDest, Deallocate) { } i_call_only(CallDest) { - $DISPATCH($CallDest); + $DISPATCH_REL($CallDest); } -i_move_call_only(CallDest, Src) { +move_call_only(Src, CallDest) { x(0) = $Src; $i_call_only($CallDest); } -move_call_only(Src, CallDest) { - $i_move_call_only($CallDest, $Src); -} - DISPATCHX(Dest) { //| -no_next DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, $Dest); @@ -156,10 +159,10 @@ i_move_call_ext_last(Dest, StackOffset, Src) { $i_call_ext_last($Dest, $StackOffset); } -APPLY(I, Deallocate) { +APPLY(I, Deallocate, Next) { //| -no_next HEAVY_SWAPOUT; - next = apply(c_p, r(0), x(1), x(2), reg, $I, $Deallocate); + $Next = apply(c_p, reg, $I, $Deallocate); HEAVY_SWAPIN; } @@ -170,59 +173,63 @@ HANDLE_APPLY_ERROR() { i_apply() { BeamInstr *next; - $APPLY(NULL, 0); - if (next != NULL) { - $i_call(next); + $APPLY(NULL, 0, next); + if (ERTS_LIKELY(next != NULL)) { + SET_CP(c_p, $NEXT_INSTRUCTION); + $DISPATCH_ABS(next); } $HANDLE_APPLY_ERROR(); } i_apply_last(Deallocate) { BeamInstr *next; - $APPLY(I, $Deallocate); - if (next != NULL) { - $i_call_last(next, $Deallocate); + $APPLY(I, $Deallocate, next); + if (ERTS_LIKELY(next != NULL)) { + $deallocate($Deallocate); + $DISPATCH_ABS(next); } $HANDLE_APPLY_ERROR(); } i_apply_only() { BeamInstr *next; - $APPLY(I, 0); - if (next != NULL) { - $i_call_only(next); + $APPLY(I, 0, next); + if (ERTS_LIKELY(next != NULL)) { + $DISPATCH_ABS(next); } $HANDLE_APPLY_ERROR(); } -FIXED_APPLY(Arity, I, Deallocate) { +FIXED_APPLY(Arity, I, Deallocate, Next) { //| -no_next HEAVY_SWAPOUT; - next = fixed_apply(c_p, reg, $Arity, $I, $Deallocate); + $Next = fixed_apply(c_p, reg, $Arity, $I, $Deallocate); HEAVY_SWAPIN; } apply(Arity) { BeamInstr *next; - $FIXED_APPLY($Arity, NULL, 0); - if (next != NULL) { - $i_call(next); + $FIXED_APPLY($Arity, NULL, 0, next); + if (ERTS_LIKELY(next != NULL)) { + SET_CP(c_p, $NEXT_INSTRUCTION); + $DISPATCH_ABS(next); } $HANDLE_APPLY_ERROR(); } apply_last(Arity, Deallocate) { BeamInstr *next; - $FIXED_APPLY($Arity, I, $Deallocate); - if (next != NULL) { - $i_call_last(next, $Deallocate); + $FIXED_APPLY($Arity, I, $Deallocate, next); + if (ERTS_LIKELY(next != NULL)) { + $deallocate($Deallocate); + $DISPATCH_ABS(next); } $HANDLE_APPLY_ERROR(); } -APPLY_FUN() { +APPLY_FUN(Next) { HEAVY_SWAPOUT; - next = apply_fun(c_p, r(0), x(1), reg); + $Next = apply_fun(c_p, r(0), x(1), reg); HEAVY_SWAPIN; } @@ -237,8 +244,8 @@ DISPATCH_FUN(I) { i_apply_fun() { BeamInstr *next; - $APPLY_FUN(); - if (next != NULL) { + $APPLY_FUN(next); + if (ERTS_LIKELY(next != NULL)) { SET_CP(c_p, $NEXT_INSTRUCTION); $DISPATCH_FUN(next); } @@ -247,8 +254,8 @@ i_apply_fun() { i_apply_fun_last(Deallocate) { BeamInstr *next; - $APPLY_FUN(); - if (next != NULL) { + $APPLY_FUN(next); + if (ERTS_LIKELY(next != NULL)) { $deallocate($Deallocate); $DISPATCH_FUN(next); } @@ -257,24 +264,24 @@ i_apply_fun_last(Deallocate) { i_apply_fun_only() { BeamInstr *next; - $APPLY_FUN(); - if (next != NULL) { + $APPLY_FUN(next); + if (ERTS_LIKELY(next != NULL)) { $DISPATCH_FUN(next); } $HANDLE_APPLY_FUN_ERROR(); } -CALL_FUN(Fun) { +CALL_FUN(Fun, Next) { //| -no_next HEAVY_SWAPOUT; - next = call_fun(c_p, $Fun, reg, THE_NON_VALUE); + $Next = call_fun(c_p, $Fun, reg, THE_NON_VALUE); HEAVY_SWAPIN; } i_call_fun(Fun) { BeamInstr *next; - $CALL_FUN($Fun); - if (next != NULL) { + $CALL_FUN($Fun, next); + if (ERTS_LIKELY(next != NULL)) { SET_CP(c_p, $NEXT_INSTRUCTION); $DISPATCH_FUN(next); } @@ -283,8 +290,8 @@ i_call_fun(Fun) { i_call_fun_last(Fun, Deallocate) { BeamInstr *next; - $CALL_FUN($Fun); - if (next != NULL) { + $CALL_FUN($Fun, next); + if (ERTS_LIKELY(next != NULL)) { $deallocate($Deallocate); $DISPATCH_FUN(next); } @@ -368,7 +375,6 @@ i_element := element_group.fetch.execute; element_group.head() { - Eterm element_index; Eterm element_tuple; } @@ -377,8 +383,8 @@ element_group.fetch(Src) { } element_group.execute(Fail, Index, Dst) { - element_index = $Index; - if (is_small(element_index) && is_tuple(element_tuple)) { + Eterm element_index = $Index; + if (ERTS_LIKELY(is_small(element_index) && is_tuple(element_tuple))) { Eterm* tp = tuple_val(element_tuple); if ((signed_val(element_index) >= 1) && @@ -402,7 +408,7 @@ fast_element_group.fetch(Src) { } fast_element_group.execute(Fail, Index, Dst) { - if (is_tuple(fast_element_tuple)) { + if (ERTS_LIKELY(is_tuple(fast_element_tuple))) { Eterm* tp = tuple_val(fast_element_tuple); Eterm pos = $Index; /* Untagged integer >= 1 */ if (pos <= arityval(*tp)) { @@ -564,6 +570,7 @@ i_put_tuple.fill(Arity) { } } while (--arity != 0); HTOP = hp; + ASSERT(VALID_INSTR(* (Eterm *)I)); Goto(*I); } @@ -735,9 +742,10 @@ is_reference(Fail, Src) { } is_tagged_tuple(Fail, Src, Arityval, Tag) { - if (!(BEAM_IS_TUPLE($Src) && - (tuple_val($Src))[0] == $Arityval && - (tuple_val($Src))[1] == $Tag)) { + Eterm term = $Src; + if (!(BEAM_IS_TUPLE(term) && + (tuple_val(term))[0] == $Arityval && + (tuple_val(term))[1] == $Tag)) { $FAIL($Fail); } } @@ -749,7 +757,8 @@ is_tuple(Fail, Src) { } is_tuple_of_arity(Fail, Src, Arityval) { - if (!(BEAM_IS_TUPLE($Src) && *tuple_val($Src) == $Arityval)) { + Eterm term = $Src; + if (!(BEAM_IS_TUPLE(term) && *tuple_val(term) == $Arityval)) { $FAIL($Fail); } } @@ -814,18 +823,21 @@ is_ge(Fail, X, Y) { badarg(Fail) { $BADARG($Fail); + //| -no_next; } badmatch(Src) { c_p->fvalue = $Src; c_p->freason = BADMATCH; goto find_func_info; + //| -no_next; } case_end(Src) { c_p->fvalue = $Src; c_p->freason = EXC_CASE_CLAUSE; goto find_func_info; + //| -no_next; } if_end() { @@ -905,5 +917,6 @@ i_raise() { c_p->freason = PRIMARY_EXCEPTION(s->freason); } goto find_func_info; + //| -no_next } diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index c8925e159e..85013af3ad 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -52,6 +52,7 @@ #include "erl_bif_unique.h" #include "erl_hl_timer.h" #include "erl_time.h" +#include "erl_io_queue.h" extern ErlDrvEntry fd_driver_entry; extern ErlDrvEntry vanilla_driver_entry; @@ -102,7 +103,7 @@ static void driver_monitor_unlock_pdl(Port *p); #define ERL_SMALL_IO_BIN_LIMIT (4*ERL_ONHEAP_BIN_LIMIT) #define SMALL_WRITE_VEC 16 -static ERTS_INLINE ErlIOQueue* +static ERTS_INLINE ErlPortIOQueue* drvport2ioq(ErlDrvPort drvport) { Port *prt = erts_thr_drvport2port(drvport, 0); @@ -117,11 +118,11 @@ is_port_ioq_empty(Port *pp) int res; ERTS_LC_ASSERT(erts_lc_is_port_locked(pp)); if (!pp->port_data_lock) - res = (pp->ioq.size == 0); + res = (erts_ioq_size(&pp->ioq) == 0); else { ErlDrvPDL pdl = pp->port_data_lock; erts_mtx_lock(&pdl->mtx); - res = (pp->ioq.size == 0); + res = (erts_ioq_size(&pp->ioq) == 0); erts_mtx_unlock(&pdl->mtx); } return res; @@ -136,14 +137,14 @@ erts_is_port_ioq_empty(Port *pp) Uint erts_port_ioq_size(Port *pp) { - int res; + ErlDrvSizeT res; ERTS_LC_ASSERT(erts_lc_is_port_locked(pp)); if (!pp->port_data_lock) - res = pp->ioq.size; + res = erts_ioq_size(&pp->ioq); else { ErlDrvPDL pdl = pp->port_data_lock; erts_mtx_lock(&pdl->mtx); - res = pp->ioq.size; + res = erts_ioq_size(&pp->ioq); erts_mtx_unlock(&pdl->mtx); } return (Uint) res; @@ -473,41 +474,17 @@ erts_port_free(Port *prt) */ static void initq(Port* prt) { - ErlIOQueue* q = &prt->ioq; - ERTS_LC_ASSERT(!prt->port_data_lock); - - q->size = 0; - q->v_head = q->v_tail = q->v_start = q->v_small; - q->v_end = q->v_small + SMALL_IO_QUEUE; - q->b_head = q->b_tail = q->b_start = q->b_small; - q->b_end = q->b_small + SMALL_IO_QUEUE; + erts_ioq_init(&prt->ioq, ERTS_ALC_T_IOQ, 1); } static void stopq(Port* prt) { - ErlIOQueue* q; - ErlDrvBinary** binp; if (prt->port_data_lock) driver_pdl_lock(prt->port_data_lock); - q = &prt->ioq; - binp = q->b_head; - - if (q->v_start != q->v_small) - erts_free(ERTS_ALC_T_IOQ, (void *) q->v_start); - - while(binp < q->b_tail) { - if (*binp != NULL) - driver_free_binary(*binp); - binp++; - } - if (q->b_start != q->b_small) - erts_free(ERTS_ALC_T_IOQ, (void *) q->b_start); - q->v_start = q->v_end = q->v_head = q->v_tail = NULL; - q->b_start = q->b_end = q->b_head = q->b_tail = NULL; - q->size = 0; + erts_ioq_clear(&prt->ioq); if (prt->port_data_lock) { driver_pdl_unlock(prt->port_data_lock); @@ -876,311 +853,6 @@ int erts_port_handle_xports(Port *prt) return reds; } -/* Fills a possibly deep list of chars and binaries into vec -** Small characters are first stored in the buffer buf of length ln -** binaries found are copied and linked into msoh -** Return vector length on succsess, -** -1 on overflow -** -2 on type error -*/ - -#ifdef DEBUG -#define MAX_SYSIOVEC_IOVLEN (1ull << (32 - 1)) -#else -#define MAX_SYSIOVEC_IOVLEN (1ull << (sizeof(((SysIOVec*)0)->iov_len) * 8 - 1)) -#endif - -static ERTS_INLINE void -io_list_to_vec_set_vec(SysIOVec **iov, ErlDrvBinary ***binv, - ErlDrvBinary *bin, byte *ptr, Uint len, - int *vlen) -{ - while (len > MAX_SYSIOVEC_IOVLEN) { - (*iov)->iov_base = ptr; - (*iov)->iov_len = MAX_SYSIOVEC_IOVLEN; - ptr += MAX_SYSIOVEC_IOVLEN; - len -= MAX_SYSIOVEC_IOVLEN; - (*iov)++; - (*vlen)++; - *(*binv)++ = bin; - } - (*iov)->iov_base = ptr; - (*iov)->iov_len = len; - *(*binv)++ = bin; - (*iov)++; - (*vlen)++; -} - -static int -io_list_to_vec(Eterm obj, /* io-list */ - SysIOVec* iov, /* io vector */ - ErlDrvBinary** binv, /* binary reference vector */ - ErlDrvBinary* cbin, /* binary to store characters */ - ErlDrvSizeT bin_limit) /* small binaries limit */ -{ - DECLARE_ESTACK(s); - Eterm* objp; - byte *buf = (byte*)cbin->orig_bytes; - Uint len = cbin->orig_size; - Uint csize = 0; - int vlen = 0; - byte* cptr = buf; - - goto L_jump_start; /* avoid push */ - - while (!ESTACK_ISEMPTY(s)) { - obj = ESTACK_POP(s); - L_jump_start: - if (is_list(obj)) { - L_iter_list: - objp = list_val(obj); - obj = CAR(objp); - if (is_byte(obj)) { - if (len == 0) - goto L_overflow; - *buf++ = unsigned_val(obj); - csize++; - len--; - } else if (is_binary(obj)) { - ESTACK_PUSH(s, CDR(objp)); - goto handle_binary; - } else if (is_list(obj)) { - ESTACK_PUSH(s, CDR(objp)); - goto L_iter_list; /* on head */ - } else if (!is_nil(obj)) { - goto L_type_error; - } - obj = CDR(objp); - if (is_list(obj)) - goto L_iter_list; /* on tail */ - else if (is_binary(obj)) { - goto handle_binary; - } else if (!is_nil(obj)) { - goto L_type_error; - } - } else if (is_binary(obj)) { - Eterm real_bin; - Uint offset; - Eterm* bptr; - ErlDrvSizeT size; - int bitoffs; - int bitsize; - - handle_binary: - size = binary_size(obj); - ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize); - ASSERT(bitsize == 0); - bptr = binary_val(real_bin); - if (*bptr == HEADER_PROC_BIN) { - ProcBin* pb = (ProcBin *) bptr; - if (bitoffs != 0) { - if (len < size) { - goto L_overflow; - } - erts_copy_bits(pb->bytes+offset, bitoffs, 1, - (byte *) buf, 0, 1, size*8); - csize += size; - buf += size; - len -= size; - } else if (bin_limit && size < bin_limit) { - if (len < size) { - goto L_overflow; - } - sys_memcpy(buf, pb->bytes+offset, size); - csize += size; - buf += size; - len -= size; - } else { - if (csize != 0) { - io_list_to_vec_set_vec(&iov, &binv, cbin, - cptr, csize, &vlen); - cptr = buf; - csize = 0; - } - if (pb->flags) { - erts_emasculate_writable_binary(pb); - } - io_list_to_vec_set_vec( - &iov, &binv, Binary2ErlDrvBinary(pb->val), - pb->bytes+offset, size, &vlen); - } - } else { - ErlHeapBin* hb = (ErlHeapBin *) bptr; - if (len < size) { - goto L_overflow; - } - copy_binary_to_buffer(buf, 0, - ((byte *) hb->data)+offset, bitoffs, - 8*size); - csize += size; - buf += size; - len -= size; - } - } else if (!is_nil(obj)) { - goto L_type_error; - } - } - - if (csize != 0) { - io_list_to_vec_set_vec(&iov, &binv, cbin, cptr, csize, &vlen); - } - - DESTROY_ESTACK(s); - return vlen; - - L_type_error: - DESTROY_ESTACK(s); - return -2; - - L_overflow: - DESTROY_ESTACK(s); - return -1; -} - -#define IO_LIST_VEC_COUNT(obj) \ -do { \ - Uint _size = binary_size(obj); \ - Eterm _real; \ - ERTS_DECLARE_DUMMY(Uint _offset); \ - int _bitoffs; \ - int _bitsize; \ - ERTS_GET_REAL_BIN(obj, _real, _offset, _bitoffs, _bitsize); \ - if (_bitsize != 0) goto L_type_error; \ - if (thing_subtag(*binary_val(_real)) == REFC_BINARY_SUBTAG && \ - _bitoffs == 0) { \ - b_size += _size; \ - if (b_size < _size) goto L_overflow_error; \ - in_clist = 0; \ - v_size++; \ - /* If iov_len is smaller then Uint we split the binary into*/ \ - /* multiple smaller (2GB) elements in the iolist.*/ \ - v_size += _size / MAX_SYSIOVEC_IOVLEN; \ - if (_size >= ERL_SMALL_IO_BIN_LIMIT) { \ - p_in_clist = 0; \ - p_v_size++; \ - } else { \ - p_c_size += _size; \ - if (!p_in_clist) { \ - p_in_clist = 1; \ - p_v_size++; \ - } \ - } \ - } else { \ - c_size += _size; \ - if (c_size < _size) goto L_overflow_error; \ - if (!in_clist) { \ - in_clist = 1; \ - v_size++; \ - } \ - p_c_size += _size; \ - if (!p_in_clist) { \ - p_in_clist = 1; \ - p_v_size++; \ - } \ - } \ -} while (0) - - -/* - * Returns 0 if successful and a non-zero value otherwise. - * - * Return values through pointers: - * *vsize - SysIOVec size needed for a writev - * *csize - Number of bytes not in binary (in the common binary) - * *pvsize - SysIOVec size needed if packing small binaries - * *pcsize - Number of bytes in the common binary if packing - * *total_size - Total size of iolist in bytes - */ - -static int -io_list_vec_len(Eterm obj, int* vsize, Uint* csize, - Uint* pvsize, Uint* pcsize, - ErlDrvSizeT* total_size) -{ - DECLARE_ESTACK(s); - Eterm* objp; - Uint v_size = 0; - Uint c_size = 0; - Uint b_size = 0; - Uint in_clist = 0; - Uint p_v_size = 0; - Uint p_c_size = 0; - Uint p_in_clist = 0; - Uint total; - - goto L_jump_start; /* avoid a push */ - - while (!ESTACK_ISEMPTY(s)) { - obj = ESTACK_POP(s); - L_jump_start: - if (is_list(obj)) { - L_iter_list: - objp = list_val(obj); - obj = CAR(objp); - - if (is_byte(obj)) { - c_size++; - if (c_size == 0) { - goto L_overflow_error; - } - if (!in_clist) { - in_clist = 1; - v_size++; - } - p_c_size++; - if (!p_in_clist) { - p_in_clist = 1; - p_v_size++; - } - } - else if (is_binary(obj)) { - IO_LIST_VEC_COUNT(obj); - } - else if (is_list(obj)) { - ESTACK_PUSH(s, CDR(objp)); - goto L_iter_list; /* on head */ - } - else if (!is_nil(obj)) { - goto L_type_error; - } - - obj = CDR(objp); - if (is_list(obj)) - goto L_iter_list; /* on tail */ - else if (is_binary(obj)) { /* binary tail is OK */ - IO_LIST_VEC_COUNT(obj); - } - else if (!is_nil(obj)) { - goto L_type_error; - } - } - else if (is_binary(obj)) { - IO_LIST_VEC_COUNT(obj); - } - else if (!is_nil(obj)) { - goto L_type_error; - } - } - - total = c_size + b_size; - if (total < c_size) { - goto L_overflow_error; - } - *total_size = (ErlDrvSizeT) total; - - DESTROY_ESTACK(s); - *vsize = v_size; - *csize = c_size; - *pvsize = p_v_size; - *pcsize = p_c_size; - return 0; - - L_type_error: - L_overflow_error: - DESTROY_ESTACK(s); - return 1; -} - typedef enum { ERTS_TRY_IMM_DRV_CALL_OK, ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK, @@ -1750,8 +1422,7 @@ cleanup_scheduled_outputv(ErlIOVec *ev, ErlDrvBinary *cbinp) int i; /* Need to free all binaries */ for (i = 1; i < ev->vsize; i++) - if (ev->binv[i]) - driver_free_binary(ev->binv[i]); + driver_free_binary(ev->binv[i]); if (cbinp) driver_free_binary(cbinp); } @@ -1919,15 +1590,14 @@ erts_port_output_async(Port *prt, Eterm from, Eterm list) size_t size; int task_flags; ErtsProc2PortSigCallback port_sig_callback; - ErlDrvBinary *cbin = NULL; - ErlIOVec *evp = NULL; + ErtsIOQBinary *cbin = NULL; + ErtsIOVec *evp = NULL; char *buf = NULL; ErtsPortTaskHandle *ns_pthp; if (drv->outputv) { - ErlIOVec ev; SysIOVec* ivp; - ErlDrvBinary** bvp; + ErtsIOQBinary** bvp; int vsize; Uint csize; Uint pvsize; @@ -1937,91 +1607,63 @@ erts_port_output_async(Port *prt, Eterm from, Eterm list) char *ptr; int i; - Eterm* bptr = NULL; - Uint offset; - - if (is_binary(list)) { - /* We optimize for when we get a procbin without offset */ - Eterm real_bin; - int bitoffs; - int bitsize; - ERTS_GET_REAL_BIN(list, real_bin, offset, bitoffs, bitsize); - bptr = binary_val(real_bin); - if (*bptr == HEADER_PROC_BIN && bitoffs == 0) { - size = binary_size(list); - vsize = 1; - } else - bptr = NULL; - } - - if (!bptr) { - if (io_list_vec_len(list, &vsize, &csize, &pvsize, &pcsize, &size)) - goto bad_value; + if (erts_ioq_iodata_vec_len(list, &vsize, &csize, &pvsize, &pcsize, + &size, ERL_SMALL_IO_BIN_LIMIT)) + goto bad_value; - /* To pack or not to pack (small binaries) ...? */ - if (vsize >= SMALL_WRITE_VEC) { - /* Do pack */ - vsize = pvsize + 1; - csize = pcsize; - blimit = ERL_SMALL_IO_BIN_LIMIT; - } - cbin = driver_alloc_binary(csize); + /* To pack or not to pack (small binaries) ...? */ + if (vsize >= SMALL_WRITE_VEC) { + /* Do pack */ + vsize = pvsize + 1; + csize = pcsize; + blimit = ERL_SMALL_IO_BIN_LIMIT; + } + if (csize) { + cbin = (ErtsIOQBinary *)driver_alloc_binary(csize); if (!cbin) erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, ERTS_SIZEOF_Binary(csize)); } - iov_offset = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErlIOVec)); binv_offset = iov_offset; binv_offset += ERTS_ALC_DATA_ALIGN_SIZE((vsize+1)*sizeof(SysIOVec)); alloc_size = binv_offset; - alloc_size += (vsize+1)*sizeof(ErlDrvBinary *); + alloc_size += (vsize+1)*sizeof(ErtsIOQBinary *); sigdp = erts_port_task_alloc_p2p_sig_data_extra(alloc_size, (void**)&ptr); - evp = (ErlIOVec *) ptr; - ivp = evp->iov = (SysIOVec *) (ptr + iov_offset); - bvp = evp->binv = (ErlDrvBinary **) (ptr + binv_offset); + evp = (ErtsIOVec *) ptr; + ivp = evp->driver.iov = (SysIOVec *) (ptr + iov_offset); + bvp = evp->common.binv = (ErtsIOQBinary **) (ptr + binv_offset); ivp[0].iov_base = NULL; ivp[0].iov_len = 0; bvp[0] = NULL; - if (bptr) { - ProcBin* pb = (ProcBin *) bptr; - - ivp[1].iov_base = pb->bytes+offset; - ivp[1].iov_len = size; - bvp[1] = Binary2ErlDrvBinary(pb->val); - - evp->vsize = 1; - } else { - - evp->vsize = io_list_to_vec(list, ivp+1, bvp+1, cbin, blimit); - if (evp->vsize < 0) { - if (evp != &ev) - erts_free(ERTS_ALC_T_DRV_CMD_DATA, evp); - driver_free_binary(cbin); - goto bad_value; - } + evp->driver.vsize = erts_ioq_iodata_to_vec(list, ivp+1, bvp+1, cbin, + blimit, 1); + if (evp->driver.vsize < 0) { + erts_free(ERTS_ALC_T_DRV_CMD_DATA, evp); + driver_free_binary(&cbin->driver); + goto bad_value; } #if 0 /* This assertion may say something useful, but it can be falsified during the emulator test suites. */ ASSERT(evp->vsize == vsize); #endif - evp->vsize++; - evp->size = size; /* total size */ + evp->driver.vsize++; + evp->driver.size = size; /* total size */ /* Need to increase refc on all binaries */ - for (i = 1; i < evp->vsize; i++) + for (i = 1; i < evp->driver.vsize; i++) if (bvp[i]) - driver_binary_inc_refc(bvp[i]); + driver_binary_inc_refc(&bvp[i]->driver); sigdp->flags = ERTS_P2P_SIG_TYPE_OUTPUTV; sigdp->u.outputv.from = from; - sigdp->u.outputv.evp = evp; - sigdp->u.outputv.cbinp = cbin; + sigdp->u.outputv.evp = &evp->driver; + sigdp->u.outputv.cbinp = &cbin->driver; port_sig_callback = port_sig_outputv; } else { ErlDrvSizeT ERTS_DECLARE_DUMMY(r); @@ -2092,8 +1734,8 @@ erts_port_output(Process *c_p, erts_aint32_t sched_flags, busy_flgs, invalid_flags; int task_flags; ErtsProc2PortSigCallback port_sig_callback; - ErlDrvBinary *cbin = NULL; - ErlIOVec *evp = NULL; + ErtsIOQBinary *cbin = NULL; + ErtsIOVec *evp = NULL; char *buf = NULL; int force_immediate_call = (flags & ERTS_PORT_SIG_FLG_FORCE_IMM_CALL); int async_nosuspend; @@ -2139,11 +1781,11 @@ erts_port_output(Process *c_p, } #endif if (drv->outputv) { - ErlIOVec ev; + ErtsIOVec ev; SysIOVec iv[SMALL_WRITE_VEC]; - ErlDrvBinary* bv[SMALL_WRITE_VEC]; + ErtsIOQBinary* bv[SMALL_WRITE_VEC]; SysIOVec* ivp; - ErlDrvBinary** bvp; + ErtsIOQBinary** bvp; int vsize; Uint csize; Uint pvsize; @@ -2151,18 +1793,19 @@ erts_port_output(Process *c_p, Uint blimit; size_t iov_offset, binv_offset, alloc_size; - if (io_list_vec_len(list, &vsize, &csize, &pvsize, &pcsize, &size)) + if (erts_ioq_iodata_vec_len(list, &vsize, &csize, &pvsize, &pcsize, + &size, ERL_SMALL_IO_BIN_LIMIT)) goto bad_value; iov_offset = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErlIOVec)); binv_offset = iov_offset; binv_offset += ERTS_ALC_DATA_ALIGN_SIZE((vsize+1)*sizeof(SysIOVec)); alloc_size = binv_offset; - alloc_size += (vsize+1)*sizeof(ErlDrvBinary *); + alloc_size += (vsize+1)*sizeof(ErtsIOQBinary *); if (try_call && vsize < SMALL_WRITE_VEC) { - ivp = ev.iov = iv; - bvp = ev.binv = bv; + ivp = ev.common.iov = iv; + bvp = ev.common.binv = bv; evp = &ev; } else { @@ -2173,9 +1816,9 @@ erts_port_output(Process *c_p, sigdp = erts_port_task_alloc_p2p_sig_data_extra( alloc_size, (void**)&ptr); } - evp = (ErlIOVec *) ptr; - ivp = evp->iov = (SysIOVec *) (ptr + iov_offset); - bvp = evp->binv = (ErlDrvBinary **) (ptr + binv_offset); + evp = (ErtsIOVec *) ptr; + ivp = evp->driver.iov = (SysIOVec *) (ptr + iov_offset); + bvp = evp->common.binv = (ErtsIOQBinary **) (ptr + binv_offset); } /* To pack or not to pack (small binaries) ...? */ @@ -2191,23 +1834,26 @@ erts_port_output(Process *c_p, } /* Use vsize and csize from now on */ - cbin = driver_alloc_binary(csize); - if (!cbin) - erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, ERTS_SIZEOF_Binary(csize)); + if (csize) { + cbin = (ErtsIOQBinary *)driver_alloc_binary(csize); + if (!cbin) + erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, ERTS_SIZEOF_Binary(csize)); + } /* Element 0 is for driver usage to add header block */ ivp[0].iov_base = NULL; ivp[0].iov_len = 0; bvp[0] = NULL; - evp->vsize = io_list_to_vec(list, ivp+1, bvp+1, cbin, blimit); - if (evp->vsize < 0) { + evp->driver.vsize = erts_ioq_iodata_to_vec(list, ivp+1, bvp+1, + cbin, blimit, 1); + if (evp->driver.vsize < 0) { if (evp != &ev) { if (try_call) erts_free(ERTS_ALC_T_TMP, evp); else erts_port_task_free_p2p_sig_data(sigdp); } - driver_free_binary(cbin); + driver_free_binary(&cbin->driver); goto bad_value; } #if 0 @@ -2215,19 +1861,19 @@ erts_port_output(Process *c_p, be falsified during the emulator test suites. */ ASSERT(evp->vsize == vsize); #endif - evp->vsize++; - evp->size = size; /* total size */ + evp->driver.vsize++; + evp->driver.size = size; /* total size */ if (!try_call) { int i; /* Need to increase refc on all binaries */ - for (i = 1; i < evp->vsize; i++) - if (bvp[i]) - driver_binary_inc_refc(bvp[i]); + for (i = 1; i < evp->driver.vsize; i++) + if (bvp[i]) + driver_binary_inc_refc(&bvp[i]->driver); } else { int i; - ErlIOVec *new_evp; + ErtsIOVec *new_evp; ErtsTryImmDrvCallResult try_call_res; ErtsTryImmDrvCallState try_call_state = ERTS_INIT_TRY_IMM_DRV_CALL_STATE( @@ -2250,14 +1896,14 @@ erts_port_output(Process *c_p, from, prt, drv, - evp); + &evp->driver); if (force_immediate_call) finalize_force_imm_drv_call(&try_call_state); else finalize_imm_drv_call(&try_call_state); /* Fall through... */ case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT: - driver_free_binary(cbin); + driver_free_binary(&cbin->driver); if (evp != &ev) { ASSERT(!sigdp); erts_free(ERTS_ALC_T_TMP, evp); @@ -2271,7 +1917,7 @@ erts_port_output(Process *c_p, sched_flags = try_call_state.sched_flags; if (async_nosuspend && (sched_flags & (busy_flgs|ERTS_PTS_FLG_EXIT))) { - driver_free_binary(cbin); + driver_free_binary(&cbin->driver); if (evp != &ev) { ASSERT(!sigdp); erts_free(ERTS_ALC_T_TMP, evp); @@ -2286,9 +1932,9 @@ erts_port_output(Process *c_p, } /* Need to increase refc on all binaries */ - for (i = 1; i < evp->vsize; i++) + for (i = 1; i < evp->driver.vsize; i++) if (bvp[i]) - driver_binary_inc_refc(bvp[i]); + driver_binary_inc_refc(&bvp[i]->driver); /* The port task and iovec is allocated in the same structure as an optimization. This @@ -2301,18 +1947,18 @@ erts_port_output(Process *c_p, if (evp != &ev) { /* Copy from TMP alloc to port task */ sys_memcpy((void *) new_evp, (void *) evp, alloc_size); - new_evp->iov = (SysIOVec *) (((char *) new_evp) - + iov_offset); - bvp = new_evp->binv = (ErlDrvBinary **) (((char *) new_evp) - + binv_offset); + new_evp->driver.iov = (SysIOVec *) (((char *) new_evp) + + iov_offset); + bvp = new_evp->common.binv = (ErtsIOQBinary **) (((char *) new_evp) + + binv_offset); #ifdef DEBUG - ASSERT(new_evp->vsize == evp->vsize); - ASSERT(new_evp->size == evp->size); - for (i = 0; i < evp->vsize; i++) { - ASSERT(new_evp->iov[i].iov_len == evp->iov[i].iov_len); - ASSERT(new_evp->iov[i].iov_base == evp->iov[i].iov_base); - ASSERT(new_evp->binv[i] == evp->binv[i]); + ASSERT(new_evp->driver.vsize == evp->driver.vsize); + ASSERT(new_evp->driver.size == evp->driver.size); + for (i = 0; i < evp->driver.vsize; i++) { + ASSERT(new_evp->driver.iov[i].iov_len == evp->driver.iov[i].iov_len); + ASSERT(new_evp->driver.iov[i].iov_base == evp->driver.iov[i].iov_base); + ASSERT(new_evp->driver.binv[i] == evp->driver.binv[i]); } #endif @@ -2321,24 +1967,24 @@ erts_port_output(Process *c_p, else { /* from stack allocated structure; offsets may differ */ sys_memcpy((void *) new_evp, (void *) evp, sizeof(ErlIOVec)); - new_evp->iov = (SysIOVec *) (((char *) new_evp) - + iov_offset); - sys_memcpy((void *) new_evp->iov, - (void *) evp->iov, - evp->vsize * sizeof(SysIOVec)); - new_evp->binv = (ErlDrvBinary **) (((char *) new_evp) - + binv_offset); - sys_memcpy((void *) new_evp->binv, - (void *) evp->binv, - evp->vsize * sizeof(ErlDrvBinary *)); + new_evp->driver.iov = (SysIOVec *) (((char *) new_evp) + + iov_offset); + sys_memcpy((void *) new_evp->driver.iov, + (void *) evp->driver.iov, + evp->driver.vsize * sizeof(SysIOVec)); + new_evp->common.binv = (ErtsIOQBinary **) (((char *) new_evp) + + binv_offset); + sys_memcpy((void *) new_evp->common.binv, + (void *) evp->common.binv, + evp->driver.vsize * sizeof(ErtsIOQBinary *)); #ifdef DEBUG - ASSERT(new_evp->vsize == evp->vsize); - ASSERT(new_evp->size == evp->size); - for (i = 0; i < evp->vsize; i++) { - ASSERT(new_evp->iov[i].iov_len == evp->iov[i].iov_len); - ASSERT(new_evp->iov[i].iov_base == evp->iov[i].iov_base); - ASSERT(new_evp->binv[i] == evp->binv[i]); + ASSERT(new_evp->driver.vsize == evp->driver.vsize); + ASSERT(new_evp->driver.size == evp->driver.size); + for (i = 0; i < evp->driver.vsize; i++) { + ASSERT(new_evp->driver.iov[i].iov_len == evp->driver.iov[i].iov_len); + ASSERT(new_evp->driver.iov[i].iov_base == evp->driver.iov[i].iov_base); + ASSERT(new_evp->driver.binv[i] == evp->driver.binv[i]); } #endif @@ -2349,8 +1995,8 @@ erts_port_output(Process *c_p, sigdp->flags = ERTS_P2P_SIG_TYPE_OUTPUTV; sigdp->u.outputv.from = from; - sigdp->u.outputv.evp = evp; - sigdp->u.outputv.cbinp = cbin; + sigdp->u.outputv.evp = &evp->driver; + sigdp->u.outputv.cbinp = &cbin->driver; port_sig_callback = port_sig_outputv; } else { @@ -5744,24 +5390,17 @@ erts_stale_drv_select(Eterm port, switch (mode) { case ERL_DRV_READ | ERL_DRV_WRITE: type = "Input/Output"; - goto deselect; case ERL_DRV_WRITE: type = "Output"; - goto deselect; case ERL_DRV_READ: type = "Input"; - deselect: - if (deselect) { - driver_select(drv_port, hndl, - mode | ERL_DRV_USE_NO_CALLBACK, - 0); - } - break; default: - type = "Event"; - if (deselect) - driver_event(drv_port, hndl, NULL); - break; + type = ""; + } + if (deselect) { + driver_select(drv_port, hndl, + mode | ERL_DRV_USE_NO_CALLBACK, + 0); } dsbufp = erts_create_logger_dsbuf(); @@ -6682,6 +6321,7 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, else erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + len)); if (state & ERTS_PORT_SFLG_DISTRIBUTION) { + erts_atomic64_inc_nob(&prt->dist_entry->in); return erts_net_message(prt, prt->dist_entry, (byte*) hbuf, hlen, @@ -6722,6 +6362,7 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, else erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + len)); if (state & ERTS_PORT_SFLG_DISTRIBUTION) { + erts_atomic64_inc_nob(&prt->dist_entry->in); if (len == 0) return erts_net_message(prt, prt->dist_entry, @@ -7088,307 +6729,51 @@ driver_pdl_dec_refc(ErlDrvPDL pdl) return refc; } -/* expand queue to hold n elements in tail or head */ -static int expandq(ErlIOQueue* q, int n, int tail) -/* tail: 0 if make room in head, make room in tail otherwise */ -{ - int h_sz; /* room before header */ - int t_sz; /* room after tail */ - int q_sz; /* occupied */ - int nvsz; - SysIOVec* niov; - ErlDrvBinary** nbinv; - - h_sz = q->v_head - q->v_start; - t_sz = q->v_end - q->v_tail; - q_sz = q->v_tail - q->v_head; - - if (tail && (n <= t_sz)) /* do we need to expand tail? */ - return 0; - else if (!tail && (n <= h_sz)) /* do we need to expand head? */ - return 0; - else if (n > (h_sz + t_sz)) { /* need to allocate */ - /* we may get little extra but it ok */ - nvsz = (q->v_end - q->v_start) + n; - - niov = erts_alloc_fnf(ERTS_ALC_T_IOQ, nvsz * sizeof(SysIOVec)); - if (!niov) - return -1; - nbinv = erts_alloc_fnf(ERTS_ALC_T_IOQ, nvsz * sizeof(ErlDrvBinary**)); - if (!nbinv) { - erts_free(ERTS_ALC_T_IOQ, (void *) niov); - return -1; - } - if (tail) { - sys_memcpy(niov, q->v_head, q_sz*sizeof(SysIOVec)); - if (q->v_start != q->v_small) - erts_free(ERTS_ALC_T_IOQ, (void *) q->v_start); - q->v_start = niov; - q->v_end = niov + nvsz; - q->v_head = q->v_start; - q->v_tail = q->v_head + q_sz; - - sys_memcpy(nbinv, q->b_head, q_sz*sizeof(ErlDrvBinary*)); - if (q->b_start != q->b_small) - erts_free(ERTS_ALC_T_IOQ, (void *) q->b_start); - q->b_start = nbinv; - q->b_end = nbinv + nvsz; - q->b_head = q->b_start; - q->b_tail = q->b_head + q_sz; - } - else { - sys_memcpy(niov+nvsz-q_sz, q->v_head, q_sz*sizeof(SysIOVec)); - if (q->v_start != q->v_small) - erts_free(ERTS_ALC_T_IOQ, (void *) q->v_start); - q->v_start = niov; - q->v_end = niov + nvsz; - q->v_tail = q->v_end; - q->v_head = q->v_tail - q_sz; - - sys_memcpy(nbinv+nvsz-q_sz, q->b_head, q_sz*sizeof(ErlDrvBinary*)); - if (q->b_start != q->b_small) - erts_free(ERTS_ALC_T_IOQ, (void *) q->b_start); - q->b_start = nbinv; - q->b_end = nbinv + nvsz; - q->b_tail = q->b_end; - q->b_head = q->b_tail - q_sz; - } - } - else if (tail) { /* move to beginning to make room in tail */ - sys_memmove(q->v_start, q->v_head, q_sz*sizeof(SysIOVec)); - q->v_head = q->v_start; - q->v_tail = q->v_head + q_sz; - sys_memmove(q->b_start, q->b_head, q_sz*sizeof(ErlDrvBinary*)); - q->b_head = q->b_start; - q->b_tail = q->b_head + q_sz; - } - else { /* move to end to make room */ - sys_memmove(q->v_end-q_sz, q->v_head, q_sz*sizeof(SysIOVec)); - q->v_tail = q->v_end; - q->v_head = q->v_tail-q_sz; - sys_memmove(q->b_end-q_sz, q->b_head, q_sz*sizeof(ErlDrvBinary*)); - q->b_tail = q->b_end; - q->b_head = q->b_tail-q_sz; - } - - return 0; -} - - - /* Put elements from vec at q tail */ int driver_enqv(ErlDrvPort ix, ErlIOVec* vec, ErlDrvSizeT skip) { - int n; - size_t len; - ErlDrvSizeT size; - SysIOVec* iov; - ErlDrvBinary** binv; - ErlDrvBinary* b; - ErlIOQueue* q = drvport2ioq(ix); - - if (q == NULL) - return -1; - - ASSERT(vec->size >= skip); /* debug only */ - if (vec->size <= skip) - return 0; - size = vec->size - skip; - - iov = vec->iov; - binv = vec->binv; - n = vec->vsize; - - /* we use do here to strip iov_len=0 from beginning */ - do { - len = iov->iov_len; - if (len <= skip) { - skip -= len; - iov++; - binv++; - n--; - } - else { - iov->iov_base = ((char *)(iov->iov_base)) + skip; - iov->iov_len -= skip; - skip = 0; - } - } while(skip > 0); - - if (q->v_tail + n >= q->v_end) - expandq(q, n, 1); - - /* Queue and reference all binaries (remove zero length items) */ - while(n--) { - if ((len = iov->iov_len) > 0) { - if ((b = *binv) == NULL) { /* speical case create binary ! */ - b = driver_alloc_binary(len); - sys_memcpy(b->orig_bytes, iov->iov_base, len); - *q->b_tail++ = b; - q->v_tail->iov_len = len; - q->v_tail->iov_base = b->orig_bytes; - q->v_tail++; - } - else { - driver_binary_inc_refc(b); - *q->b_tail++ = b; - *q->v_tail++ = *iov; - } - } - iov++; - binv++; - } - q->size += size; /* update total size in queue */ - return 0; + ASSERT(vec->size >= skip); + return erts_ioq_enqv(drvport2ioq(ix), (ErtsIOVec*)vec, skip); } /* Put elements from vec at q head */ int driver_pushqv(ErlDrvPort ix, ErlIOVec* vec, ErlDrvSizeT skip) { - int n; - size_t len; - ErlDrvSizeT size; - SysIOVec* iov; - ErlDrvBinary** binv; - ErlDrvBinary* b; - ErlIOQueue* q = drvport2ioq(ix); - - if (q == NULL) - return -1; - - if (vec->size <= skip) - return 0; - size = vec->size - skip; - - iov = vec->iov; - binv = vec->binv; - n = vec->vsize; - - /* we use do here to strip iov_len=0 from beginning */ - do { - len = iov->iov_len; - if (len <= skip) { - skip -= len; - iov++; - binv++; - n--; - } - else { - iov->iov_base = ((char *)(iov->iov_base)) + skip; - iov->iov_len -= skip; - skip = 0; - } - } while(skip > 0); - - if (q->v_head - n < q->v_start) - expandq(q, n, 0); - - /* Queue and reference all binaries (remove zero length items) */ - iov += (n-1); /* move to end */ - binv += (n-1); /* move to end */ - while(n--) { - if ((len = iov->iov_len) > 0) { - if ((b = *binv) == NULL) { /* speical case create binary ! */ - b = driver_alloc_binary(len); - sys_memcpy(b->orig_bytes, iov->iov_base, len); - *--q->b_head = b; - q->v_head--; - q->v_head->iov_len = len; - q->v_head->iov_base = b->orig_bytes; - } - else { - driver_binary_inc_refc(b); - *--q->b_head = b; - *--q->v_head = *iov; - } - } - iov--; - binv--; - } - q->size += size; /* update total size in queue */ - return 0; + ASSERT(vec->size >= skip); + return erts_ioq_pushqv(drvport2ioq(ix), (ErtsIOVec*)vec, skip); } - /* ** Remove size bytes from queue head ** Return number of bytes that remain in queue */ ErlDrvSizeT driver_deq(ErlDrvPort ix, ErlDrvSizeT size) { - ErlIOQueue* q = drvport2ioq(ix); - ErlDrvSizeT len; - - if ((q == NULL) || (q->size < size)) - return -1; - q->size -= size; - while (size > 0) { - ASSERT(q->v_head != q->v_tail); - - len = q->v_head->iov_len; - if (len <= size) { - size -= len; - driver_free_binary(*q->b_head); - *q->b_head++ = NULL; - q->v_head++; - } - else { - q->v_head->iov_base = ((char *)(q->v_head->iov_base)) + size; - q->v_head->iov_len -= size; - size = 0; - } - } - - /* restart pointers (optimised for enq) */ - if (q->v_head == q->v_tail) { - q->v_head = q->v_tail = q->v_start; - q->b_head = q->b_tail = q->b_start; - } - return q->size; + ErlPortIOQueue *q = drvport2ioq(ix); + if (erts_ioq_deq(q, size) == -1) + return -1; + return erts_ioq_size(q); } -ErlDrvSizeT driver_peekqv(ErlDrvPort ix, ErlIOVec *ev) { - ErlIOQueue *q = drvport2ioq(ix); - ASSERT(ev); - - if (! q) { - return (ErlDrvSizeT) -1; - } else { - if ((ev->vsize = q->v_tail - q->v_head) == 0) { - ev->size = 0; - ev->iov = NULL; - ev->binv = NULL; - } else { - ev->size = q->size; - ev->iov = q->v_head; - ev->binv = q->b_head; - } - return q->size; - } +ErlDrvSizeT driver_peekqv(ErlDrvPort ix, ErlIOVec *ev) +{ + return erts_ioq_peekqv(drvport2ioq(ix), (ErtsIOVec*)ev); } SysIOVec* driver_peekq(ErlDrvPort ix, int* vlenp) /* length of io-vector */ { - ErlIOQueue* q = drvport2ioq(ix); - - if (q == NULL) { - *vlenp = -1; - return NULL; - } - if ((*vlenp = (q->v_tail - q->v_head)) == 0) - return NULL; - return q->v_head; + return erts_ioq_peekq(drvport2ioq(ix), vlenp); } ErlDrvSizeT driver_sizeq(ErlDrvPort ix) { - ErlIOQueue* q = drvport2ioq(ix); + ErlPortIOQueue *q = drvport2ioq(ix); if (q == NULL) - return (size_t) -1; - return q->size; + return (ErlDrvSizeT) -1; + return erts_ioq_size(q); } @@ -8092,11 +7477,7 @@ driver_system_info(ErlDrvSysInfo *sip, size_t si_size) */ if (si_size >= ERL_DRV_SYS_INFO_SIZE(dirty_scheduler_support)) { sip->dirty_scheduler_support = -#ifdef ERTS_DIRTY_SCHEDULERS 1 -#else - 0 -#endif ; } @@ -8123,14 +7504,6 @@ no_output_callback(ErlDrvData drv_data, char *buf, ErlDrvSizeT len) } static void -no_event_callback(ErlDrvData drv_data, ErlDrvEvent event, ErlDrvEventData event_data) -{ - Port *prt = get_current_port(); - report_missing_drv_callback(prt, "Event", "event()"); - driver_event(ERTS_Port2ErlDrvPort(prt), event, NULL); -} - -static void no_ready_input_callback(ErlDrvData drv_data, ErlDrvEvent event) { Port *prt = get_current_port(); @@ -8197,7 +7570,6 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle) drv->outputv = de->outputv; drv->control = de->control; drv->call = de->call; - drv->event = de->event ? de->event : no_event_callback; drv->ready_input = de->ready_input ? de->ready_input : no_ready_input_callback; drv->ready_output = de->ready_output ? de->ready_output : no_ready_output_callback; drv->timeout = de->timeout ? de->timeout : no_timeout_callback; diff --git a/erts/emulator/beam/macros.tab b/erts/emulator/beam/macros.tab index 41dc761e90..e0b5f56b53 100644 --- a/erts/emulator/beam/macros.tab +++ b/erts/emulator/beam/macros.tab @@ -28,21 +28,45 @@ REFRESH_GEN_DEST() { dst_ptr = REG_TARGET_PTR(dst); } +// $Offset is relative to the start of the instruction (not to the +// location of the failure label reference). Since combined +// instructions may increment the instruction pointer (e.g. in +// 'increment') for some of the instructions in the group, we actually +// use a virtual start position common to all instructions in the +// group. To calculate the correct virtual position, we will need to +// add $IP_ADJUSTMENT to the offset. ($IP_ADJUSTMENT will usually be +// zero, except in a few bit syntax instructions.) + +SET_I_REL(Offset) { + ASSERT(VALID_INSTR(*(I + ($Offset) + $IP_ADJUSTMENT))); + I += $Offset + $IP_ADJUSTMENT; +} + +SET_CP_I_ABS(Target) { + c_p->i = $Target; + ASSERT(VALID_INSTR(*c_p->i)); +} + +SET_REL_I(Dst, Offset) { + $Dst = I + ($Offset); + ASSERT(VALID_INSTR(*$Dst)); +} + FAIL(Fail) { //| -no_prefetch - SET_I((BeamInstr *) $Fail); + $SET_I_REL($Fail); Goto(*I); } JUMP(Fail) { //| -no_next - SET_I((BeamInstr *) $Fail); + $SET_I_REL($Fail); Goto(*I); } GC_TEST(Ns, Nh, Live) { Uint need = $Nh + $Ns; - if (E - HTOP < need) { + if (ERTS_UNLIKELY(E - HTOP < need)) { SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, $Live, FCALLS); @@ -55,7 +79,7 @@ GC_TEST(Ns, Nh, Live) { GC_TEST_PRESERVE(NeedHeap, Live, PreserveTerm) { Uint need = $NeedHeap; - if (E - HTOP < need) { + if (ERTS_UNLIKELY(E - HTOP < need)) { SWAPOUT; reg[$Live] = $PreserveTerm; PROCESS_MAIN_CHK_LOCKS(c_p); @@ -65,7 +89,7 @@ GC_TEST_PRESERVE(NeedHeap, Live, PreserveTerm) { $PreserveTerm = reg[$Live]; SWAPIN; } - HEAP_SPACE_VERIFIED($Nh); + HEAP_SPACE_VERIFIED($NeedHeap); } @@ -99,7 +123,13 @@ FAIL_BODY() { FAIL_HEAD_OR_BODY(Fail) { //| -no_prefetch - if ($Fail) { + + /* + * In a correctly working program, we expect failures in + * guards to be more likely than failures in bodies. + */ + + if (ERTS_LIKELY($Fail)) { $FAIL($Fail); } goto find_func_info; @@ -122,7 +152,7 @@ SYSTEM_LIMIT(Fail) { BIF_ERROR_ARITY_1(Fail, BIF, Op1) { //| -no_prefetch - if ($Fail) { + if (ERTS_LIKELY($Fail)) { $FAIL($Fail); } reg[0] = $Op1; @@ -133,7 +163,7 @@ BIF_ERROR_ARITY_1(Fail, BIF, Op1) { BIF_ERROR_ARITY_2(Fail, BIF, Op1, Op2) { //| -no_prefetch - if ($Fail) { + if (ERTS_LIKELY($Fail)) { $FAIL($Fail); } reg[0] = $Op1; diff --git a/erts/emulator/beam/map_instrs.tab b/erts/emulator/beam/map_instrs.tab index 30c3d7743f..bbb2f49b66 100644 --- a/erts/emulator/beam/map_instrs.tab +++ b/erts/emulator/beam/map_instrs.tab @@ -31,22 +31,24 @@ new_map(Dst, Live, N) { Eterm res; HEAVY_SWAPOUT; - res = new_map(c_p, reg, I-1); + res = new_map(c_p, reg, $Live, $N, $NEXT_INSTRUCTION); HEAVY_SWAPIN; $REFRESH_GEN_DEST(); $Dst = res; $NEXT($NEXT_INSTRUCTION+$N); } -i_new_small_map_lit(Dst, Live, Literal) { +i_new_small_map_lit(Dst, Live, Keys) { Eterm res; Uint n; + Eterm keys = $Keys; HEAVY_SWAPOUT; - res = new_small_map_lit(c_p, reg, &n, I-1); + res = new_small_map_lit(c_p, reg, keys, $Live, $NEXT_INSTRUCTION); HEAVY_SWAPIN; $REFRESH_GEN_DEST(); $Dst = res; + n = arityval(*tuple_val(keys)); $NEXT($NEXT_INSTRUCTION+n); } @@ -127,11 +129,11 @@ i_get_map_elements(Fail, Src, N) { update_map_assoc(Src, Dst, Live, N) { Eterm res; - Eterm map; + Uint live = $Live; - map = $Src; + reg[live] = $Src; HEAVY_SWAPOUT; - res = update_map_assoc(c_p, reg, map, I); + res = update_map_assoc(c_p, reg, live, $N, $NEXT_INSTRUCTION); HEAVY_SWAPIN; ASSERT(is_value(res)); $REFRESH_GEN_DEST(); @@ -141,11 +143,11 @@ update_map_assoc(Src, Dst, Live, N) { update_map_exact(Fail, Src, Dst, Live, N) { Eterm res; - Eterm map; + Uint live = $Live; - map = $Src; + reg[live] = $Src; HEAVY_SWAPOUT; - res = update_map_exact(c_p, reg, map, I); + res = update_map_exact(c_p, reg, live, $N, $NEXT_INSTRUCTION); HEAVY_SWAPIN; if (is_value(res)) { $REFRESH_GEN_DEST(); diff --git a/erts/emulator/beam/msg_instrs.tab b/erts/emulator/beam/msg_instrs.tab index 509143268b..8055a8616f 100644 --- a/erts/emulator/beam/msg_instrs.tab +++ b/erts/emulator/beam/msg_instrs.tab @@ -49,7 +49,7 @@ recv_mark(Dest) { * the label for the loop_rec/2 instruction for the * the receive statement. */ - c_p->msg.mark = (BeamInstr *) $Dest; + $SET_REL_I(c_p->msg.mark, $Dest); c_p->msg.saved_last = c_p->msg.last; } @@ -66,7 +66,9 @@ i_recv_set() { if (c_p->msg.mark == (BeamInstr *) ($NEXT_INSTRUCTION)) { c_p->msg.save = c_p->msg.saved_last; } - /* Fall through to the loop_rec/2 instruction */ + SET_I($NEXT_INSTRUCTION); + goto loop_rec_top__; + //| -no_next } i_loop_rec(Dest) { @@ -79,6 +81,10 @@ i_loop_rec(Dest) { ErtsMessage* msgp; + /* Entry point from recv_set */ + loop_rec_top__: + ; + /* * We need to disable GC while matching messages * in the queue. This since messages with data outside @@ -87,7 +93,8 @@ i_loop_rec(Dest) { ASSERT(!(c_p->flags & F_DELAY_GC)); c_p->flags |= F_DELAY_GC; -loop_rec__: + /* Entry point from loop_rec_end */ + loop_rec__: PROCESS_MAIN_CHK_LOCKS(c_p); @@ -109,7 +116,7 @@ loop_rec__: erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); } else { c_p->flags &= ~F_DELAY_GC; - SET_I((BeamInstr *) $Dest); + $SET_I_REL($Dest); Goto(*I); /* Jump to a wait or wait_timeout instruction */ } } @@ -246,7 +253,7 @@ loop_rec_end(Dest) { ASSERT(c_p->flags & F_DELAY_GC); - SET_I((BeamInstr *) $Dest); + $SET_I_REL($Dest); SAVE_MESSAGE(c_p); if (FCALLS > 0 || FCALLS > neg_o_reds) { FCALLS--; @@ -254,7 +261,7 @@ loop_rec_end(Dest) { } c_p->flags &= ~F_DELAY_GC; - c_p->i = I; + $SET_CP_I_ABS(I); SWAPOUT; c_p->arity = 0; c_p->current = NULL; @@ -285,6 +292,7 @@ timeout() { TIMEOUT_VALUE() { c_p->freason = EXC_TIMEOUT_VALUE; goto find_func_info; + //| -no_next } i_wait_error_locked() { @@ -366,7 +374,7 @@ wait.src(Src) { // wait.execute(JumpTarget) { - c_p->i = (BeamInstr *) $JumpTarget; /* L1 */ + $SET_REL_I(c_p->i, $JumpTarget); /* L1 */ SWAPOUT; c_p->arity = 0; diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 92e67bb470..4a915c7762 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -59,6 +59,7 @@ put_tuple u==0 d => too_old_compiler # All the other instructions. # +%cold label L i_func_info I a a I int_code_end @@ -68,6 +69,8 @@ i_debug_breakpoint i_return_time_trace i_return_to_trace i_yield +trace_jump W +%hot return @@ -96,21 +99,21 @@ line Loc | func_info M F A => func_info M F A | line Loc line I -allocate t t -allocate_heap t I t +allocate t t? +allocate_heap t I t? %cold deallocate Q %hot init y -allocate_zero t t -allocate_heap_zero t I t +allocate_zero t t? +allocate_heap_zero t I t? trim N Remaining => i_trim N -i_trim I +i_trim t -test_heap I t +test_heap I t? allocate_heap S u==0 R => allocate S R allocate_heap_zero S u==0 R => allocate_zero S R @@ -155,19 +158,19 @@ is_tuple Fail=f S | select_tuple_arity S=d Fail=f Size=u Rest=* => \ select_tuple_arity S=d Fail=f Size=u Rest=* => \ gen_select_tuple_arity(S, Fail, Size, Rest) -i_select_val_bins xy f I +i_select_val_bins xy f? I -i_select_val_lins xy f I +i_select_val_lins xy f? I -i_select_val2 xy f c c f f +i_select_val2 xy f? c c -i_select_tuple_arity xy f I +i_select_tuple_arity xy f? I -i_select_tuple_arity2 xy f A A f f +i_select_tuple_arity2 xy f? A A -i_jump_on_val_zero xy f I +i_jump_on_val_zero xy f? I -i_jump_on_val xy f I I +i_jump_on_val xy f? I W get_list xy xy xy @@ -188,11 +191,13 @@ try Y F => catch Y F try_case Y => try_end Y try_end y +%cold try_case_end s +%hot # Destructive set tuple element -set_tuple_element s d P +set_tuple_element s S P # Get tuple element @@ -208,15 +213,20 @@ i_get_tuple_element2y x P y y i_get_tuple_element3 x P x %cold -is_number f x -is_number f y +is_number f? xy %hot + is_number Fail=f i => is_number Fail=f na => jump Fail is_number Fail Literal=q => move Literal x | is_number Fail x jump f +# +# Expection rasing instructions. Infrequently executed. +# + +%cold case_end NotInX=cy => move NotInX x | case_end x badmatch NotInX=cy => move NotInX x | badmatch x @@ -238,6 +248,12 @@ i_raise badarg j system_limit j +%hot + +# +# Move instructions. +# + move C=cxy x==0 | jump Lbl => move_jump Lbl C move_jump f ncxy @@ -382,8 +398,8 @@ label L | wait_timeout Fail Src | smp_already_locked(L) => \ label L | wait_timeout_locked Src Fail wait_timeout Fail Src => wait_timeout_unlocked Src Fail -wait_timeout_unlocked Fail Src=aiq => gen_literal_timeout(Fail, Src) -wait_timeout_locked Fail Src=aiq => gen_literal_timeout_locked(Fail, Src) +wait_timeout_unlocked Src=aiq Fail => gen_literal_timeout(Fail, Src) +wait_timeout_locked Src=aiq Fail => gen_literal_timeout_locked(Fail, Src) label L | wait Fail | smp_already_locked(L) => label L | wait_locked Fail wait Fail => wait_unlocked Fail @@ -398,13 +414,16 @@ loop_rec_end f wait_locked f wait_unlocked f +# Note that a timeout value must fit in 32 bits. wait_timeout_unlocked_int I f wait_timeout_unlocked s f wait_timeout_locked_int I f wait_timeout_locked s f +%cold i_wait_error i_wait_error_locked +%hot send @@ -412,43 +431,52 @@ send # Optimized comparisons with one immediate/literal operand. # -is_eq_exact Lbl R=xy C=ian => i_is_eq_exact_immed Lbl R C +is_eq_exact Lbl S S => +is_eq_exact Lbl C1=c C2=c => move C1 x | is_eq_exact Lbl x C2 +is_eq_exact Lbl C=c R=xy => is_eq_exact Lbl R C + +is_eq_exact Lbl R=xy n => is_nil Lbl R +is_eq_exact Lbl R=xy C=ia => i_is_eq_exact_immed Lbl R C is_eq_exact Lbl R=xy C=q => i_is_eq_exact_literal Lbl R C +is_ne_exact Lbl S S => jump Lbl +is_ne_exact Lbl C1=c C2=c => move C1 x | is_ne_exact Lbl x C2 +is_ne_exact Lbl C=c R=xy => is_ne_exact Lbl R C + is_ne_exact Lbl R=xy C=ian => i_is_ne_exact_immed Lbl R C is_ne_exact Lbl R=xy C=q => i_is_ne_exact_literal Lbl R C -i_is_eq_exact_immed f rxy c +i_is_eq_exact_immed f? rxy c -i_is_eq_exact_literal f xy c +i_is_eq_exact_literal f? xy c -i_is_ne_exact_immed f xy c +i_is_ne_exact_immed f? xy c -i_is_ne_exact_literal f xy c +i_is_ne_exact_literal f? xy c is_eq_exact Lbl Y=y X=x => is_eq_exact Lbl X Y -is_eq_exact f x xy -is_eq_exact f s s +is_eq_exact f? x xy +is_eq_exact f? y y -is_lt f x x -is_lt f x c -is_lt f c x +is_ne_exact f? S S + +is_lt f? x x +is_lt f? x c +is_lt f? c x %cold -is_lt f s s +is_lt f? s s %hot -is_ge f x x -is_ge f x c -is_ge f c x +is_ge f? x x +is_ge f? x c +is_ge f? c x %cold -is_ge f s s +is_ge f? s s %hot -is_ne_exact f s s - -is_eq f s s +is_eq f? s s -is_ne f s s +is_ne f? s s # # Putting things. @@ -514,6 +542,7 @@ put_list s s d # Some more only used by the emulator # +%cold normal_exit continue_exit apply_bif @@ -521,6 +550,7 @@ call_nif call_error_handler error_action_code return_trace +%hot # # Instruction transformations & folded instructions. @@ -553,7 +583,7 @@ is_tagged_tuple Fail Literal=q Arity Atom => \ move Literal x | is_tagged_tuple Fail x Arity Atom is_tagged_tuple Fail=f c Arity Atom => jump Fail -is_tagged_tuple f rxy A a +is_tagged_tuple f? rxy A a # Test tuple & arity (head) @@ -561,14 +591,14 @@ is_tuple Fail Literal=q => move Literal x | is_tuple Fail x is_tuple Fail=f c => jump Fail is_tuple Fail=f S=xy | test_arity Fail=f S=xy Arity => is_tuple_of_arity Fail S Arity -is_tuple_of_arity f rxy A +is_tuple_of_arity f? rxy A -is_tuple f rxy +is_tuple f? rxy test_arity Fail Literal=q Arity => move Literal x | test_arity Fail x Arity test_arity Fail=f c Arity => jump Fail -test_arity f xy A +test_arity f? xy A get_tuple_element Reg=x P1 D1=x | get_tuple_element Reg=x P2 D2=x | \ get_tuple_element Reg=x P3 D3=x | \ @@ -589,16 +619,16 @@ is_integer Fail Literal=q => move Literal x | is_integer Fail x is_integer Fail=f S=x | allocate Need Regs => is_integer_allocate Fail S Need Regs -is_integer_allocate f x I I +is_integer_allocate f? x t t -is_integer f xy +is_integer f? xy is_list Fail=f n => is_list Fail Literal=q => move Literal x | is_list Fail x is_list Fail=f c => jump Fail -is_list f x +is_list f? x %cold -is_list f y +is_list f? y %hot is_nonempty_list Fail=f S=x | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs @@ -608,21 +638,21 @@ is_nonempty_list F=f x==0 | test_heap I1 I2 => is_nonempty_list_test_heap F I1 I is_nonempty_list Fail=f S=x | get_list S D1=x D2=x => \ is_nonempty_list_get_list Fail S D1 D2 -is_nonempty_list_allocate f rx I t -is_nonempty_list_test_heap f I t -is_nonempty_list_get_list f rx x x -is_nonempty_list f xy +is_nonempty_list_allocate f? rx t t +is_nonempty_list_test_heap f? I t +is_nonempty_list_get_list f? rx x x +is_nonempty_list f? xy -is_atom f x +is_atom f? x %cold -is_atom f y +is_atom f? y %hot is_atom Fail=f a => is_atom Fail=f niq => jump Fail -is_float f x +is_float f? x %cold -is_float f y +is_float f? y %hot is_float Fail=f nai => jump Fail is_float Fail Literal=q => move Literal x | is_float Fail x @@ -630,13 +660,13 @@ is_float Fail Literal=q => move Literal x | is_float Fail x is_nil Fail=f n => is_nil Fail=f qia => jump Fail -is_nil f xy +is_nil f? xy is_binary Fail Literal=q => move Literal x | is_binary Fail x is_binary Fail=f c => jump Fail -is_binary f x +is_binary f? x %cold -is_binary f y +is_binary f? y %hot # XXX Deprecated. @@ -644,27 +674,27 @@ is_bitstr Fail Term => is_bitstring Fail Term is_bitstring Fail Literal=q => move Literal x | is_bitstring Fail x is_bitstring Fail=f c => jump Fail -is_bitstring f x +is_bitstring f? x %cold -is_bitstring f y +is_bitstring f? y %hot is_reference Fail=f cq => jump Fail -is_reference f x +is_reference f? x %cold -is_reference f y +is_reference f? y %hot is_pid Fail=f cq => jump Fail -is_pid f x +is_pid f? x %cold -is_pid f y +is_pid f? y %hot is_port Fail=f cq => jump Fail -is_port f x +is_port f? x %cold -is_port f y +is_port f? y %hot is_boolean Fail=f a==am_true => @@ -672,19 +702,19 @@ is_boolean Fail=f a==am_false => is_boolean Fail=f ac => jump Fail %cold -is_boolean f xy +is_boolean f? xy %hot is_function2 Fail=f acq Arity => jump Fail is_function2 Fail=f Fun a => jump Fail -is_function2 f s s +is_function2 f? S s # Allocating & initializing. allocate Need Regs | init Y => allocate_init Need Regs Y init Y1 | init Y2 => init2 Y1 Y2 -allocate_init t I y +allocate_init t t? y ################################################################# # External function and bif calls. @@ -931,16 +961,18 @@ call_ext_last Ar Func D => i_call_ext_last Func D call_ext_only Ar Func => i_call_ext_only Func i_apply -i_apply_last P +i_apply_last Q i_apply_only i_apply_fun -i_apply_fun_last P +i_apply_fun_last Q i_apply_fun_only +%cold i_hibernate i_perf_counter +%hot call_bif e @@ -970,13 +1002,15 @@ node x node y %hot -i_fast_element xy j I d +# Note: 'I' is sufficient because this instruction will only be used +# if the arity fits in 24 bits. +i_fast_element xy j? I d -i_element xy j s d +i_element xy j? s d -bif1 f b s d +bif1 f? b s d bif1_body b s d -i_bif2 f b s s d +i_bif2 f? b s s d i_bif2_body b s s d # @@ -1003,15 +1037,15 @@ call_last Ar Func D => i_call_last Func D call_only Ar Func => i_call_only Func i_call f -i_call_last f P +i_call_last f Q i_call_only f i_call_ext e -i_call_ext_last e P +i_call_ext_last e Q i_call_ext_only e i_move_call_ext c e -i_move_call_ext_last e P c +i_move_call_ext_last e Q c i_move_call_ext_only e c # Fun calls. @@ -1019,16 +1053,16 @@ i_move_call_ext_only e c call_fun Arity | deallocate D | return => i_call_fun_last Arity D call_fun Arity => i_call_fun Arity -i_call_fun I -i_call_fun_last I P +i_call_fun t +i_call_fun_last t Q make_fun2 OldIndex=u => gen_make_fun2(OldIndex) %cold -i_make_fun I t +i_make_fun W t %hot -is_function f xy +is_function f? xy is_function Fail=f c => jump Fail func_info M F A => i_func_info u M F A @@ -1037,44 +1071,44 @@ func_info M F A => i_func_info u M F A # New bit syntax matching (R11B). # ================================================================ -%cold +%warm bs_start_match2 Fail=f ica X Y D => jump Fail bs_start_match2 Fail Bin X Y D => i_bs_start_match2 Bin Fail X Y D -i_bs_start_match2 xy f I I x +i_bs_start_match2 xy f t t x bs_save2 Reg Index => gen_bs_save(Reg, Index) -i_bs_save2 x I +i_bs_save2 x t bs_restore2 Reg Index => gen_bs_restore(Reg, Index) -i_bs_restore2 x I +i_bs_restore2 x t # Matching integers bs_match_string Fail Ms Bits Val => i_bs_match_string Ms Fail Bits Val -i_bs_match_string x f I I +i_bs_match_string x f W W # Fetching integers from binaries. bs_get_integer2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \ gen_get_integer2(Fail, Ms, Live, Sz, Unit, Flags, Dst) -i_bs_get_integer_small_imm x I f I x -i_bs_get_integer_imm x I I f I x -i_bs_get_integer f I I s s x -i_bs_get_integer_8 x f x -i_bs_get_integer_16 x f x +i_bs_get_integer_small_imm x W f? t x +i_bs_get_integer_imm x W t f? t x +i_bs_get_integer f? t t x s x +i_bs_get_integer_8 x f? x +i_bs_get_integer_16 x f? x %if ARCH_64 -i_bs_get_integer_32 x f x +i_bs_get_integer_32 x f? x %endif # Fetching binaries from binaries. bs_get_binary2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \ gen_get_binary2(Fail, Ms, Live, Sz, Unit, Flags, Dst) -i_bs_get_binary_imm2 f x I I I x -i_bs_get_binary2 f x I s I x -i_bs_get_binary_all2 f x I I x -i_bs_get_binary_all_reuse x f I +i_bs_get_binary_imm2 f? x t W t x +i_bs_get_binary2 f x t? s t x +i_bs_get_binary_all2 f? x t t x +i_bs_get_binary_all_reuse x f? t # Fetching float from binaries. bs_get_float2 Fail=f Ms=x Live=u Sz=s Unit=u Flags=u Dst=d => \ @@ -1082,29 +1116,32 @@ bs_get_float2 Fail=f Ms=x Live=u Sz=s Unit=u Flags=u Dst=d => \ bs_get_float2 Fail=f Ms=x Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail -i_bs_get_float2 f x I s I x +i_bs_get_float2 f? x t s t x # Miscellanous bs_skip_bits2 Fail=f Ms=x Sz=sq Unit=u Flags=u => \ gen_skip_bits2(Fail, Ms, Sz, Unit, Flags) -i_bs_skip_bits_imm2 f x I -i_bs_skip_bits2 f x xy I -i_bs_skip_bits_all2 f x I +i_bs_skip_bits_imm2 f? x W +i_bs_skip_bits2 f? x xy t +i_bs_skip_bits_all2 f? x t bs_test_tail2 Fail=f Ms=x Bits=u==0 => bs_test_zero_tail2 Fail Ms bs_test_tail2 Fail=f Ms=x Bits=u => bs_test_tail_imm2 Fail Ms Bits -bs_test_zero_tail2 f x -bs_test_tail_imm2 f x I +bs_test_zero_tail2 f? x +bs_test_tail_imm2 f? x W bs_test_unit F Ms Unit=u==8 => bs_test_unit8 F Ms -bs_test_unit f x I -bs_test_unit8 f x +bs_test_unit f? x t +bs_test_unit8 f? x # An y register operand for bs_context_to_binary is rare, # but can happen because of inlining. +bs_context_to_binary Y=y | line L | badmatch Y => \ + move Y x | bs_context_to_binary x | line L | badmatch x + bs_context_to_binary Y=y => move Y x | bs_context_to_binary x bs_context_to_binary x @@ -1113,14 +1150,14 @@ bs_context_to_binary x # Utf8/utf16/utf32 support. (R12B-5) # bs_get_utf8 Fail=f Ms=x u u Dst=d => i_bs_get_utf8 Ms Fail Dst -i_bs_get_utf8 x f x +i_bs_get_utf8 x f? x bs_skip_utf8 Fail=f Ms=x u u => i_bs_get_utf8 Ms Fail x bs_get_utf16 Fail=f Ms=x u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst bs_skip_utf16 Fail=f Ms=x u Flags=u => i_bs_get_utf16 Ms Fail Flags x -i_bs_get_utf16 x f I x +i_bs_get_utf16 x f? t x bs_get_utf32 Fail=f Ms=x Live=u Flags=u Dst=d => \ bs_get_integer2 Fail Ms Live i=32 u=1 Flags Dst | \ @@ -1129,13 +1166,13 @@ bs_skip_utf32 Fail=f Ms=x Live=u Flags=u => \ bs_get_integer2 Fail Ms Live i=32 u=1 Flags x | \ i_bs_validate_unicode_retract Fail x Ms -i_bs_validate_unicode_retract j s s +i_bs_validate_unicode_retract j s S %hot # # Constructing binaries # -%cold +%warm bs_init2 Fail Sz Words Regs Flags Dst | binary_too_big(Sz) => system_limit Fail @@ -1149,13 +1186,13 @@ bs_init2 Fail Sz Words=u==0 Regs Flags Dst => \ bs_init2 Fail Sz Words Regs Flags Dst => \ i_bs_init_fail_heap Sz Words Fail Regs Dst -i_bs_init_fail xy j I x +i_bs_init_fail xy j? t? x -i_bs_init_fail_heap s I j I x +i_bs_init_fail_heap s I j? t? x -i_bs_init I I x +i_bs_init W t? x -i_bs_init_heap I I I x +i_bs_init_heap W I t? x bs_init_bits Fail Sz=o Words Regs Flags Dst => system_limit Fail @@ -1168,16 +1205,16 @@ bs_init_bits Fail Sz Words=u==0 Regs Flags Dst => \ bs_init_bits Fail Sz Words Regs Flags Dst => \ i_bs_init_bits_fail_heap Sz Words Fail Regs Dst -i_bs_init_bits_fail xy j I x +i_bs_init_bits_fail xy j? t? x -i_bs_init_bits_fail_heap s I j I x +i_bs_init_bits_fail_heap s I j? t? x -i_bs_init_bits I I x -i_bs_init_bits_heap I I I x +i_bs_init_bits W t? x +i_bs_init_bits_heap W I t? x bs_add Fail S1=i==0 S2 Unit=u==1 D => move S2 D -bs_add j s s I x +bs_add j? s s t? x bs_append Fail Size Extra Live Unit Bin Flags Dst => \ move Bin x | i_bs_append Fail Extra Live Unit Size Dst @@ -1187,8 +1224,8 @@ bs_private_append Fail Size Unit Bin Flags Dst => \ bs_init_writable -i_bs_append j I I I s x -i_bs_private_append j I s s x +i_bs_append j? I t? t s x +i_bs_private_append j? t s S x # # Storing integers into binaries. @@ -1197,8 +1234,8 @@ i_bs_private_append j I s s x bs_put_integer Fail=j Sz=sq Unit=u Flags=u Src=s => \ gen_put_integer(Fail, Sz, Unit, Flags, Src) -i_new_bs_put_integer j s I s -i_new_bs_put_integer_imm j I I s +i_new_bs_put_integer j? s t s +i_new_bs_put_integer_imm j? W t s # # Utf8/utf16/utf32 support. (R12B-5) @@ -1214,14 +1251,14 @@ i_bs_utf16_size s x bs_put_utf8 Fail u Src=s => i_bs_put_utf8 Fail Src -i_bs_put_utf8 j s +i_bs_put_utf8 j? s -bs_put_utf16 j I s +bs_put_utf16 j? t s bs_put_utf32 Fail=j Flags=u Src=s => \ i_bs_validate_unicode Fail Src | bs_put_integer Fail i=32 u=1 Flags Src -i_bs_validate_unicode j s +i_bs_validate_unicode j? s # # Storing floats into binaries. @@ -1231,8 +1268,8 @@ bs_put_float Fail Sz=q Unit Flags Val => badarg Fail bs_put_float Fail=j Sz=s Unit=u Flags=u Src=s => \ gen_put_float(Fail, Sz, Unit, Flags, Src) -i_new_bs_put_float j s I s -i_new_bs_put_float_imm j I I s +i_new_bs_put_float j? s t s +i_new_bs_put_float_imm j? W t s # # Storing binaries into binaries. @@ -1241,9 +1278,9 @@ i_new_bs_put_float_imm j I I s bs_put_binary Fail=j Sz=s Unit=u Flags=u Src=s => \ gen_put_binary(Fail, Sz, Unit, Flags, Src) -i_new_bs_put_binary j s I s -i_new_bs_put_binary_imm j I s -i_new_bs_put_binary_all j s I +i_new_bs_put_binary j? s t s +i_new_bs_put_binary_imm j? W s +i_new_bs_put_binary_all j? s t # # Warning: The i_bs_put_string and i_new_bs_put_string instructions @@ -1251,7 +1288,7 @@ i_new_bs_put_binary_all j s I # Don't change the instruction format unless you change the loader too. # -bs_put_string I I +bs_put_string W W # # New floating point instructions (R8). @@ -1269,9 +1306,9 @@ fmove Arg=l Dst=d => fstore Arg Dst fmove Arg=dq Dst=l => fload Arg Dst fstore l d -fload dq l +fload Sq l -fconv d l +fconv S l i_fadd l l l i_fsub l l l @@ -1295,8 +1332,8 @@ fclearerror # New apply instructions in R10B. # -apply I -apply_last I P +apply t +apply_last t Q # # Handle compatibility with OTP 17 here. @@ -1354,15 +1391,15 @@ sorted_put_map_exact F Src Dst Live Size Rest=* => \ new_map Dst Live Size Rest=* | is_small_map_literal_keys(Size, Rest) => \ gen_new_small_map_lit(Dst, Live, Size, Rest) -new_map d I I -i_new_small_map_lit d I q -update_map_assoc s d I I -update_map_exact j s d I I +new_map d t I +i_new_small_map_lit d t q +update_map_assoc s d t I +update_map_exact j? s d t I is_map Fail Lit=q | literal_is_map(Lit) => is_map Fail cq => jump Fail -is_map f xy +is_map f? xy ## Transform has_map_fields #{ K1 := _, K2 := _ } to has_map_elements @@ -1376,14 +1413,14 @@ get_map_elements Fail Src=xy Size=u==2 Rest=* => \ get_map_elements Fail Src Size Rest=* | map_key_sort(Size, Rest) => \ gen_get_map_elements(Fail, Src, Size, Rest) -i_get_map_elements f s I +i_get_map_elements f? s I i_get_map_element Fail Src=xy Key=y Dst => \ move Key x | i_get_map_element Fail Src x Dst -i_get_map_element_hash f xy c I xy +i_get_map_element_hash f? xy c I xy -i_get_map_element f xy x xy +i_get_map_element f? xy x xy # # Convert the plus operations to a generic plus instruction. @@ -1449,32 +1486,34 @@ gc_bif2 Fail Live u$bif:erlang:bxor/2 S1 S2 Dst => \ gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst -i_increment rxy I I d +i_increment rxy W t d + +i_plus x xy j? t d +i_plus s s j? t d -i_plus x xy j I d -i_plus s s j I d +i_minus x x j? t d +i_minus s s j? t d -i_minus x x j I d -i_minus s s j I d +i_times j? t s s d -i_times j I s s d +i_m_div j? t s s d +i_int_div j? t s s d -i_m_div j I s s d -i_int_div j I s s d +i_rem x x j? t d +i_rem s s j? t d -i_rem x x j I d -i_rem s s j I d +i_bsl s s j? t d +i_bsr s s j? t d -i_bsl s s j I d -i_bsr s s j I d +i_band x c j? t d +i_band s s j? t d -i_band x c j I d -i_band s s j I d +i_bor j? I s s d +i_bxor j? I s s d -i_bor j I s s d -i_bxor j I s s d +i_int_bnot Fail Src=c Live Dst => move Src x | i_int_bnot Fail x Live Dst -i_int_bnot j s I d +i_int_bnot j? S t d # # Old guard BIFs that creates heap fragments are no longer allowed. @@ -1498,9 +1537,9 @@ gc_bif2 Fail I Bif S1 S2 Dst => \ gc_bif3 Fail I Bif S1 S2 S3 Dst => \ gen_guard_bif3(Fail, I, Bif, S1, S2, S3, Dst) -i_gc_bif1 j I s I d +i_gc_bif1 j? W s t? d -i_gc_bif2 j I I s s d +i_gc_bif2 j? W t? s s d ii_gc_bif3/7 @@ -1509,7 +1548,7 @@ ii_gc_bif3/7 ii_gc_bif3 Fail Bif Live S1 S2 S3 Dst => \ move S1 x | i_gc_bif3 Fail Bif Live S2 S3 Dst -i_gc_bif3 j I I s s d +i_gc_bif3 j? W t? s s d # # The following instruction is specially handled in beam_load.c diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c index ac9ebd4714..73306030ae 100644 --- a/erts/emulator/beam/safe_hash.c +++ b/erts/emulator/beam/safe_hash.c @@ -260,16 +260,17 @@ void* safe_hash_erase(SafeHash* h, void* tmpl) } /* -** Call 'func(obj,func_arg2)' for all objects in table. NOT SAFE!!! +** Call 'func(obj,func_arg2,func_arg3)' for all objects in table. NOT SAFE!!! */ -void safe_hash_for_each(SafeHash* h, void (*func)(void *, void *), void *func_arg2) +void safe_hash_for_each(SafeHash* h, void (*func)(void *, void *, void *), + void *func_arg2, void *func_arg3) { int i; for (i = 0; i <= h->size_mask; i++) { SafeHashBucket* b = h->tab[i]; while (b != NULL) { - (*func)((void *) b, func_arg2); + (*func)((void *) b, func_arg2, func_arg3); b = b->next; } } diff --git a/erts/emulator/beam/safe_hash.h b/erts/emulator/beam/safe_hash.h index 259c58cff9..af97b4cb4d 100644 --- a/erts/emulator/beam/safe_hash.h +++ b/erts/emulator/beam/safe_hash.h @@ -95,7 +95,7 @@ void* safe_hash_get(SafeHash*, void*); void* safe_hash_put(SafeHash*, void*); void* safe_hash_erase(SafeHash*, void*); -void safe_hash_for_each(SafeHash*, void (*func)(void *, void *), void *); +void safe_hash_for_each(SafeHash*, void (*func)(void *, void *, void *), void *, void *); #ifdef ERTS_ENABLE_LOCK_COUNT void erts_lcnt_enable_hash_lock_count(SafeHash*, erts_lock_flags_t, int); diff --git a/erts/emulator/beam/select_instrs.tab b/erts/emulator/beam/select_instrs.tab index e85ed2c304..2951949d38 100644 --- a/erts/emulator/beam/select_instrs.tab +++ b/erts/emulator/beam/select_instrs.tab @@ -30,16 +30,15 @@ select_val_bins.fetch(Src) { } select_val_bins.select(Fail, NumElements) { - struct Pairs { + struct Singleton { BeamInstr val; - BeamInstr* addr; }; - struct Pairs* low; - struct Pairs* high; - struct Pairs* mid; + struct Singleton* low; + struct Singleton* high; + struct Singleton* mid; int bdiff; /* int not long because the arrays aren't that large */ - low = (struct Pairs *) (&$NumElements + 1); + low = (struct Singleton *) ($NEXT_INSTRUCTION); high = low + $NumElements; /* The pointer subtraction (high-low) below must produce @@ -60,80 +59,73 @@ select_val_bins.select(Fail, NumElements) { * */ while ((bdiff = (int)((char*)high - (char*)low)) > 0) { - unsigned int boffset = ((unsigned int)bdiff >> 1) & ~(sizeof(struct Pairs)-1); + unsigned int boffset = ((unsigned int)bdiff >> 1) & ~(sizeof(struct Singleton)-1); - mid = (struct Pairs*)((char*)low + boffset); + mid = (struct Singleton*)((char*)low + boffset); if (select_val < mid->val) { high = mid; } else if (select_val > mid->val) { low = mid + 1; } else { - $NEXT(mid->addr); + Sint32* jump_tab = (Sint32 *) ($NEXT_INSTRUCTION + $NumElements); + Sint32 offset = jump_tab[mid - (struct Singleton *)($NEXT_INSTRUCTION)]; + $JUMP(offset); } } - $NEXT($Fail); + $JUMP($Fail); } -i_select_tuple_arity2 := select_val2.src.ta_fail.execute; -i_select_val2 := select_val2.src.fail.execute; +i_select_tuple_arity2 := select_val2.src.get_arity.execute; +i_select_val2 := select_val2.src.execute; select_val2.head() { Eterm select_val2; - BeamInstr* select_fail; } select_val2.src(Src) { select_val2 = $Src; } -select_val2.ta_fail(Fail) { - select_fail = &$Fail; - if (is_not_tuple(select_val2)) { - $FAIL(*select_fail); +select_val2.get_arity() { + if (ERTS_LIKELY(is_tuple(select_val2))) { + select_val2 = *tuple_val(select_val2); + } else { + select_val2 = NIL; } - select_val2 = *tuple_val(select_val2); } -select_val2.fail(Fail) { - select_fail = &$Fail; -} +select_val2.execute(Fail, T1, T2) { + Sint32* jump_tab = (Sint32 *) ($NEXT_INSTRUCTION); -select_val2.execute(T1, T2, D1, D2) { if (select_val2 == $T1) { - $JUMP($D1); + $JUMP(jump_tab[0]); } else if (select_val2 == $T2) { - $JUMP($D2); + $JUMP(jump_tab[1]); } else { - $FAIL(*select_fail); + $FAIL($Fail); } } -i_select_tuple_arity := select_val_lin.fetch.ta_fail.execute; -i_select_val_lins := select_val_lin.fetch.fail.execute; +i_select_tuple_arity := select_val_lin.fetch.get_arity.execute; +i_select_val_lins := select_val_lin.fetch.execute; select_val_lin.head() { Eterm select_val; - BeamInstr* select_fail; } select_val_lin.fetch(Src) { select_val = $Src; } -select_val_lin.ta_fail(Fail) { - select_fail = &$Fail; - if (is_tuple(select_val)) { +select_val_lin.get_arity() { + if (ERTS_LIKELY(is_tuple(select_val))) { select_val = *tuple_val(select_val); } else { - $JUMP(*select_fail); + select_val = NIL; } } -select_val_lin.fail(Fail) { - select_fail = &$Fail; -} - -select_val_lin.execute(N) { +select_val_lin.execute(Fail, N) { BeamInstr* vs = $NEXT_INSTRUCTION; int ix = 0; @@ -150,10 +142,11 @@ select_val_lin.execute(N) { } if (vs[ix] == select_val) { - I = $NEXT_INSTRUCTION + $N + ix; - $JUMP(*I); + Sint32* jump_tab = (Sint32 *) ($NEXT_INSTRUCTION + $N); + Eterm offset = jump_tab[ix]; + $JUMP(offset); } else { - $JUMP(*select_fail); + $JUMP($Fail); } } @@ -161,7 +154,8 @@ JUMP_ON_VAL(Fail, Index, N, Base) { if (is_small($Index)) { $Index = (Uint) (signed_val($Index) - $Base); if ($Index < $N) { - $JUMP((($NEXT_INSTRUCTION)[$Index])); + Sint32* jump_tab = (Sint32 *) ($NEXT_INSTRUCTION); + $JUMP(jump_tab[$Index]); } } $FAIL($Fail); diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 9dc339053f..bf7d310568 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -575,8 +575,6 @@ __decl_noreturn void __noreturn erts_exit(int n, char*, ...); erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \ __FILE__, __LINE__, __func__, What) -Eterm erts_check_io_info(void *p); - UWord erts_sys_get_page_size(void); /* Size of misc memory allocated from system dependent code */ @@ -739,8 +737,6 @@ extern char *erts_sys_ddll_error(int code); /* * System interfaces for startup. */ -void erts_sys_schedule_interrupt(int set); -void erts_sys_schedule_interrupt_timed(int, ErtsMonotonicTime); void erts_sys_main_thread(void); extern int erts_sys_prepare_crash_dump(int secs); @@ -757,10 +753,6 @@ Preload* sys_preloaded(void); unsigned char* sys_preload_begin(Preload*); void sys_preload_end(Preload*); int sys_get_key(int); -void elapsed_time_both(ErtsMonotonicTime *ms_user, ErtsMonotonicTime *ms_sys, - ErtsMonotonicTime *ms_user_diff, ErtsMonotonicTime *ms_sys_diff); -void wall_clock_elapsed_time_both(ErtsMonotonicTime *ms_total, - ErtsMonotonicTime *ms_diff); void get_time(int *hour, int *minute, int *second); void get_date(int *year, int *month, int *day); void get_localtime(int *year, int *month, int *day, @@ -798,7 +790,7 @@ void fini_getenv_state(GETENV_STATE *); typedef struct { int no_used_fds; int no_driver_select_structs; - int no_driver_event_structs; + int no_enif_select_structs; } ErtsCheckIoDebugInfo; int erts_check_io_debug(ErtsCheckIoDebugInfo *ip); @@ -1005,10 +997,6 @@ erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val) #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ -#ifdef ERTS_ENABLE_KERNEL_POLL -extern int erts_use_kernel_poll; -#endif - #define sys_memcpy(s1,s2,n) memcpy(s1,s2,n) #define sys_memmove(s1,s2,n) memmove(s1,s2,n) #define sys_memcmp(s1,s2,n) memcmp(s1,s2,n) @@ -1114,6 +1102,14 @@ void erl_bin_write(unsigned char *, int, int); # define DEBUGF(x) #endif +#ifndef MAX +#define MAX(A, B) ((A) > (B) ? (A) : (B)) +#endif + +#ifndef MIN +#define MIN(A, B) ((A) < (B) ? (A) : (B)) +#endif + #ifdef __WIN32__ #ifdef ARCH_64 #define ERTS_ALLOC_ALIGN_BYTES 16 @@ -1179,4 +1175,52 @@ int erts_get_printable_characters(void); void erts_init_sys_common_misc(void); +ERTS_GLB_INLINE Sint erts_raw_env_7bit_ascii_char_need(int encoding); +ERTS_GLB_INLINE byte *erts_raw_env_7bit_ascii_char_put(byte c, byte *p, + int encoding); +ERTS_GLB_INLINE int erts_raw_env_char_is_7bit_ascii_char(byte c, byte *p, + int encoding); +ERTS_GLB_INLINE byte *erts_raw_env_next_char(byte *p, int encoding); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE Sint +erts_raw_env_7bit_ascii_char_need(int encoding) +{ + return (encoding == ERL_FILENAME_WIN_WCHAR) ? 2 : 1; +} + +ERTS_GLB_INLINE byte * +erts_raw_env_7bit_ascii_char_put(byte c, + byte *p, + int encoding) +{ + *(p++) = c; + if (encoding == ERL_FILENAME_WIN_WCHAR) + *(p++) = 0; + return p; +} + +ERTS_GLB_INLINE int +erts_raw_env_char_is_7bit_ascii_char(byte c, + byte *p, + int encoding) +{ + if (encoding == ERL_FILENAME_WIN_WCHAR) + return (p[0] == c) & (p[1] == 0); + else + return p[0] == c; +} + +ERTS_GLB_INLINE byte * +erts_raw_env_next_char(byte *p, int encoding) +{ + if (encoding == ERL_FILENAME_WIN_WCHAR) + return p + 2; + else + return p + 1; +} + +#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ + #endif diff --git a/erts/emulator/beam/trace_instrs.tab b/erts/emulator/beam/trace_instrs.tab index dfd1d16d58..3eee81c053 100644 --- a/erts/emulator/beam/trace_instrs.tab +++ b/erts/emulator/beam/trace_instrs.tab @@ -61,12 +61,12 @@ i_return_to_trace() { Uint *cpp = (Uint*) E; for(;;) { ASSERT(is_CP(*cpp)); - if (*cp_val(*cpp) == (BeamInstr) OpCode(return_trace)) { + if (IsOpCode(*cp_val(*cpp), return_trace)) { do ++cpp; while (is_not_CP(*cpp)); cpp += 2; - } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_to_trace)) { + } else if (IsOpCode(*cp_val(*cpp), i_return_to_trace)) { do ++cpp; while (is_not_CP(*cpp)); @@ -94,7 +94,7 @@ i_yield() { c_p->arg_reg[0] = am_true; c_p->arity = 1; /* One living register (the 'true' return value) */ SWAPOUT; - c_p->i = $NEXT_INSTRUCTION; + $SET_CP_I_ABS($NEXT_INSTRUCTION); c_p->current = NULL; goto do_schedule; //| -no_next @@ -102,7 +102,7 @@ i_yield() { i_hibernate() { HEAVY_SWAPOUT; - if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) { + if (erts_hibernate(c_p, reg)) { FCALLS = c_p->fcalls; c_p->flags &= ~F_HIBERNATE_SCHED; goto do_schedule; @@ -153,3 +153,16 @@ i_debug_breakpoint() { goto handle_error; //| -no_next } + + + +// +// Special jump instruction used for tracing. Takes an absolute +// failure address. +// + +trace_jump(Fail) { + //| -no_next + SET_I((BeamInstr *) $Fail); + Goto(*I); +} diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index dcb1468d60..993585be10 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -51,6 +51,7 @@ #include "erl_ptab.h" #include "erl_check_io.h" #include "erl_bif_unique.h" +#include "erl_io_queue.h" #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" #ifdef HIPE @@ -3605,13 +3606,78 @@ intlist_to_buf(Eterm list, char *buf, Sint len) return -2; /* not enough space */ } -/* Fill buf with the contents of the unicode list. - * Return the number of bytes in the buffer, - * or -1 for type error, - * or -2 for not enough buffer space (buffer contains truncated result). +/** @brief Fill buf with the UTF8 contents of the unicode list + * @param len Max number of characters to write. + * @param written NULL or bytes written. + * @return 0 ok, + * -1 type error, + * -2 list too long, only \c len characters written */ +int +erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len, Sint* written) +{ + Eterm* listptr; + Sint sz = 0; + Sint val; + int res; + + while (1) { + if (is_nil(list)) { + res = 0; + break; + } + if (is_not_list(list)) { + res = -1; + break; + } + listptr = list_val(list); + + if (len-- <= 0) { + res = -2; + break; + } + + if (is_not_small(CAR(listptr))) { + res = -1; + break; + } + val = signed_val(CAR(listptr)); + if (0 <= val && val < 0x80) { + buf[sz] = val; + sz++; + } else if (val < 0x800) { + buf[sz+0] = 0xC0 | (val >> 6); + buf[sz+1] = 0x80 | (val & 0x3F); + sz += 2; + } else if (val < 0x10000UL) { + if (0xD800 <= val && val <= 0xDFFF) { + res = -1; + break; + } + buf[sz+0] = 0xE0 | (val >> 12); + buf[sz+1] = 0x80 | ((val >> 6) & 0x3F); + buf[sz+2] = 0x80 | (val & 0x3F); + sz += 3; + } else if (val < 0x110000) { + buf[sz+0] = 0xF0 | (val >> 18); + buf[sz+1] = 0x80 | ((val >> 12) & 0x3F); + buf[sz+2] = 0x80 | ((val >> 6) & 0x3F); + buf[sz+3] = 0x80 | (val & 0x3F); + sz += 4; + } else { + res = -1; + break; + } + list = CDR(listptr); + } + + if (written) + *written = sz; + return res; +} + Sint -erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len) +erts_unicode_list_to_buf_len(Eterm list) { Eterm* listptr; Sint sz = 0; @@ -3624,7 +3690,7 @@ erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len) } listptr = list_val(list); - while (len-- > 0) { + while (1) { Sint val; if (is_not_small(CAR(listptr))) { @@ -3632,25 +3698,15 @@ erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len) } val = signed_val(CAR(listptr)); if (0 <= val && val < 0x80) { - buf[sz] = val; sz++; } else if (val < 0x800) { - buf[sz+0] = 0xC0 | (val >> 6); - buf[sz+1] = 0x80 | (val & 0x3F); sz += 2; } else if (val < 0x10000UL) { if (0xD800 <= val && val <= 0xDFFF) { return -1; } - buf[sz+0] = 0xE0 | (val >> 12); - buf[sz+1] = 0x80 | ((val >> 6) & 0x3F); - buf[sz+2] = 0x80 | (val & 0x3F); sz += 3; } else if (val < 0x110000) { - buf[sz+0] = 0xF0 | (val >> 18); - buf[sz+1] = 0x80 | ((val >> 12) & 0x3F); - buf[sz+2] = 0x80 | ((val >> 6) & 0x3F); - buf[sz+3] = 0x80 | (val & 0x3F); sz += 4; } else { return -1; @@ -3664,7 +3720,6 @@ erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len) } listptr = list_val(list); } - return -2; /* not enough space */ } /* diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c index a1b15a2199..4e1d2f0d7f 100644 --- a/erts/emulator/drivers/common/efile_drv.c +++ b/erts/emulator/drivers/common/efile_drv.c @@ -1261,6 +1261,50 @@ static void free_read_line(void *data) EF_FREE(d); } +void read_file_zero_size(struct t_data* d); +#define ZERO_FILE_CHUNK (64 * 1024) + +/* [ERL-327] Some special files like /proc/... have reported size 0 */ +void read_file_zero_size(struct t_data* d) { + size_t total_read_size = 0; + size_t allocated_size = ZERO_FILE_CHUNK; /* allocd in invoke_read_file */ + for (;;) { + size_t read_result; + + /* Read until we hit EOF (read less than FILE_SEGMENT_READ) */ + d->result_ok = efile_read(&d->errInfo, + EFILE_MODE_READ, + (int) d->fd, + (d->c.read_file.binp->orig_bytes + + total_read_size), + ZERO_FILE_CHUNK, + &read_result); + if (!d->result_ok) { + break; + } + + total_read_size += read_result; + d->c.read_file.offset += read_result; + if (read_result < ZERO_FILE_CHUNK) { + break; + } + + /* Grow before the next read call */ + allocated_size = total_read_size + ZERO_FILE_CHUNK; + d->c.read_file.binp = driver_realloc_binary(d->c.read_file.binp, + allocated_size); + } + + /* Finalize the memory usage. Hopefully it was read fully on the first + * go, so the binary allocation overhead becomes: + * alloc ZERO_FILE_CHUNK (64kb) -> realloc real_size */ + if (allocated_size != total_read_size) { + d->c.read_file.binp = driver_realloc_binary(d->c.read_file.binp, + total_read_size); + } + d->again = 0; +} + static void invoke_read_file(void *data) { struct t_data *d = (struct t_data *) data; @@ -1279,9 +1323,15 @@ static void invoke_read_file(void *data) } d->fd = fd; d->c.read_file.size = (int) size; - if (size < 0 || size != d->c.read_file.size || - ! (d->c.read_file.binp = - driver_alloc_binary(d->c.read_file.size))) { + + /* For zero sized files allocate a reasonable chunk to attempt reading + * anyway. Note: This will eat ZERO_FILE_CHUNK bytes for any 0 file + * and free them immediately after (if the file was empty). */ + ERTS_ASSERT(size >= 0); + d->c.read_file.binp = driver_alloc_binary(size != 0 ? (size_t)size + : ZERO_FILE_CHUNK); + + if (size < 0 || size != d->c.read_file.size || !d->c.read_file.binp) { d->result_ok = 0; d->errInfo.posix_errno = ENOMEM; goto close; @@ -1290,6 +1340,11 @@ static void invoke_read_file(void *data) } /* Invariant: d->c.read_file.size >= d->c.read_file.offset */ + if (d->c.read_file.size == 0) { + read_file_zero_size(d); + goto close; + } + read_size = (size_t) (d->c.read_file.size - d->c.read_file.offset); if (! read_size) goto close; chop = d->again && read_size >= FILE_SEGMENT_READ*2; diff --git a/erts/emulator/drivers/common/zlib_drv.c b/erts/emulator/drivers/common/zlib_drv.c deleted file mode 100644 index e342e414b5..0000000000 --- a/erts/emulator/drivers/common/zlib_drv.c +++ /dev/null @@ -1,792 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2003-2017. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -/* - * ZLib interface for erlang - * - */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif -#include <stdio.h> -#include <zlib.h> -#include <errno.h> -#include <string.h> - -#include "erl_driver.h" - - -#define DEFLATE_INIT 1 -#define DEFLATE_INIT2 2 -#define DEFLATE_SETDICT 3 -#define DEFLATE_RESET 4 -#define DEFLATE_END 5 -#define DEFLATE_PARAMS 6 -#define DEFLATE 7 - -#define INFLATE_INIT 8 -#define INFLATE_INIT2 9 -#define INFLATE_SETDICT 10 -#define INFLATE_GETDICT 11 -#define INFLATE_SYNC 12 -#define INFLATE_RESET 13 -#define INFLATE_END 14 -#define INFLATE 15 - -#define CRC32_0 16 -#define CRC32_1 17 -#define CRC32_2 18 - -#define SET_BUFSZ 19 -#define GET_BUFSZ 20 -#define GET_QSIZE 21 - -#define ADLER32_1 22 -#define ADLER32_2 23 - -#define CRC32_COMBINE 24 -#define ADLER32_COMBINE 25 - -#define INFLATE_CHUNK 26 - - -#define DEFAULT_BUFSZ 4000 - -/* According to zlib documentation, it can never exceed this */ -#define INFL_DICT_SZ 32768 - -/* This flag is used in the same places, where zlib return codes - * (Z_OK, Z_STREAM_END, Z_NEED_DICT) are. So, we need to set it to - * relatively large value to avoid possible value clashes in future. - * */ -#define INFLATE_HAS_MORE 100 - -static int zlib_init(void); -static ErlDrvData zlib_start(ErlDrvPort port, char* buf); -static void zlib_stop(ErlDrvData e); -static void zlib_flush(ErlDrvData e); -static ErlDrvSSizeT zlib_ctl(ErlDrvData drv_data, unsigned int command, char *buf, - ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen); -static void zlib_outputv(ErlDrvData drv_data, ErlIOVec *ev); - -ErlDrvEntry zlib_driver_entry = { - zlib_init, - zlib_start, - zlib_stop, - NULL, /* output */ - NULL, /* ready_input */ - NULL, /* ready_output */ - "zlib_drv", - NULL, /* finish */ - NULL, /* handle */ - zlib_ctl, - NULL, /* timeout */ - zlib_outputv, - NULL, /* read_async */ - zlib_flush, - NULL, /* call */ - NULL, /* event */ - ERL_DRV_EXTENDED_MARKER, - ERL_DRV_EXTENDED_MAJOR_VERSION, - ERL_DRV_EXTENDED_MINOR_VERSION, - ERL_DRV_FLAG_USE_PORT_LOCKING, - NULL, /* handle2 */ - NULL, /* process_exit */ -}; - -typedef enum { - ST_NONE = 0, - ST_DEFLATE = 1, - ST_INFLATE = 2 -} ZLibState; - - -typedef struct { - z_stream s; - ZLibState state; - ErlDrvBinary* bin; - int binsz; - int binsz_need; - uLong crc; - int inflate_eos_seen; - int want_crc; /* 1 if crc is calculated on clear text */ - ErlDrvPort port; /* the associcated port */ -} ZLibData; - -static int zlib_inflate(ZLibData* d, int flush); -static int zlib_deflate(ZLibData* d, int flush); - -#if defined(__WIN32__) -static int i32(char* buf) -#else -static __inline__ int i32(char* buf) -#endif -{ - return (int) ( - (((int)((unsigned char*)buf)[0]) << 24) | - (((int)((unsigned char*)buf)[1]) << 16) | - (((int)((unsigned char*)buf)[2]) << 8) | - (((int)((unsigned char*)buf)[3]) << 0)); -} - -static char* zlib_reason(int code, int* err) -{ - switch(code) { - case Z_OK: - *err = 0; - return "ok"; - case Z_STREAM_END: - *err = 0; - return "stream_end"; - case Z_ERRNO: - *err = 1; - return erl_errno_id(errno); - case Z_STREAM_ERROR: - *err = 1; - return "stream_error"; - case Z_DATA_ERROR: - *err = 1; - return "data_error"; - case Z_MEM_ERROR: - *err = 1; - return "mem_error"; - case Z_BUF_ERROR: - *err = 1; - return "buf_error"; - case Z_VERSION_ERROR: - *err = 1; - return "version_error"; - default: - *err = 1; - return "unknown_error"; - } -} - - -static ErlDrvSSizeT zlib_return(int code, char** rbuf, ErlDrvSizeT rlen) -{ - int msg_code = 0; /* 0=ok, 1=error */ - char* dst = *rbuf; - char* src; - ErlDrvSizeT len = 0; - - src = zlib_reason(code, &msg_code); - *dst++ = msg_code; - rlen--; - len = 1; - - while((rlen > 0) && *src) { - *dst++ = *src++; - rlen--; - len++; - } - return len; -} - -static ErlDrvSSizeT zlib_value2(int msg_code, int value, - char** rbuf, ErlDrvSizeT rlen) -{ - char* dst = *rbuf; - - if (rlen < 5) { - return -1; - } - *dst++ = msg_code; - *dst++ = (value >> 24) & 0xff; - *dst++ = (value >> 16) & 0xff; - *dst++ = (value >> 8) & 0xff; - *dst++ = value & 0xff; - return 5; -} - -static ErlDrvSSizeT zlib_value(int value, char** rbuf, ErlDrvSizeT rlen) -{ - return zlib_value2(2, value, rbuf, rlen); -} - -static int zlib_output_init(ZLibData* d) -{ - if (d->bin != NULL) - driver_free_binary(d->bin); - if ((d->bin = driver_alloc_binary(d->binsz_need)) == NULL) - return -1; - d->binsz = d->binsz_need; - d->s.next_out = (unsigned char*)d->bin->orig_bytes; - d->s.avail_out = d->binsz; - return 0; -} - -/* - * Send compressed or uncompressed data - * and restart output procesing - */ -static int zlib_output(ZLibData* d) -{ - if (d->bin != NULL) { - int len = d->binsz - d->s.avail_out; - if (len > 0) { - if (driver_output_binary(d->port, NULL, 0, d->bin, 0, len) < 0) - return -1; - } - driver_free_binary(d->bin); - d->bin = NULL; - d->binsz = 0; - } - return zlib_output_init(d); -} - -static int zlib_inflate_get_dictionary(ZLibData* d) -{ -#ifdef HAVE_ZLIB_INFLATEGETDICTIONARY - ErlDrvBinary* dbin = driver_alloc_binary(INFL_DICT_SZ); - uInt dlen = 0; - int res = inflateGetDictionary(&d->s, (unsigned char*)dbin->orig_bytes, &dlen); - if ((res == Z_OK) && (driver_output_binary(d->port, NULL, 0, dbin, 0, dlen) < 0)) { - res = Z_ERRNO; - } - driver_free_binary(dbin); - return res; -#else - abort(); /* never called, just to silence 'unresolved symbol' - for non-optimizing compiler */ -#endif -} - -static int zlib_inflate(ZLibData* d, int flush) -{ - int res = Z_OK; - - if ((d->bin == NULL) && (zlib_output_init(d) < 0)) { - errno = ENOMEM; - return Z_ERRNO; - } - - while ((driver_sizeq(d->port) > 0) && (res != Z_STREAM_END)) { - int vlen; - SysIOVec* iov = driver_peekq(d->port, &vlen); - int len; - int possibly_more_output = 0; - - d->s.next_in = iov[0].iov_base; - d->s.avail_in = iov[0].iov_len; - while((possibly_more_output || (d->s.avail_in > 0)) && (res != Z_STREAM_END)) { - res = inflate(&d->s, Z_NO_FLUSH); - if (res == Z_NEED_DICT) { - /* Essential to eat the header bytes that zlib has looked at */ - len = iov[0].iov_len - d->s.avail_in; - driver_deq(d->port, len); - return res; - } - if (res == Z_BUF_ERROR) { - /* Was possible more output, but actually not */ - res = Z_OK; - } - else if (res < 0) { - return res; - } - if (d->s.avail_out != 0) { - possibly_more_output = 0; - } else { - if (d->want_crc) - d->crc = crc32(d->crc, (unsigned char*)d->bin->orig_bytes, - d->binsz - d->s.avail_out); - zlib_output(d); - possibly_more_output = 1; - } - } - len = iov[0].iov_len - d->s.avail_in; - driver_deq(d->port, len); - } - - if (d->want_crc) { - d->crc = crc32(d->crc, (unsigned char*) d->bin->orig_bytes, - d->binsz - d->s.avail_out); - } - zlib_output(d); - if (res == Z_STREAM_END) { - d->inflate_eos_seen = 1; - } - return res; -} - -static int zlib_inflate_chunk(ZLibData* d) -{ - int res = Z_OK; - - if ((d->bin == NULL) && (zlib_output_init(d) < 0)) { - errno = ENOMEM; - return Z_ERRNO; - } - - while ((driver_sizeq(d->port) > 0) && (d->s.avail_out > 0) && - (res != Z_STREAM_END)) { - int vlen; - SysIOVec* iov = driver_peekq(d->port, &vlen); - int len; - - d->s.next_in = iov[0].iov_base; - d->s.avail_in = iov[0].iov_len; - while((d->s.avail_in > 0) && (d->s.avail_out > 0) && (res != Z_STREAM_END)) { - res = inflate(&d->s, Z_NO_FLUSH); - if (res == Z_NEED_DICT) { - /* Essential to eat the header bytes that zlib has looked at */ - len = iov[0].iov_len - d->s.avail_in; - driver_deq(d->port, len); - return res; - } - if (res == Z_BUF_ERROR) { - /* Was possible more output, but actually not */ - res = Z_OK; - } - else if (res < 0) { - return res; - } - } - len = iov[0].iov_len - d->s.avail_in; - driver_deq(d->port, len); - } - - /* We are here because all input was consumed or EOS reached or output - * buffer is full */ - if (d->want_crc) { - d->crc = crc32(d->crc, (unsigned char*) d->bin->orig_bytes, - d->binsz - d->s.avail_out); - } - zlib_output(d); - if ((res == Z_OK) && (d->s.avail_in > 0)) - res = INFLATE_HAS_MORE; - else if (res == Z_STREAM_END) { - d->inflate_eos_seen = 1; - } - return res; -} - -static int zlib_deflate(ZLibData* d, int flush) -{ - int res = Z_OK; - - if ((d->bin == NULL) && (zlib_output_init(d) < 0)) { - errno = ENOMEM; - return Z_ERRNO; - } - - while ((driver_sizeq(d->port) > 0) && (res != Z_STREAM_END)) { - int vlen; - SysIOVec* iov = driver_peekq(d->port, &vlen); - int len; - - d->s.next_in = iov[0].iov_base; - d->s.avail_in = iov[0].iov_len; - - while((d->s.avail_in > 0) && (res != Z_STREAM_END)) { - if ((res = deflate(&d->s, Z_NO_FLUSH)) < 0) { - return res; - } - if (d->s.avail_out == 0) { - zlib_output(d); - } - } - len = iov[0].iov_len - d->s.avail_in; - if (d->want_crc) { - d->crc = crc32(d->crc, iov[0].iov_base, len); - } - driver_deq(d->port, len); - } - - if (flush != Z_NO_FLUSH) { - if ((res = deflate(&d->s, flush)) < 0) { - return res; - } - if (flush == Z_FINISH) { - while (d->s.avail_out < d->binsz) { - zlib_output(d); - if (res == Z_STREAM_END) { - break; - } - if ((res = deflate(&d->s, flush)) < 0) { - return res; - } - } - } else { - while (d->s.avail_out == 0) { - zlib_output(d); - if ((res = deflate(&d->s, flush)) < 0) { - return res; - } - } - if (d->s.avail_out < d->binsz) { - zlib_output(d); - } - } - } - return res; -} - - - -static void* zlib_alloc(void* data, unsigned int items, unsigned int size) -{ - return (void*) driver_alloc(items*size); -} - -static void zlib_free(void* data, void* addr) -{ - driver_free(addr); -} - -#if defined(__APPLE__) && defined(__MACH__) && defined(HAVE_ZLIB_INFLATEGETDICTIONARY) - -/* Work around broken build system with runtime version test */ -static int have_inflateGetDictionary; - -static int zlib_init() -{ - unsigned int v[4] = {0, 0, 0, 0}; - unsigned hexver; - - sscanf(zlibVersion(), "%u.%u.%u.%u", &v[0], &v[1], &v[2], &v[3]); - - hexver = (v[0] << (8*3)) | (v[1] << (8*2)) | (v[2] << (8)) | v[3]; - - have_inflateGetDictionary = (hexver >= 0x1020701); /* 1.2.7.1 */ - - return 0; -} -#else /* trust configure got it right */ -# ifdef HAVE_ZLIB_INFLATEGETDICTIONARY -# define have_inflateGetDictionary 1 -# else -# define have_inflateGetDictionary 0 -# endif -static int zlib_init() -{ - return 0; -} -#endif - -static ErlDrvData zlib_start(ErlDrvPort port, char* buf) -{ - ZLibData* d; - - if ((d = (ZLibData*) driver_alloc(sizeof(ZLibData))) == NULL) - return ERL_DRV_ERROR_GENERAL; - - memset(&d->s, 0, sizeof(z_stream)); - - d->s.zalloc = zlib_alloc; - d->s.zfree = zlib_free; - d->s.opaque = d; - d->s.data_type = Z_BINARY; - - d->port = port; - d->state = ST_NONE; - d->bin = NULL; - d->binsz = 0; - d->binsz_need = DEFAULT_BUFSZ; - d->crc = crc32(0L, Z_NULL, 0); - d->inflate_eos_seen = 0; - d->want_crc = 0; - return (ErlDrvData)d; -} - - -static void zlib_stop(ErlDrvData e) -{ - ZLibData* d = (ZLibData*)e; - - if (d->state == ST_DEFLATE) - deflateEnd(&d->s); - else if (d->state == ST_INFLATE) - inflateEnd(&d->s); - - if (d->bin != NULL) - driver_free_binary(d->bin); - - driver_free(d); -} - -static void zlib_flush(ErlDrvData drv_data) -{ - ZLibData* d = (ZLibData*) drv_data; - - driver_deq(d->port, driver_sizeq(d->port)); -} - -static ErlDrvSSizeT zlib_ctl(ErlDrvData drv_data, unsigned int command, char *buf, - ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen) -{ - ZLibData* d = (ZLibData*)drv_data; - int res; - - switch(command) { - case DEFLATE_INIT: - if (len != 4) goto badarg; - if (d->state != ST_NONE) goto badarg; - res = deflateInit(&d->s, i32(buf)); - if (res == Z_OK) { - d->state = ST_DEFLATE; - d->want_crc = 0; - d->crc = crc32(0L, Z_NULL, 0); - } - return zlib_return(res, rbuf, rlen); - - case DEFLATE_INIT2: { - int wbits; - - if (len != 20) goto badarg; - if (d->state != ST_NONE) goto badarg; - wbits = i32(buf+8); - res = deflateInit2(&d->s, i32(buf), i32(buf+4), wbits, - i32(buf+12), i32(buf+16)); - if (res == Z_OK) { - d->state = ST_DEFLATE; - d->want_crc = (wbits < 0); - d->crc = crc32(0L, Z_NULL, 0); - } - return zlib_return(res, rbuf, rlen); - } - - case DEFLATE_SETDICT: - if (d->state != ST_DEFLATE) goto badarg; - res = deflateSetDictionary(&d->s, (unsigned char*)buf, len); - if (res == Z_OK) { - return zlib_value(d->s.adler, rbuf, rlen); - } else { - return zlib_return(res, rbuf, rlen); - } - - case DEFLATE_RESET: - if (len != 0) goto badarg; - if (d->state != ST_DEFLATE) goto badarg; - driver_deq(d->port, driver_sizeq(d->port)); - res = deflateReset(&d->s); - return zlib_return(res, rbuf, rlen); - - case DEFLATE_END: - if (len != 0) goto badarg; - if (d->state != ST_DEFLATE) goto badarg; - driver_deq(d->port, driver_sizeq(d->port)); - res = deflateEnd(&d->s); - d->state = ST_NONE; - return zlib_return(res, rbuf, rlen); - - case DEFLATE_PARAMS: - if (len != 8) goto badarg; - if (d->state != ST_DEFLATE) goto badarg; - res = deflateParams(&d->s, i32(buf), i32(buf+4)); - return zlib_return(res, rbuf, rlen); - - case DEFLATE: - if (d->state != ST_DEFLATE) goto badarg; - if (len != 4) goto badarg; - res = zlib_deflate(d, i32(buf)); - return zlib_return(res, rbuf, rlen); - - case INFLATE_INIT: - if (len != 0) goto badarg; - if (d->state != ST_NONE) goto badarg; - res = inflateInit(&d->s); - if (res == Z_OK) { - d->state = ST_INFLATE; - d->inflate_eos_seen = 0; - d->want_crc = 0; - d->crc = crc32(0L, Z_NULL, 0); - } - return zlib_return(res, rbuf, rlen); - - case INFLATE_INIT2: { - int wbits; - - if (len != 4) goto badarg; - if (d->state != ST_NONE) goto badarg; - wbits = i32(buf); - res = inflateInit2(&d->s, wbits); - if (res == Z_OK) { - d->state = ST_INFLATE; - d->inflate_eos_seen = 0; - d->want_crc = (wbits < 0); - d->crc = crc32(0L, Z_NULL, 0); - } - return zlib_return(res, rbuf, rlen); - } - - case INFLATE_SETDICT: - if (d->state != ST_INFLATE) goto badarg; - res = inflateSetDictionary(&d->s, (unsigned char*)buf, len); - return zlib_return(res, rbuf, rlen); - - case INFLATE_GETDICT: - if (have_inflateGetDictionary) { - if (d->state != ST_INFLATE) goto badarg; - res = zlib_inflate_get_dictionary(d); - } else { - errno = ENOTSUP; - res = Z_ERRNO; - } - return zlib_return(res, rbuf, rlen); - - case INFLATE_SYNC: - if (d->state != ST_INFLATE) goto badarg; - if (len != 0) goto badarg; - if (driver_sizeq(d->port) == 0) { - res = Z_BUF_ERROR; - } else { - int vlen; - SysIOVec* iov = driver_peekq(d->port, &vlen); - - d->s.next_in = iov[0].iov_base; - d->s.avail_in = iov[0].iov_len; - res = inflateSync(&d->s); - } - return zlib_return(res, rbuf, rlen); - - case INFLATE_RESET: - if (d->state != ST_INFLATE) goto badarg; - if (len != 0) goto badarg; - driver_deq(d->port, driver_sizeq(d->port)); - res = inflateReset(&d->s); - d->inflate_eos_seen = 0; - return zlib_return(res, rbuf, rlen); - - case INFLATE_END: - if (d->state != ST_INFLATE) goto badarg; - if (len != 0) goto badarg; - driver_deq(d->port, driver_sizeq(d->port)); - res = inflateEnd(&d->s); - if (res == Z_OK && d->inflate_eos_seen == 0) { - res = Z_DATA_ERROR; - } - d->state = ST_NONE; - return zlib_return(res, rbuf, rlen); - - case INFLATE: - if (d->state != ST_INFLATE) goto badarg; - if (len != 4) goto badarg; - res = zlib_inflate(d, i32(buf)); - if (res == Z_NEED_DICT) { - return zlib_value2(3, d->s.adler, rbuf, rlen); - } else { - return zlib_return(res, rbuf, rlen); - } - - case INFLATE_CHUNK: - if (d->state != ST_INFLATE) goto badarg; - if (len != 0) goto badarg; - res = zlib_inflate_chunk(d); - if (res == INFLATE_HAS_MORE) { - return zlib_value2(4, 0, rbuf, rlen); - } else if (res == Z_NEED_DICT) { - return zlib_value2(3, d->s.adler, rbuf, rlen); - } else { - return zlib_return(res, rbuf, rlen); - } - - case GET_QSIZE: - return zlib_value(driver_sizeq(d->port), rbuf, rlen); - - case GET_BUFSZ: - return zlib_value(d->binsz_need, rbuf, rlen); - - case SET_BUFSZ: { - int need; - if (len != 4) goto badarg; - need = i32(buf); - if ((need < 16) || (need > 0x00ffffff)) - goto badarg; - if (d->binsz_need != need) { - d->binsz_need = need; - if (d->bin != NULL) { - if (d->s.avail_out == d->binsz) { - driver_free_binary(d->bin); - d->bin = NULL; - d->binsz = 0; - } - else - zlib_output(d); - } - } - return zlib_return(Z_OK, rbuf, rlen); - } - - case CRC32_0: - return zlib_value(d->crc, rbuf, rlen); - - case CRC32_1: { - uLong crc = crc32(0L, Z_NULL, 0); - crc = crc32(crc, (unsigned char*) buf, len); - return zlib_value(crc, rbuf, rlen); - } - - case CRC32_2: { - uLong crc; - if (len < 4) goto badarg; - crc = (unsigned int) i32(buf); - crc = crc32(crc, (unsigned char*) buf+4, len-4); - return zlib_value(crc, rbuf, rlen); - } - - case ADLER32_1: { - uLong adler = adler32(0L, Z_NULL, 0); - adler = adler32(adler, (unsigned char*) buf, len); - return zlib_value(adler, rbuf, rlen); - } - - case ADLER32_2: { - uLong adler; - if (len < 4) goto badarg; - adler = (unsigned int) i32(buf); - adler = adler32(adler, (unsigned char*) buf+4, len-4); - return zlib_value(adler, rbuf, rlen); - } - - case CRC32_COMBINE: { - uLong crc, crc1, crc2, len2; - if (len != 12) goto badarg; - crc1 = (unsigned int) i32(buf); - crc2 = (unsigned int) i32(buf+4); - len2 = (unsigned int) i32(buf+8); - crc = crc32_combine(crc1, crc2, len2); - return zlib_value(crc, rbuf, rlen); - } - - case ADLER32_COMBINE: { - uLong adler, adler1, adler2, len2; - if (len != 12) goto badarg; - adler1 = (unsigned int) i32(buf); - adler2 = (unsigned int) i32(buf+4); - len2 = (unsigned int) i32(buf+8); - adler = adler32_combine(adler1, adler2, len2); - return zlib_value(adler, rbuf, rlen); - } - } - - badarg: - errno = EINVAL; - return zlib_return(Z_ERRNO, rbuf, rlen); -} - - - -static void zlib_outputv(ErlDrvData drv_data, ErlIOVec *ev) -{ - ZLibData* d = (ZLibData*) drv_data; - - driver_enqv(d->port, ev, 0); -} diff --git a/erts/emulator/drivers/unix/ttsl_drv.c b/erts/emulator/drivers/unix/ttsl_drv.c index bce097d944..7355df6059 100644 --- a/erts/emulator/drivers/unix/ttsl_drv.c +++ b/erts/emulator/drivers/unix/ttsl_drv.c @@ -892,8 +892,8 @@ static void ttysl_from_tty(ErlDrvData ttysl_data, ErlDrvEvent fd) tpos = 0; } } - } else { - DEBUGLOG(("ttysl_from_tty: driver failure in read(%d,..) = %d\n", (int)(SWord)fd, i)); + } else if (errno != EAGAIN && errno != EWOULDBLOCK) { + DEBUGLOG(("ttysl_from_tty: driver failure in read(%d,..) = %d (errno = %d)\n", (int)(SWord)fd, i, errno)); driver_failure(ttysl_port, -1); } } @@ -936,10 +936,10 @@ static int put_chars(byte *s, int l) int n; n = insert_buf(s, l); + if (lpos > llen) + llen = lpos; if (n > 0) write_buf(lbuf + lpos - n, n); - if (lpos > llen) - llen = lpos; return TRUE; } @@ -1016,7 +1016,7 @@ static int del_chars(int n) outc(' '); move_left(1); } - move_cursor(llen + l, lpos); + move_cursor(llen + gcs, lpos); } else if (pos < lpos) { l = lpos - pos; /* Buffer characters */ @@ -1036,7 +1036,7 @@ static int del_chars(int n) outc(' '); move_left(1); } - move_cursor(llen + l, lpos); + move_cursor(llen + gcs, lpos); } return TRUE; } @@ -1094,15 +1094,14 @@ static int insert_buf(byte *s, int n) lbuf[lpos++] = (CONTROL_TAG | ((Uint32) ch)); ch = 0; } while (lpos % 8); - } else if (ch == '\e' || ch == '\n' || ch == '\r') { + } else if (ch == '\e') { + DEBUGLOG(("insert_buf: ANSI Escape: \\e")); + lbuf[lpos++] = (CONTROL_TAG | ((Uint32) ch)); + } else if (ch == '\n' || ch == '\r') { write_buf(lbuf + buffpos, lpos - buffpos); - if (ch == '\e') { - outc('\e'); - } else { outc('\r'); if (ch == '\n') outc('\n'); - } if (llen > lpos) { memcpy(lbuf, lbuf + lpos, llen - lpos); } @@ -1150,14 +1149,17 @@ static int write_buf(Uint32 *s, int n) } --n; ++s; - } - else if (*s == (CONTROL_TAG | ((Uint32) '\t'))) { + } else if (*s == (CONTROL_TAG | ((Uint32) '\t'))) { outc(lastput = ' '); --n; s++; while (n > 0 && *s == CONTROL_TAG) { outc(lastput = ' '); --n; s++; } + } else if (*s == (CONTROL_TAG | ((Uint32) '\e'))) { + outc(lastput = '\e'); + --n; + ++s; } else if (*s & CONTROL_TAG) { outc('^'); outc(lastput = ((byte) ((*s == 0177) ? '?' : *s | 0x40))); @@ -1250,26 +1252,63 @@ static int move_cursor(int from_pos, int to_pos) return to_col-from_col; } -static int cp_pos_to_col(int cp_pos) +/* + * Returns the length of an ANSI escape code in a buffer, this function only consider + * color escape sequences like `\e[33m` or `\e[21;33m`. If a sequence has no valid + * terminator, the length is equal the number of characters between `\e` and the first + * invalid character, inclusive. + */ + +static int ansi_escape_width(Uint32 *s, int max_length) { -#ifdef HAVE_WCWIDTH int i; - int col = 0; - - for (i = 0; i < cp_pos; i++) { - int w = wcwidth(lbuf[i]); - if (w > 0) { - col += w; - } + + if (*s != (CONTROL_TAG | ((Uint32) '\e'))) { + return 0; + } else if (max_length <= 1) { + return 1; + } else if (s[1] != '[') { + return 2; } - return col; -#else + + for (i = 2; i < max_length && (s[i] == ';' || (s[i] >= '0' && s[i] <= '9')); i++); + + return i + 1; +} + +static int cp_pos_to_col(int cp_pos) +{ /* - * We dont' have any character width information. Assume that - * code points are one column wide. + * If we don't have any character width information. Assume that + * code points are one column wide */ - return cp_pos; + int w = 1; + int col = 0; + int i = 0; + int j; + + if (cp_pos > llen) { + col += cp_pos - llen; + cp_pos = llen; + } + + while (i < cp_pos) { + j = ansi_escape_width(lbuf + i, llen - i); + + if (j > 0) { + i += j; + } else { +#ifdef HAVE_WCWIDTH + w = wcwidth(lbuf[i]); #endif + if (w > 0) { + col += w; + } + i++; + } + } + + return col; } static int start_termcap(void) diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 05663648e9..380031bf13 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -53,8 +53,6 @@ #include "hipe_literals.h" #endif -#define BeamOpCode(Op) ((Uint)BeamOp(Op)) - int term_to_Sint32(Eterm term, Sint *sp) { @@ -615,7 +613,7 @@ static ErtsCodeInfo* hipe_find_emu_address(Eterm mod, Eterm name, unsigned int a n = code_hdr->num_functions; for (i = 0; i < n; ++i) { ErtsCodeInfo *ci = code_hdr->functions[i]; - ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI)); if (ci->mfa.function == name && ci->mfa.arity == arity) return ci; } diff --git a/erts/emulator/hipe/hipe_bif1.c b/erts/emulator/hipe/hipe_bif1.c index 3d3df4fd48..73d07f0ce5 100644 --- a/erts/emulator/hipe/hipe_bif1.c +++ b/erts/emulator/hipe/hipe_bif1.c @@ -32,11 +32,10 @@ #include "big.h" #include "error.h" #include "beam_load.h" +#include "erl_vm.h" #include "hipe_bif0.h" #include "hipe_bif1.h" -#define BeamOpCode(Op) ((Uint)BeamOp(Op)) - BIF_RETTYPE hipe_bifs_call_count_on_1(BIF_ALIST_1) { ErtsCodeInfo *ci; @@ -46,17 +45,17 @@ BIF_RETTYPE hipe_bifs_call_count_on_1(BIF_ALIST_1) ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); if (!ci) BIF_ERROR(BIF_P, BADARG); - ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI)); pc = erts_codeinfo_to_code(ci); - if (pc[0] == BeamOpCode(op_hipe_trap_call)) + if (BeamIsOpCode(pc[0], op_hipe_trap_call)) BIF_ERROR(BIF_P, BADARG); - if (pc[0] == BeamOpCode(op_hipe_call_count)) + if (BeamIsOpCode(pc[0], op_hipe_call_count)) BIF_RET(NIL); hcc = erts_alloc(ERTS_ALC_T_HIPE_SL, sizeof(*hcc)); hcc->count = 0; hcc->opcode = pc[0]; ci->u.hcc = hcc; - pc[0] = BeamOpCode(op_hipe_call_count); + pc[0] = BeamOpCodeAddr(op_hipe_call_count); BIF_RET(am_true); } @@ -70,9 +69,9 @@ BIF_RETTYPE hipe_bifs_call_count_off_1(BIF_ALIST_1) ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); if (!ci) BIF_ERROR(BIF_P, BADARG); - ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI)); pc = erts_codeinfo_to_code(ci); - if (pc[0] != BeamOpCode(op_hipe_call_count)) + if (! BeamIsOpCode(pc[0], op_hipe_call_count)) BIF_RET(am_false); hcc = ci->u.hcc; count = hcc->count; @@ -91,9 +90,9 @@ BIF_RETTYPE hipe_bifs_call_count_get_1(BIF_ALIST_1) ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); if (!ci) BIF_ERROR(BIF_P, BADARG); - ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI)); pc = erts_codeinfo_to_code(ci); - if (pc[0] != BeamOpCode(op_hipe_call_count)) + if (! BeamIsOpCode(pc[0], op_hipe_call_count)) BIF_RET(am_false); hcc = ci->u.hcc; BIF_RET(make_small(hcc->count)); @@ -109,9 +108,9 @@ BIF_RETTYPE hipe_bifs_call_count_clear_1(BIF_ALIST_1) ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); if (!ci) BIF_ERROR(BIF_P, BADARG); - ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI)); pc = erts_codeinfo_to_code(ci); - if (pc[0] != BeamOpCode(op_hipe_call_count)) + if (! BeamIsOpCode(pc[0], op_hipe_call_count)) BIF_RET(am_false); hcc = ci->u.hcc; count = hcc->count; diff --git a/erts/emulator/hipe/hipe_instrs.tab b/erts/emulator/hipe/hipe_instrs.tab index bcce196a1d..a01baebddf 100644 --- a/erts/emulator/hipe/hipe_instrs.tab +++ b/erts/emulator/hipe/hipe_instrs.tab @@ -45,7 +45,7 @@ hipe_trap.call() { * ... remainder of original BEAM code */ ErtsCodeInfo *ci = erts_code_to_codeinfo(I); - ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); + ASSERT(IsOpCode(ci->op, i_func_info_IaaI)); c_p->hipe.u.ncallee = ci->u.ncallee; ++hipe_trap_count; $HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (ci->mfa.arity << 8)); @@ -53,7 +53,7 @@ hipe_trap.call() { hipe_trap.call_closure() { ErtsCodeInfo *ci = erts_code_to_codeinfo(I); - ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); + ASSERT(IsOpCode(ci->op, i_func_info_IaaI)); c_p->hipe.u.ncallee = ci->u.ncallee; ++hipe_trap_count; $HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (ci->mfa.arity << 8)); @@ -132,7 +132,7 @@ hipe_call_count() { */ ErtsCodeInfo *ci = erts_code_to_codeinfo(I); struct hipe_call_count *hcc = ci->u.hcc; - ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); + ASSERT(IsOpCode(ci->op, i_func_info_IaaI)); ASSERT(hcc != NULL); ASSERT(VALID_INSTR(hcc->opcode)); ++(hcc->count); diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c index b7f81fc4a6..8b497c9970 100644 --- a/erts/emulator/hipe/hipe_mode_switch.c +++ b/erts/emulator/hipe/hipe_mode_switch.c @@ -155,8 +155,6 @@ void hipe_check_pcb(Process *p, const char *file, unsigned line) #include "hipe_arm_glue.h" #endif -#define BeamOpCode(Op) ((Uint)BeamOp(Op)) - Uint hipe_beam_pc_return[1]; /* needed in hipe_debug.c */ Uint hipe_beam_pc_throw[1]; /* needed in hipe_debug.c */ Uint hipe_beam_pc_resume[1]; /* needed by hipe_set_timeout() */ @@ -166,9 +164,9 @@ void hipe_mode_switch_init(void) { hipe_arch_glue_init(); - hipe_beam_pc_return[0] = BeamOpCode(op_hipe_trap_return); - hipe_beam_pc_throw[0] = BeamOpCode(op_hipe_trap_throw); - hipe_beam_pc_resume[0] = BeamOpCode(op_hipe_trap_resume); + hipe_beam_pc_return[0] = BeamOpCodeAddr(op_hipe_trap_return); + hipe_beam_pc_throw[0] = BeamOpCodeAddr(op_hipe_trap_throw); + hipe_beam_pc_resume[0] = BeamOpCodeAddr(op_hipe_trap_resume); hipe_beam_catch_throw = make_catch(beam_catches_cons(hipe_beam_pc_throw, BEAM_CATCHES_NIL)); @@ -182,8 +180,8 @@ void hipe_set_call_trap(ErtsCodeInfo* ci, void *nfun, int is_closure) HIPE_ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); bfun[0] = is_closure - ? BeamOpCode(op_hipe_trap_call_closure) - : BeamOpCode(op_hipe_trap_call); + ? BeamOpCodeAddr(op_hipe_trap_call_closure) + : BeamOpCodeAddr(op_hipe_trap_call); ci->u.ncallee = (void (*)(void)) nfun; } diff --git a/erts/emulator/hipe/hipe_ops.tab b/erts/emulator/hipe/hipe_ops.tab index 96e4c0da91..19a3820a6a 100644 --- a/erts/emulator/hipe/hipe_ops.tab +++ b/erts/emulator/hipe/hipe_ops.tab @@ -23,4 +23,7 @@ hipe_trap_call_closure hipe_trap_return hipe_trap_throw hipe_trap_resume + +%cold hipe_call_count +%hot diff --git a/erts/emulator/nifs/common/zlib_nif.c b/erts/emulator/nifs/common/zlib_nif.c new file mode 100644 index 0000000000..a9c5b05e47 --- /dev/null +++ b/erts/emulator/nifs/common/zlib_nif.c @@ -0,0 +1,986 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson 2017. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#define STATIC_ERLANG_NIF 1 + +#include <stdio.h> +#include <zlib.h> + +#include "erl_nif.h" +#include "config.h" +#include "sys.h" + +#ifdef VALGRIND +# include <valgrind/memcheck.h> +#endif + +#define INFL_DICT_SZ (32768) + +/* NIF interface declarations */ +static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM load_info); +static int upgrade(ErlNifEnv *env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info); +static void unload(ErlNifEnv *env, void* priv_data); + +static ErlNifResourceType *rtype_zlib; + +static ERL_NIF_TERM am_not_on_controlling_process; + +static ERL_NIF_TERM am_not_initialized; +static ERL_NIF_TERM am_already_initialized; + +static ERL_NIF_TERM am_ok; +static ERL_NIF_TERM am_error; + +static ERL_NIF_TERM am_continue; +static ERL_NIF_TERM am_finished; + +static ERL_NIF_TERM am_not_supported; +static ERL_NIF_TERM am_need_dictionary; + +static ERL_NIF_TERM am_empty; + +static ERL_NIF_TERM am_stream_end; +static ERL_NIF_TERM am_stream_error; +static ERL_NIF_TERM am_data_error; +static ERL_NIF_TERM am_mem_error; +static ERL_NIF_TERM am_buf_error; +static ERL_NIF_TERM am_version_error; +static ERL_NIF_TERM am_unknown_error; + +typedef enum { + ST_NONE = 0, + ST_DEFLATE = 1, + ST_INFLATE = 2, + ST_CLOSED = 3 +} zlib_state_t; + +/* Controls what to do when the user attempts to decompress more data after + * Z_STREAM_END has been returned: + * + * - 'cut' wipes all further input and returns empty results until reset by + * the user. This is the default behavior, matching that of the old driver. + * - 'reset' resets the state without discarding any input, making it possible + * to decompress blindly concatenated streams. + * - 'error' crashes with a data error. */ +typedef enum { + EOS_BEHAVIOR_ERROR = 0, + EOS_BEHAVIOR_RESET = 1, + EOS_BEHAVIOR_CUT = 2 +} zlib_eos_behavior_t; + +typedef struct { + z_stream s; + zlib_state_t state; + + zlib_eos_behavior_t eos_behavior; + + /* These refer to the plaintext CRC, and are only needed for zlib:crc32/1 + * which is deprecated. */ + uLong input_crc; + uLong output_crc; + int want_input_crc; + int want_output_crc; + + int is_raw_stream; + + int eos_seen; + + /* DEPRECATED */ + int inflateChunk_buffer_size; + + ErlNifPid controlling_process; + + ErlNifIOQueue *input_queue; + + ErlNifEnv *stash_env; + ERL_NIF_TERM stash_term; +} zlib_data_t; + +/* The NIFs: */ + +static ERL_NIF_TERM zlib_open(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_close(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_deflateInit(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_deflateSetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_deflateReset(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_deflateEnd(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_deflateParams(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_deflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ERL_NIF_TERM zlib_inflateInit(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_inflateSetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_inflateGetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_inflateReset(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_inflateEnd(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_inflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ERL_NIF_TERM zlib_crc32(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ERL_NIF_TERM zlib_clearStash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_setStash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_getStash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ERL_NIF_TERM zlib_getBufSize(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_setBufSize(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ERL_NIF_TERM zlib_enqueue_input(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ErlNifFunc nif_funcs[] = { + /* deflate */ + {"deflateInit_nif", 6, zlib_deflateInit}, + {"deflateSetDictionary_nif", 2, zlib_deflateSetDictionary}, + {"deflateReset_nif", 1, zlib_deflateReset}, + {"deflateEnd_nif", 1, zlib_deflateEnd}, + {"deflateParams_nif", 3, zlib_deflateParams}, + {"deflate_nif", 4, zlib_deflate}, + + /* inflate */ + {"inflateInit_nif", 3, zlib_inflateInit}, + {"inflateSetDictionary_nif", 2, zlib_inflateSetDictionary}, + {"inflateGetDictionary_nif", 1, zlib_inflateGetDictionary}, + {"inflateReset_nif", 1, zlib_inflateReset}, + {"inflateEnd_nif", 1, zlib_inflateEnd}, + {"inflate_nif", 4, zlib_inflate}, + + /* running checksum */ + {"crc32_nif", 1, zlib_crc32}, + + /* open & close */ + {"close_nif", 1, zlib_close}, + {"open_nif", 0, zlib_open}, + + /* The stash keeps a single term alive across calls, and is used in + * exception_on_need_dict/1 to retain the old error behavior, and for + * saving data flushed through deflateParams/3. */ + {"getStash_nif", 1, zlib_getStash}, + {"clearStash_nif", 1, zlib_clearStash}, + {"setStash_nif", 2, zlib_setStash}, + + /* DEPRECATED: buffer size for inflateChunk */ + {"getBufSize_nif", 1, zlib_getBufSize}, + {"setBufSize_nif", 2, zlib_setBufSize}, + + {"enqueue_nif", 2, zlib_enqueue_input}, +}; + +ERL_NIF_INIT(zlib, nif_funcs, load, NULL, upgrade, unload) + +static void gc_zlib(ErlNifEnv *env, void* data); + +static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM load_info) +{ + am_not_on_controlling_process = + enif_make_atom(env, "not_on_controlling_process"); + + am_not_initialized = enif_make_atom(env, "not_initialized"); + am_already_initialized = enif_make_atom(env, "already_initialized"); + + am_ok = enif_make_atom(env, "ok"); + am_error = enif_make_atom(env, "error"); + + am_continue = enif_make_atom(env, "continue"); + am_finished = enif_make_atom(env, "finished"); + + am_not_supported = enif_make_atom(env, "not_supported"); + am_need_dictionary = enif_make_atom(env, "need_dictionary"); + + am_empty = enif_make_atom(env, "empty"); + + am_stream_end = enif_make_atom(env, "stream_end"); + am_stream_error = enif_make_atom(env, "stream_error"); + am_data_error = enif_make_atom(env, "data_error"); + am_mem_error = enif_make_atom(env, "mem_error"); + am_buf_error = enif_make_atom(env, "buf_error"); + am_version_error = enif_make_atom(env, "version_error"); + am_unknown_error = enif_make_atom(env, "unknown_error"); + + rtype_zlib = enif_open_resource_type(env, NULL, + "gc_zlib", gc_zlib, ERL_NIF_RT_CREATE, NULL); + *priv_data = NULL; + + return 0; +} + +static void unload(ErlNifEnv *env, void* priv_data) +{ + +} + +static int upgrade(ErlNifEnv *env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) +{ + if(*old_priv_data != NULL) { + return -1; /* Don't know how to do that */ + } + if(*priv_data != NULL) { + return -1; /* Don't know how to do that */ + } + if(load(env, priv_data, load_info)) { + return -1; + } + return 0; +} + +static void* zlib_alloc(void* data, unsigned int items, unsigned int size) +{ + return (void*) enif_alloc(items * size); +} + +static void zlib_free(void* data, void* addr) +{ + enif_free(addr); +} + +static ERL_NIF_TERM zlib_return(ErlNifEnv *env, int code) { + ERL_NIF_TERM reason; + switch(code) { + case Z_OK: + reason = am_ok; + break; + case Z_STREAM_END: + reason = am_stream_end; + break; + case Z_ERRNO: + reason = enif_make_int(env, errno); + break; + case Z_STREAM_ERROR: + reason = enif_raise_exception(env, am_stream_error); + break; + case Z_DATA_ERROR: + reason = enif_raise_exception(env, am_data_error); + break; + case Z_MEM_ERROR: + reason = am_mem_error; + break; + case Z_BUF_ERROR: + reason = am_buf_error; + break; + case Z_VERSION_ERROR: + reason = am_version_error; + break; + default: + reason = am_unknown_error; + break; + } + return reason; +} + +static void gc_zlib(ErlNifEnv *env, void* data) { + zlib_data_t *d = (zlib_data_t*)data; + + if(d->state == ST_DEFLATE) { + deflateEnd(&d->s); + } else if(d->state == ST_INFLATE) { + inflateEnd(&d->s); + } + + if(d->state != ST_CLOSED) { + enif_ioq_destroy(d->input_queue); + + if(d->stash_env != NULL) { + enif_free_env(d->stash_env); + } + + d->state = ST_CLOSED; + } +} + +static int get_zlib_data(ErlNifEnv *env, ERL_NIF_TERM opaque, zlib_data_t **d) { + return enif_get_resource(env, opaque, rtype_zlib, (void **)d); +} + +static int zlib_process_check(ErlNifEnv *env, zlib_data_t *d) { + ErlNifPid current_process; + + enif_self(env, ¤t_process); + + return enif_is_identical(enif_make_pid(env, ¤t_process), + enif_make_pid(env, &d->controlling_process)); +} + +static void zlib_reset_input(zlib_data_t *d) { + enif_ioq_destroy(d->input_queue); + d->input_queue = enif_ioq_create(ERL_NIF_IOQ_NORMAL); + + if(d->stash_env != NULL) { + enif_free_env(d->stash_env); + d->stash_env = NULL; + d->stash_term = NIL; + } +} + +static int zlib_flush_queue(int (*codec)(z_stream*, int), ErlNifEnv *env, + zlib_data_t *d, size_t input_limit, ErlNifBinary *output_buffer, int flush, + size_t *bytes_produced, size_t *bytes_consumed, size_t *bytes_remaining) { + + int vec_len, vec_idx; + SysIOVec *input_vec; + int res; + + input_vec = enif_ioq_peek(d->input_queue, &vec_len); + vec_idx = 0; + res = Z_OK; + + *bytes_produced = 0; + *bytes_consumed = 0; + + d->s.avail_out = output_buffer->size; + d->s.next_out = output_buffer->data; + + while(res == Z_OK && vec_idx < vec_len && *bytes_consumed < input_limit) { + size_t timeslice_percent, block_consumed, block_size; + + block_size = MIN(input_vec[vec_idx].iov_len, input_limit); + + d->s.next_in = input_vec[vec_idx].iov_base; + d->s.avail_in = block_size; + + res = codec(&d->s, Z_NO_FLUSH); + + ASSERT(d->s.avail_in == 0 || d->s.avail_out == 0 || res != Z_OK); + + block_consumed = block_size - d->s.avail_in; + *bytes_consumed += block_consumed; + + if(d->want_input_crc) { + d->input_crc = + crc32(d->input_crc, input_vec[vec_idx].iov_base, block_consumed); + } + + timeslice_percent = (100 * block_consumed) / input_limit; + if(enif_consume_timeslice(env, MAX(1, timeslice_percent))) { + break; + } + + vec_idx++; + } + + if(!enif_ioq_deq(d->input_queue, *bytes_consumed, bytes_remaining)) { + *bytes_remaining = 0; + res = Z_BUF_ERROR; + } + + if(res == Z_OK && flush != Z_NO_FLUSH && (*bytes_remaining == 0)) { + d->s.next_in = NULL; + d->s.avail_in = 0; + + res = codec(&d->s, flush); + } + + *bytes_produced = output_buffer->size - d->s.avail_out; + + return res; +} + +static ERL_NIF_TERM zlib_codec(int (*codec)(z_stream*, int), + ErlNifEnv *env, zlib_data_t *d, + int input_chunk_size, + int output_chunk_size, + int flush) { + + size_t bytes_produced, bytes_consumed, bytes_remaining; + ErlNifBinary output_buffer; + int res; + + if(!enif_alloc_binary(output_chunk_size, &output_buffer)) { + return zlib_return(env, Z_MEM_ERROR); + } + + res = zlib_flush_queue(codec, env, d, input_chunk_size, &output_buffer, + flush, &bytes_produced, &bytes_consumed, &bytes_remaining); + + if(res < 0 && res != Z_BUF_ERROR) { + enif_release_binary(&output_buffer); + return zlib_return(env, res); + } + + if(res == Z_STREAM_END) { + d->eos_seen = 1; + } + + if(d->want_output_crc) { + d->output_crc = + crc32(d->output_crc, output_buffer.data, bytes_produced); + } + + if(bytes_consumed == 0 && bytes_produced == 0 && bytes_remaining != 0) { + /* Die if we've made zero progress; this should not happen on + * well-formed input. */ + + enif_release_binary(&output_buffer); + return zlib_return(env, Z_DATA_ERROR); + } else { + ERL_NIF_TERM flushed_output; + + if(bytes_produced > 0) { + if(bytes_produced < output_buffer.size) { + enif_realloc_binary(&output_buffer, bytes_produced); + } + + flushed_output = + enif_make_list1(env, enif_make_binary(env, &output_buffer)); + } else { + enif_release_binary(&output_buffer); + flushed_output = enif_make_list(env, 0); + } + + if(bytes_remaining == 0 && bytes_produced < output_chunk_size) { + return enif_make_tuple2(env, am_finished, flushed_output); + } else if(res != Z_NEED_DICT) { + return enif_make_tuple2(env, am_continue, flushed_output); + } + + return enif_make_tuple3(env, am_need_dictionary, + enif_make_int(env, d->s.adler), flushed_output); + } +} + +/* zlib nifs */ + +static ERL_NIF_TERM zlib_getStash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + if(argc != 1 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } + + if(d->stash_env == NULL) { + return am_empty; + } + + return enif_make_tuple2(env, am_ok, enif_make_copy(env, d->stash_term)); +} + +static ERL_NIF_TERM zlib_clearStash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + if(argc != 1 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->stash_env == NULL) { + return enif_raise_exception(env, am_error); + } + + enif_free_env(d->stash_env); + d->stash_env = NULL; + d->stash_term = NIL; + + return am_ok; +} + +static ERL_NIF_TERM zlib_setStash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + if(argc != 2 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->stash_env != NULL) { + return enif_raise_exception(env, am_error); + } + + d->stash_env = enif_alloc_env(); + d->stash_term = enif_make_copy(d->stash_env, argv[1]); + + return am_ok; +} + +static ERL_NIF_TERM zlib_open(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + ERL_NIF_TERM result; + + d = (zlib_data_t *) enif_alloc_resource(rtype_zlib, sizeof(zlib_data_t)); + + memset(&d->s, 0, sizeof(z_stream)); + + enif_self(env, &d->controlling_process); + + d->input_queue = enif_ioq_create(ERL_NIF_IOQ_NORMAL); + + d->s.zalloc = zlib_alloc; + d->s.zfree = zlib_free; + d->s.opaque = d; + d->s.data_type = Z_BINARY; + + d->eos_behavior = EOS_BEHAVIOR_CUT; + d->eos_seen = 0; + + d->state = ST_NONE; + + d->want_output_crc = 0; + d->want_input_crc = 0; + d->is_raw_stream = 0; + + d->output_crc = crc32(0L, Z_NULL, 0); + d->input_crc = crc32(0L, Z_NULL, 0); + + d->stash_env = NULL; + d->stash_term = NIL; + + d->inflateChunk_buffer_size = 4000; + + result = enif_make_resource(env, d); + enif_release_resource(d); + + return result; +} + +static ERL_NIF_TERM zlib_close(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + /* strictly speaking not needed since the gc will handle this */ + if(argc != 1 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state == ST_CLOSED) { + return enif_raise_exception(env, am_not_initialized); + } + + gc_zlib(env, d); + + return am_ok; +} + +/* deflate */ + +static ERL_NIF_TERM zlib_deflateInit(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + int level, method, windowBits, memLevel, strategy, res; + + if(argc != 6 || !get_zlib_data(env, argv[0], &d) + || !enif_get_int(env, argv[1], &level) + || !enif_get_int(env, argv[2], &method) + || !enif_get_int(env, argv[3], &windowBits) + || !enif_get_int(env, argv[4], &memLevel) + || !enif_get_int(env, argv[5], &strategy)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_NONE) { + return enif_raise_exception(env, am_already_initialized); + } + + res = deflateInit2(&d->s, level, method, windowBits, memLevel, strategy); + + if(res == Z_OK) { + d->state = ST_DEFLATE; + d->eos_seen = 0; + + d->is_raw_stream = (windowBits < 0); + + d->want_output_crc = 0; + d->want_input_crc = d->is_raw_stream; + + d->output_crc = crc32(0L, Z_NULL, 0); + d->input_crc = crc32(0L, Z_NULL, 0); + } + + return zlib_return(env, res); +} + +static ERL_NIF_TERM zlib_deflateSetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + ErlNifBinary bin; + int res; + + if(argc != 2 || !get_zlib_data(env, argv[0], &d) + || !enif_inspect_iolist_as_binary(env, argv[1], &bin)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_DEFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + + if((res = deflateSetDictionary(&d->s, bin.data, bin.size)) == Z_OK) { + uLong checksum = d->s.adler; + + /* d->s.adler is not updated in raw deflate mode, so we'll calculate it + * ourselves in case the user wants to rely on that behavior. */ + if(d->is_raw_stream) { + checksum = adler32(0, bin.data, bin.size); + } + + return enif_make_int(env, checksum); + } + + return zlib_return(env, res); +} + +static ERL_NIF_TERM zlib_deflateReset(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + int res; + + if(argc != 1 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_DEFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + + res = deflateReset(&d->s); + + d->input_crc = crc32(0L, Z_NULL, 0); + d->eos_seen = 0; + + zlib_reset_input(d); + + return zlib_return(env, res); +} + +static ERL_NIF_TERM zlib_deflateEnd(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + int res; + + if(argc != 1 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_DEFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + + res = deflateEnd(&d->s); + + if(res == Z_OK && enif_ioq_size(d->input_queue) > 0) { + res = Z_DATA_ERROR; + } + + zlib_reset_input(d); + d->state = ST_NONE; + + return zlib_return(env, res); +} + +static ERL_NIF_TERM zlib_deflateParams(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + int res, level, strategy; + + if(argc != 3 || !get_zlib_data(env, argv[0], &d) + || !enif_get_int(env, argv[1], &level) + || !enif_get_int(env, argv[2], &strategy)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_DEFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + + /* deflateParams will flush everything currently in the stream, corrupting + * the heap unless it's empty. We therefore pretend to have a full output + * buffer, forcing a Z_BUF_ERROR if there's anything left to be flushed. */ + d->s.avail_out = 0; + res = deflateParams(&d->s, level, strategy); + + return zlib_return(env, res); +} + +static ERL_NIF_TERM zlib_deflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + int input_chunk_size, output_chunk_size, flush; + + if(argc != 4 || !get_zlib_data(env, argv[0], &d) + || !enif_get_int(env, argv[1], &input_chunk_size) + || !enif_get_int(env, argv[2], &output_chunk_size) + || !enif_get_int(env, argv[3], &flush)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_DEFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + + return zlib_codec(&deflate, env, d, input_chunk_size, output_chunk_size, flush); +} + +/* inflate */ + +static ERL_NIF_TERM zlib_inflateInit(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + int windowBits, eosBehavior, res; + + if(argc != 3 || !get_zlib_data(env, argv[0], &d) + || !enif_get_int(env, argv[1], &windowBits) + || !enif_get_int(env, argv[2], &eosBehavior)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_NONE) { + return enif_raise_exception(env, am_already_initialized); + } + + res = inflateInit2(&d->s, windowBits); + + if(res == Z_OK) { + d->state = ST_INFLATE; + + d->eos_behavior = eosBehavior; + d->eos_seen = 0; + + d->is_raw_stream = (windowBits < 0); + + d->want_output_crc = d->is_raw_stream; + d->want_input_crc = 0; + + d->output_crc = crc32(0L, Z_NULL, 0); + d->input_crc = crc32(0L, Z_NULL, 0); + } + + return zlib_return(env, res); +} + +static ERL_NIF_TERM zlib_inflateSetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + ErlNifBinary bin; + int res; + + if(argc != 2 || !get_zlib_data(env, argv[0], &d) + || !enif_inspect_iolist_as_binary(env, argv[1], &bin)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_INFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + + res = inflateSetDictionary(&d->s, bin.data, bin.size); + + return zlib_return(env, res); +} + +#ifdef HAVE_ZLIB_INFLATEGETDICTIONARY +/* Work around broken build system with runtime version test */ +static int zlib_supports_inflateGetDictionary(void) { + static int supportsGetDictionary = -1; + +#if defined(__APPLE__) && defined(__MACH__) + if(supportsGetDictionary < 0) { + unsigned int v[4] = {0, 0, 0, 0}; + unsigned hexver; + + sscanf(zlibVersion(), "%u.%u.%u.%u", &v[0], &v[1], &v[2], &v[3]); + + hexver = (v[0] << (8*3)) | (v[1] << (8*2)) | (v[2] << (8)) | v[3]; + supportsGetDictionary = (hexver >= 0x1020701); /* 1.2.7.1 */ + } +#endif + + return supportsGetDictionary; +} +#endif + +static ERL_NIF_TERM zlib_inflateGetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + if(argc != 1 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_INFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + +#ifdef HAVE_ZLIB_INFLATEGETDICTIONARY + if(zlib_supports_inflateGetDictionary()) { + ErlNifBinary obin; + uInt len; + int res; + + enif_alloc_binary(INFL_DICT_SZ, &obin); + len = 0; + + if((res = inflateGetDictionary(&d->s, obin.data, &len)) < 0) { + enif_release_binary(&obin); + return zlib_return(env, res); + } + + enif_realloc_binary(&obin, (size_t)len); + return enif_make_binary(env, &obin); + } +#endif + + return enif_raise_exception(env, am_not_supported); +} + +static ERL_NIF_TERM zlib_inflateReset(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + int res; + + if(argc != 1 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_INFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + + res = inflateReset(&d->s); + + d->output_crc = crc32(0L, Z_NULL, 0); + d->eos_seen = 0; + + zlib_reset_input(d); + + return zlib_return(env, res); +} + +static ERL_NIF_TERM zlib_inflateEnd(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + int res; + + if(argc != 1 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_INFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + + res = inflateEnd(&d->s); + + if(res == Z_OK && (!d->eos_seen || enif_ioq_size(d->input_queue) > 0)) { + res = Z_DATA_ERROR; + } + + zlib_reset_input(d); + d->state = ST_NONE; + + return zlib_return(env, res); +} + +static ERL_NIF_TERM zlib_inflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + int input_chunk_size, output_chunk_size, flush; + + if(argc != 4 || !get_zlib_data(env, argv[0], &d) + || !enif_get_int(env, argv[1], &input_chunk_size) + || !enif_get_int(env, argv[2], &output_chunk_size) + || !enif_get_int(env, argv[3], &flush)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_INFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + + if(d->eos_seen) { + int res; + + switch(d->eos_behavior) { + case EOS_BEHAVIOR_ERROR: + return zlib_return(env, Z_DATA_ERROR); + case EOS_BEHAVIOR_RESET: + res = inflateReset(&d->s); + + if(res != Z_OK) { + return zlib_return(env, res); + } + + d->eos_seen = 0; + break; + case EOS_BEHAVIOR_CUT: + zlib_reset_input(d); + + return enif_make_tuple2(env, am_finished, enif_make_list(env, 0)); + } + } + + return zlib_codec(&inflate, env, d, input_chunk_size, output_chunk_size, flush); +} + +static ERL_NIF_TERM zlib_crc32(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + if(argc != 1 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } + + if(d->state == ST_DEFLATE) { + return enif_make_ulong(env, d->input_crc); + } else if(d->state == ST_INFLATE) { + return enif_make_ulong(env, d->output_crc); + } + + return enif_raise_exception(env, am_not_initialized); +} + +static ERL_NIF_TERM zlib_getBufSize(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + if(argc != 1 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } + + return enif_make_int(env, d->inflateChunk_buffer_size); +} + +static ERL_NIF_TERM zlib_setBufSize(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + if(argc != 2 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } + + if(!enif_get_int(env, argv[1], &d->inflateChunk_buffer_size)) { + return enif_make_badarg(env); + } + + return am_ok; +} + +static ERL_NIF_TERM zlib_enqueue_input(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + ErlNifIOVec prealloc, *iovec = &prealloc; + ERL_NIF_TERM tail; + + if(argc != 2 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_DEFLATE && d->state != ST_INFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + + if(!enif_inspect_iovec(env, 256, argv[1], &tail, &iovec)) { + return enif_make_badarg(env); + } else if(!enif_ioq_enqv(d->input_queue, iovec, 0)) { + return enif_make_badarg(env); + } + + if(!enif_is_empty_list(env, tail)) { + return enif_make_tuple2(env, am_continue, tail); + } + + return am_ok; +} diff --git a/erts/emulator/pcre/README.pcre_update.md b/erts/emulator/pcre/README.pcre_update.md index 8caf575d31..599e3d0d12 100644 --- a/erts/emulator/pcre/README.pcre_update.md +++ b/erts/emulator/pcre/README.pcre_update.md @@ -2,7 +2,7 @@ ## The basic changes to the PCRE library -To work with the Erlang VM, PCRE has been changed in two important ways: +To work with the Erlang VM, PCRE has been changed in three important ways: 1. The main execution machine in pcre\_exec has been modified so that matching can be interrupted and restarted. This functionality utilizes diff --git a/erts/emulator/pcre/local_config.h b/erts/emulator/pcre/local_config.h index e90f4dcada..c6af423d72 100644 --- a/erts/emulator/pcre/local_config.h +++ b/erts/emulator/pcre/local_config.h @@ -86,4 +86,4 @@ #define SUPPORT_UTF /* Version number of package */ -#define VERSION "8.40" +#define VERSION "8.41" diff --git a/erts/emulator/pcre/pcre-8.40.tar.bz2 b/erts/emulator/pcre/pcre-8.40.tar.bz2 Binary files differdeleted file mode 100644 index 6147917f4e..0000000000 --- a/erts/emulator/pcre/pcre-8.40.tar.bz2 +++ /dev/null diff --git a/erts/emulator/pcre/pcre-8.41.tar.bz2 b/erts/emulator/pcre/pcre-8.41.tar.bz2 Binary files differnew file mode 100644 index 0000000000..1798432dc9 --- /dev/null +++ b/erts/emulator/pcre/pcre-8.41.tar.bz2 diff --git a/erts/emulator/pcre/pcre.h b/erts/emulator/pcre/pcre.h index 9cbd9c0293..ab8f40cfc1 100644 --- a/erts/emulator/pcre/pcre.h +++ b/erts/emulator/pcre/pcre.h @@ -43,9 +43,9 @@ POSSIBILITY OF SUCH DAMAGE. /* The current PCRE version information. */ #define PCRE_MAJOR 8 -#define PCRE_MINOR 40 +#define PCRE_MINOR 41 #define PCRE_PRERELEASE -#define PCRE_DATE 2017-01-11 +#define PCRE_DATE 2017-07-05 /* When an application links to a PCRE DLL in Windows, the symbols that are imported have to be identified as such. When building PCRE, the appropriate diff --git a/erts/emulator/pcre/pcre_compile.c b/erts/emulator/pcre/pcre_compile.c index 6e841c9cf8..e79284ab79 100644 --- a/erts/emulator/pcre/pcre_compile.c +++ b/erts/emulator/pcre/pcre_compile.c @@ -5740,6 +5740,21 @@ for (;; ptr++) ptr = p - 1; /* Character before the next significant one. */ } + /* We also need to skip over (?# comments, which are not dependent on + extended mode. */ + + if (ptr[1] == CHAR_LEFT_PARENTHESIS && ptr[2] == CHAR_QUESTION_MARK && + ptr[3] == CHAR_NUMBER_SIGN) + { + ptr += 4; + while (*ptr != CHAR_NULL && *ptr != CHAR_RIGHT_PARENTHESIS) ptr++; + if (*ptr == CHAR_NULL) + { + *errorcodeptr = ERR18; + goto FAILED; + } + } + /* If the next character is '+', we have a possessive quantifier. This implies greediness, whatever the setting of the PCRE_UNGREEDY option. If the next character is '?' this is a minimizing repeat, by default, @@ -8211,7 +8226,6 @@ for (;; ptr++) if (mclength == 1 || req_caseopt == 0) { - firstchar = mcbuffer[0] | req_caseopt; firstchar = mcbuffer[0]; firstcharflags = req_caseopt; diff --git a/erts/emulator/pcre/pcre_dfa_exec.c b/erts/emulator/pcre/pcre_dfa_exec.c index 529f40685b..c859d67fc7 100644 --- a/erts/emulator/pcre/pcre_dfa_exec.c +++ b/erts/emulator/pcre/pcre_dfa_exec.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language (but see below for why this module is different). Written by Philip Hazel - Copyright (c) 1997-2014 University of Cambridge + Copyright (c) 1997-2017 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -2626,7 +2626,7 @@ for (;;) if (isinclass) { int max = (int)GET2(ecode, 1 + IMM2_SIZE); - if (*ecode == OP_CRPOSRANGE) + if (*ecode == OP_CRPOSRANGE && count >= (int)GET2(ecode, 1)) { active_count--; /* Remove non-match possibility */ next_active_state--; diff --git a/erts/emulator/pcre/pcre_exec.c b/erts/emulator/pcre/pcre_exec.c index 0f682d3daf..6708ba92a6 100644 --- a/erts/emulator/pcre/pcre_exec.c +++ b/erts/emulator/pcre/pcre_exec.c @@ -755,7 +755,7 @@ if (ecode == NULL) return match((PCRE_PUCHAR)&rdepth, NULL, NULL, 0, NULL, NULL, 1); else { - int len = (char *)&rdepth - (char *)eptr; + int len = (int)((char *)&rdepth - (char *)eptr); return (len > 0)? -len : len; } } diff --git a/erts/emulator/pcre/pcre_internal.h b/erts/emulator/pcre/pcre_internal.h index cc4f171438..c84dcb5a38 100644 --- a/erts/emulator/pcre/pcre_internal.h +++ b/erts/emulator/pcre/pcre_internal.h @@ -2791,6 +2791,9 @@ extern const pcre_uint8 PRIV(ucd_stage1)[]; extern const pcre_uint16 PRIV(ucd_stage2)[]; extern const pcre_uint32 PRIV(ucp_gentype)[]; extern const pcre_uint32 PRIV(ucp_gbtable)[]; +#ifdef COMPILE_PCRE32 +extern const ucd_record PRIV(dummy_ucd_record)[]; +#endif #ifdef SUPPORT_JIT extern const int PRIV(ucp_typerange)[]; #endif @@ -2799,10 +2802,16 @@ extern const int PRIV(ucp_typerange)[]; /* UCD access macros */ #define UCD_BLOCK_SIZE 128 -#define GET_UCD(ch) (PRIV(ucd_records) + \ +#define REAL_GET_UCD(ch) (PRIV(ucd_records) + \ PRIV(ucd_stage2)[PRIV(ucd_stage1)[(int)(ch) / UCD_BLOCK_SIZE] * \ UCD_BLOCK_SIZE + (int)(ch) % UCD_BLOCK_SIZE]) +#ifdef COMPILE_PCRE32 +#define GET_UCD(ch) ((ch > 0x10ffff)? PRIV(dummy_ucd_record) : REAL_GET_UCD(ch)) +#else +#define GET_UCD(ch) REAL_GET_UCD(ch) +#endif + #define UCD_CHARTYPE(ch) GET_UCD(ch)->chartype #define UCD_SCRIPT(ch) GET_UCD(ch)->script #define UCD_CATEGORY(ch) PRIV(ucp_gentype)[UCD_CHARTYPE(ch)] diff --git a/erts/emulator/pcre/pcre_jit_compile.c b/erts/emulator/pcre/pcre_jit_compile.c index 89400498f0..932ca2c389 100644 --- a/erts/emulator/pcre/pcre_jit_compile.c +++ b/erts/emulator/pcre/pcre_jit_compile.c @@ -487,7 +487,7 @@ typedef struct compare_context { #undef CMP /* Used for accessing the elements of the stack. */ -#define STACK(i) ((-(i) - 1) * (int)sizeof(sljit_sw)) +#define STACK(i) ((i) * (int)sizeof(sljit_sw)) #define TMP1 SLJIT_R0 #define TMP2 SLJIT_R2 @@ -552,13 +552,15 @@ the start pointers when the end of the capturing group has not yet reached. */ sljit_emit_cmp(compiler, (type), (src1), (src1w), (src2), (src2w)) #define CMPTO(type, src1, src1w, src2, src2w, label) \ sljit_set_label(sljit_emit_cmp(compiler, (type), (src1), (src1w), (src2), (src2w)), (label)) -#define OP_FLAGS(op, dst, dstw, src, srcw, type) \ - sljit_emit_op_flags(compiler, (op), (dst), (dstw), (src), (srcw), (type)) +#define OP_FLAGS(op, dst, dstw, type) \ + sljit_emit_op_flags(compiler, (op), (dst), (dstw), (type)) #define GET_LOCAL_BASE(dst, dstw, offset) \ sljit_get_local_base(compiler, (dst), (dstw), (offset)) #define READ_CHAR_MAX 0x7fffffff +#define INVALID_UTF_CHAR 888 + static pcre_uchar *bracketend(pcre_uchar *cc) { SLJIT_ASSERT((*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NOT) || (*cc >= OP_ONCE && *cc <= OP_SCOND)); @@ -784,7 +786,7 @@ switch(*cc) default: /* All opcodes are supported now! */ - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); return NULL; } } @@ -1660,9 +1662,9 @@ while (cc < ccend) { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0)); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -OVECTOR(0)); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); setsom_found = TRUE; } cc += 1; @@ -1676,9 +1678,9 @@ while (cc < ccend) { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->mark_ptr); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); setmark_found = TRUE; } cc += 1 + 2 + cc[1]; @@ -1689,27 +1691,27 @@ while (cc < ccend) { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0)); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -OVECTOR(0)); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); setsom_found = TRUE; } if (common->mark_ptr != 0 && !setmark_found) { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->mark_ptr); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); setmark_found = TRUE; } if (common->capture_last_ptr != 0 && !capture_last_found) { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->capture_last_ptr); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); capture_last_found = TRUE; } cc += 1 + LINK_SIZE; @@ -1723,20 +1725,20 @@ while (cc < ccend) { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->capture_last_ptr); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); capture_last_found = TRUE; } offset = (GET2(cc, 1 + LINK_SIZE)) << 1; OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, OVECTOR(offset)); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset)); OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1)); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP2, 0); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); cc += 1 + LINK_SIZE + IMM2_SIZE; break; @@ -1887,18 +1889,17 @@ BOOL tmp1empty = TRUE; BOOL tmp2empty = TRUE; pcre_uchar *alternative; enum { - start, loop, end } status; -status = save ? start : loop; -stackptr = STACK(stackptr - 2); +status = loop; +stackptr = STACK(stackptr); stacktop = STACK(stacktop - 1); if (!save) { - stackptr += (needs_control_head ? 2 : 1) * sizeof(sljit_sw); + stacktop -= (needs_control_head ? 2 : 1) * sizeof(sljit_sw); if (stackptr < stacktop) { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), stackptr); @@ -1914,196 +1915,186 @@ if (!save) /* The tmp1next must be TRUE in either way. */ } +SLJIT_ASSERT(common->recursive_head_ptr != 0); + do { count = 0; - switch(status) + if (cc >= ccend) { - case start: - SLJIT_ASSERT(save && common->recursive_head_ptr != 0); + if (!save) + break; + count = 1; srcw[0] = common->recursive_head_ptr; if (needs_control_head) { SLJIT_ASSERT(common->control_head_ptr != 0); count = 2; - srcw[1] = common->control_head_ptr; + srcw[0] = common->control_head_ptr; + srcw[1] = common->recursive_head_ptr; + } + status = end; + } + else switch(*cc) + { + case OP_KET: + if (PRIVATE_DATA(cc) != 0) + { + count = 1; + srcw[0] = PRIVATE_DATA(cc); + SLJIT_ASSERT(PRIVATE_DATA(cc + 1) != 0); + cc += PRIVATE_DATA(cc + 1); } - status = loop; + cc += 1 + LINK_SIZE; + break; + + case OP_ASSERT: + case OP_ASSERT_NOT: + case OP_ASSERTBACK: + case OP_ASSERTBACK_NOT: + case OP_ONCE: + case OP_ONCE_NC: + case OP_BRAPOS: + case OP_SBRA: + case OP_SBRAPOS: + case OP_SCOND: + count = 1; + srcw[0] = PRIVATE_DATA(cc); + SLJIT_ASSERT(srcw[0] != 0); + cc += 1 + LINK_SIZE; break; - case loop: - if (cc >= ccend) + case OP_CBRA: + case OP_SCBRA: + if (common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0) { - status = end; - break; + count = 1; + srcw[0] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE)); } + cc += 1 + LINK_SIZE + IMM2_SIZE; + break; - switch(*cc) - { - case OP_KET: - if (PRIVATE_DATA(cc) != 0) - { - count = 1; - srcw[0] = PRIVATE_DATA(cc); - SLJIT_ASSERT(PRIVATE_DATA(cc + 1) != 0); - cc += PRIVATE_DATA(cc + 1); - } - cc += 1 + LINK_SIZE; - break; + case OP_CBRAPOS: + case OP_SCBRAPOS: + count = 2; + srcw[0] = PRIVATE_DATA(cc); + srcw[1] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE)); + SLJIT_ASSERT(srcw[0] != 0 && srcw[1] != 0); + cc += 1 + LINK_SIZE + IMM2_SIZE; + break; - case OP_ASSERT: - case OP_ASSERT_NOT: - case OP_ASSERTBACK: - case OP_ASSERTBACK_NOT: - case OP_ONCE: - case OP_ONCE_NC: - case OP_BRAPOS: - case OP_SBRA: - case OP_SBRAPOS: - case OP_SCOND: + case OP_COND: + /* Might be a hidden SCOND. */ + alternative = cc + GET(cc, 1); + if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN) + { count = 1; srcw[0] = PRIVATE_DATA(cc); SLJIT_ASSERT(srcw[0] != 0); - cc += 1 + LINK_SIZE; - break; - - case OP_CBRA: - case OP_SCBRA: - if (common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0) - { - count = 1; - srcw[0] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE)); - } - cc += 1 + LINK_SIZE + IMM2_SIZE; - break; + } + cc += 1 + LINK_SIZE; + break; - case OP_CBRAPOS: - case OP_SCBRAPOS: - count = 2; + CASE_ITERATOR_PRIVATE_DATA_1 + if (PRIVATE_DATA(cc)) + { + count = 1; srcw[0] = PRIVATE_DATA(cc); - srcw[1] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE)); - SLJIT_ASSERT(srcw[0] != 0 && srcw[1] != 0); - cc += 1 + LINK_SIZE + IMM2_SIZE; - break; - - case OP_COND: - /* Might be a hidden SCOND. */ - alternative = cc + GET(cc, 1); - if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN) - { - count = 1; - srcw[0] = PRIVATE_DATA(cc); - SLJIT_ASSERT(srcw[0] != 0); - } - cc += 1 + LINK_SIZE; - break; - - CASE_ITERATOR_PRIVATE_DATA_1 - if (PRIVATE_DATA(cc)) - { - count = 1; - srcw[0] = PRIVATE_DATA(cc); - } - cc += 2; + } + cc += 2; #ifdef SUPPORT_UTF - if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); + if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); #endif - break; + break; - CASE_ITERATOR_PRIVATE_DATA_2A - if (PRIVATE_DATA(cc)) - { - count = 2; - srcw[0] = PRIVATE_DATA(cc); - srcw[1] = PRIVATE_DATA(cc) + sizeof(sljit_sw); - } - cc += 2; + CASE_ITERATOR_PRIVATE_DATA_2A + if (PRIVATE_DATA(cc)) + { + count = 2; + srcw[0] = PRIVATE_DATA(cc); + srcw[1] = PRIVATE_DATA(cc) + sizeof(sljit_sw); + } + cc += 2; #ifdef SUPPORT_UTF - if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); + if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); #endif - break; + break; - CASE_ITERATOR_PRIVATE_DATA_2B - if (PRIVATE_DATA(cc)) - { - count = 2; - srcw[0] = PRIVATE_DATA(cc); - srcw[1] = PRIVATE_DATA(cc) + sizeof(sljit_sw); - } - cc += 2 + IMM2_SIZE; + CASE_ITERATOR_PRIVATE_DATA_2B + if (PRIVATE_DATA(cc)) + { + count = 2; + srcw[0] = PRIVATE_DATA(cc); + srcw[1] = PRIVATE_DATA(cc) + sizeof(sljit_sw); + } + cc += 2 + IMM2_SIZE; #ifdef SUPPORT_UTF - if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); + if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); #endif - break; + break; - CASE_ITERATOR_TYPE_PRIVATE_DATA_1 - if (PRIVATE_DATA(cc)) + CASE_ITERATOR_TYPE_PRIVATE_DATA_1 + if (PRIVATE_DATA(cc)) + { + count = 1; + srcw[0] = PRIVATE_DATA(cc); + } + cc += 1; + break; + + CASE_ITERATOR_TYPE_PRIVATE_DATA_2A + if (PRIVATE_DATA(cc)) + { + count = 2; + srcw[0] = PRIVATE_DATA(cc); + srcw[1] = srcw[0] + sizeof(sljit_sw); + } + cc += 1; + break; + + CASE_ITERATOR_TYPE_PRIVATE_DATA_2B + if (PRIVATE_DATA(cc)) + { + count = 2; + srcw[0] = PRIVATE_DATA(cc); + srcw[1] = srcw[0] + sizeof(sljit_sw); + } + cc += 1 + IMM2_SIZE; + break; + + case OP_CLASS: + case OP_NCLASS: +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + case OP_XCLASS: + size = (*cc == OP_XCLASS) ? GET(cc, 1) : 1 + 32 / (int)sizeof(pcre_uchar); +#else + size = 1 + 32 / (int)sizeof(pcre_uchar); +#endif + if (PRIVATE_DATA(cc)) + switch(get_class_iterator_size(cc + size)) { + case 1: count = 1; srcw[0] = PRIVATE_DATA(cc); - } - cc += 1; - break; + break; - CASE_ITERATOR_TYPE_PRIVATE_DATA_2A - if (PRIVATE_DATA(cc)) - { + case 2: count = 2; srcw[0] = PRIVATE_DATA(cc); srcw[1] = srcw[0] + sizeof(sljit_sw); - } - cc += 1; - break; + break; - CASE_ITERATOR_TYPE_PRIVATE_DATA_2B - if (PRIVATE_DATA(cc)) - { - count = 2; - srcw[0] = PRIVATE_DATA(cc); - srcw[1] = srcw[0] + sizeof(sljit_sw); + default: + SLJIT_UNREACHABLE(); + break; } - cc += 1 + IMM2_SIZE; - break; - - case OP_CLASS: - case OP_NCLASS: -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - case OP_XCLASS: - size = (*cc == OP_XCLASS) ? GET(cc, 1) : 1 + 32 / (int)sizeof(pcre_uchar); -#else - size = 1 + 32 / (int)sizeof(pcre_uchar); -#endif - if (PRIVATE_DATA(cc)) - switch(get_class_iterator_size(cc + size)) - { - case 1: - count = 1; - srcw[0] = PRIVATE_DATA(cc); - break; - - case 2: - count = 2; - srcw[0] = PRIVATE_DATA(cc); - srcw[1] = srcw[0] + sizeof(sljit_sw); - break; - - default: - SLJIT_ASSERT_STOP(); - break; - } - cc += size; - break; - - default: - cc = next_opcode(common, cc); - SLJIT_ASSERT(cc != NULL); - break; - } + cc += size; break; - case end: - SLJIT_ASSERT_STOP(); + default: + cc = next_opcode(common, cc); + SLJIT_ASSERT(cc != NULL); break; } @@ -2312,7 +2303,7 @@ static SLJIT_INLINE void count_match(compiler_common *common) { DEFINE_COMPILER; -OP2(SLJIT_SUB | SLJIT_SET_E, COUNT_MATCH, 0, COUNT_MATCH, 0, SLJIT_IMM, 1); +OP2(SLJIT_SUB | SLJIT_SET_Z, COUNT_MATCH, 0, COUNT_MATCH, 0, SLJIT_IMM, 1); add_jump(compiler, &common->calllimit, JUMP(SLJIT_ZERO)); } @@ -2322,7 +2313,7 @@ static SLJIT_INLINE void allocate_stack(compiler_common *common, int size) DEFINE_COMPILER; SLJIT_ASSERT(size > 0); -OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_sw)); +OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_sw)); #ifdef DESTROY_REGISTERS OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 12345); OP1(SLJIT_MOV, TMP3, 0, TMP1, 0); @@ -2330,7 +2321,7 @@ OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, TMP1, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP1, 0); #endif -add_stub(common, CMP(SLJIT_GREATER, STACK_TOP, 0, STACK_LIMIT, 0)); +add_stub(common, CMP(SLJIT_LESS, STACK_TOP, 0, STACK_LIMIT, 0)); } static SLJIT_INLINE void free_stack(compiler_common *common, int size) @@ -2338,7 +2329,7 @@ static SLJIT_INLINE void free_stack(compiler_common *common, int size) DEFINE_COMPILER; SLJIT_ASSERT(size > 0); -OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_sw)); +OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_sw)); } static sljit_uw * allocate_read_only_data(compiler_common *common, sljit_uw size) @@ -2396,7 +2387,7 @@ else OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_IMM, length - 1); loop = LABEL(); OP1(SLJIT_MOVU, SLJIT_MEM1(SLJIT_R1), sizeof(sljit_sw), SLJIT_R0, 0); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, loop); } } @@ -2434,7 +2425,7 @@ else OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_IMM, length - 2); loop = LABEL(); OP1(SLJIT_MOVU, SLJIT_MEM1(TMP2), sizeof(sljit_sw), TMP1, 0); - OP2(SLJIT_SUB | SLJIT_SET_E, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, loop); } @@ -2452,22 +2443,22 @@ static sljit_sw SLJIT_CALL do_search_mark(sljit_sw *current, const pcre_uchar *s { while (current != NULL) { - switch (current[-2]) + switch (current[1]) { case type_then_trap: break; case type_mark: - if (STRCMP_UC_UC(skip_arg, (pcre_uchar *)current[-3]) == 0) - return current[-4]; + if (STRCMP_UC_UC(skip_arg, (pcre_uchar *)current[2]) == 0) + return current[3]; break; default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); break; } - SLJIT_ASSERT(current > (sljit_sw*)current[-1]); - current = (sljit_sw*)current[-1]; + SLJIT_ASSERT(current[0] == 0 || current < (sljit_sw*)current[0]); + current = (sljit_sw*)current[0]; } return -1; } @@ -2501,7 +2492,7 @@ OP2(SLJIT_ADD, SLJIT_S0, 0, SLJIT_S0, 0, SLJIT_IMM, sizeof(sljit_sw)); OP2(SLJIT_ASHR, SLJIT_S1, 0, SLJIT_S1, 0, SLJIT_IMM, UCHAR_SHIFT); #endif OP1(SLJIT_MOVU_S32, SLJIT_MEM1(SLJIT_R2), sizeof(int), SLJIT_S1, 0); -OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_R1, 0, SLJIT_R1, 0, SLJIT_IMM, 1); +OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_R1, 0, SLJIT_R1, 0, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, loop); JUMPHERE(early_quit); @@ -3106,8 +3097,8 @@ if (common->utf) OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); /* Skip low surrogate if necessary. */ OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xdc00); - OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xdc00); + OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP1, 0); return; @@ -3126,6 +3117,7 @@ struct sljit_jump *jump; if (nltype == NLTYPE_ANY) { add_jump(compiler, &common->anynewline, JUMP(SLJIT_FAST_CALL)); + sljit_set_current_flags(compiler, SLJIT_SET_Z); add_jump(compiler, backtracks, JUMP(jumpifmatch ? SLJIT_NOT_ZERO : SLJIT_ZERO)); } else if (nltype == NLTYPE_ANYCRLF) @@ -3167,7 +3159,7 @@ OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); /* Searching for the first zero. */ -OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); +OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); jump = JUMP(SLJIT_NOT_ZERO); /* Two byte sequence. */ OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); @@ -3181,7 +3173,7 @@ OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); -OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x10000); +OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x10000); jump = JUMP(SLJIT_NOT_ZERO); /* Three byte sequence. */ OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); @@ -3215,15 +3207,15 @@ OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); /* Searching for the first zero. */ -OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); +OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); jump = JUMP(SLJIT_NOT_ZERO); /* Two byte sequence. */ OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); sljit_emit_fast_return(compiler, RETURN_ADDR, 0); JUMPHERE(jump); -OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x400); -OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_NOT_ZERO); +OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x400); +OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_NOT_ZERO); /* This code runs only in 8 bit mode. No need to shift the value. */ OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); @@ -3246,7 +3238,7 @@ struct sljit_jump *compare; sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); -OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0x20); +OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0x20); jump = JUMP(SLJIT_NOT_ZERO); /* Two byte sequence. */ OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); @@ -3287,10 +3279,30 @@ static void do_getucd(compiler_common *common) /* Search the UCD record for the character comes in TMP1. Returns chartype in TMP1 and UCD offset in TMP2. */ DEFINE_COMPILER; +#ifdef COMPILE_PCRE32 +struct sljit_jump *jump; +#endif + +#if defined SLJIT_DEBUG && SLJIT_DEBUG +/* dummy_ucd_record */ +const ucd_record *record = GET_UCD(INVALID_UTF_CHAR); +SLJIT_ASSERT(record->script == ucp_Common && record->chartype == ucp_Cn && record->gbprop == ucp_gbOther); +SLJIT_ASSERT(record->caseset == 0 && record->other_case == 0); +#endif SLJIT_ASSERT(UCD_BLOCK_SIZE == 128 && sizeof(ucd_record) == 8); sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); + +#ifdef COMPILE_PCRE32 +if (!common->utf) + { + jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x10ffff + 1); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); + JUMPHERE(jump); + } +#endif + OP2(SLJIT_LSHR, TMP2, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_SHIFT); OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_stage1)); OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_MASK); @@ -3365,8 +3377,8 @@ if (newlinecheck) OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); end = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, common->newline & 0xff); - OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, common->newline & 0xff); + OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); #if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT); #endif @@ -3403,8 +3415,8 @@ if (common->utf) { singlechar = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xd800); OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800); - OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800); + OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); JUMPHERE(singlechar); @@ -3853,7 +3865,7 @@ while (TRUE) } } -#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) +#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) && !(defined SUPPORT_VALGRIND) static sljit_s32 character_to_int32(pcre_uchar chr) { @@ -4019,6 +4031,7 @@ instruction[0] = 0x0f; instruction[1] = 0xbc; instruction[2] = 0xc0 | (tmp1_ind << 3) | tmp1_ind; sljit_emit_op_custom(compiler, instruction, 3); +sljit_set_current_flags(compiler, SLJIT_SET_Z); nomatch = JUMP(SLJIT_ZERO); @@ -4119,6 +4132,7 @@ instruction[0] = 0x0f; instruction[1] = 0xbc; instruction[2] = 0xc0 | (tmp1_ind << 3) | tmp1_ind; sljit_emit_op_custom(compiler, instruction, 3); +sljit_set_current_flags(compiler, SLJIT_SET_Z); JUMPTO(SLJIT_ZERO, start); @@ -4155,18 +4169,8 @@ if (has_match_end) OP1(SLJIT_MOV, TMP3, 0, STR_END, 0); OP2(SLJIT_ADD, STR_END, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr, SLJIT_IMM, IN_UCHARS(offset + 1)); -#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) - if (sljit_x86_is_cmov_available()) - { - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, STR_END, 0, TMP3, 0); - sljit_x86_emit_cmov(compiler, SLJIT_GREATER, STR_END, TMP3, 0); - } -#endif - { - quit = CMP(SLJIT_LESS_EQUAL, STR_END, 0, TMP3, 0); - OP1(SLJIT_MOV, STR_END, 0, TMP3, 0); - JUMPHERE(quit); - } + OP2(SLJIT_SUB | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, STR_END, 0, TMP3, 0); + sljit_emit_cmov(compiler, SLJIT_GREATER, STR_END, TMP3, 0); } #if defined SUPPORT_UTF && !defined COMPILE_PCRE32 @@ -4174,11 +4178,11 @@ if (common->utf && offset > 0) utf_start = LABEL(); #endif -#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) +#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) && !(defined SUPPORT_VALGRIND) /* SSE2 accelerated first character search. */ -if (sljit_x86_is_sse2_available()) +if (sljit_has_cpu_feature(SLJIT_HAS_SSE2)) { fast_forward_first_char2_sse2(common, char1, char2); @@ -4213,16 +4217,16 @@ if (sljit_x86_is_sse2_available()) if (offset > 0) OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset)); } - else if (sljit_x86_is_cmov_available()) - { - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, STR_PTR, 0, STR_END, 0); - sljit_x86_emit_cmov(compiler, SLJIT_GREATER_EQUAL, STR_PTR, has_match_end ? SLJIT_MEM1(SLJIT_SP) : STR_END, has_match_end ? common->match_end_ptr : 0); - } else { - quit = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0); - OP1(SLJIT_MOV, STR_PTR, 0, has_match_end ? SLJIT_MEM1(SLJIT_SP) : STR_END, has_match_end ? common->match_end_ptr : 0); - JUMPHERE(quit); + OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, STR_PTR, 0, STR_END, 0); + if (has_match_end) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr); + sljit_emit_cmov(compiler, SLJIT_GREATER_EQUAL, STR_PTR, TMP1, 0); + } + else + sljit_emit_cmov(compiler, SLJIT_GREATER_EQUAL, STR_PTR, STR_END, 0); } if (has_match_end) @@ -4249,10 +4253,10 @@ else } else { - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, char1); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, char2); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, char1); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, char2); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL); found = JUMP(SLJIT_NOT_ZERO); } } @@ -4571,8 +4575,8 @@ if (common->nltype == NLTYPE_FIXED && common->newline > 255) firstchar = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0); OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(2)); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, STR_PTR, 0, TMP1, 0); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_GREATER_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, STR_PTR, 0, TMP1, 0); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_GREATER_EQUAL); #if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCHAR_SHIFT); #endif @@ -4616,8 +4620,8 @@ if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF) JUMPHERE(foundcr); notfoundnl = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_NL); - OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_NL); + OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); #if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT); #endif @@ -4670,7 +4674,7 @@ if (!check_class_ranges(common, start_bits, (start_bits[31] & 0x80) != 0, TRUE, OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3); OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)start_bits); OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0); - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); found = JUMP(SLJIT_NOT_ZERO); } @@ -4692,8 +4696,8 @@ if (common->utf) { CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xd800, start); OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800); - OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800); + OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); } @@ -4780,31 +4784,31 @@ struct sljit_jump *jump; struct sljit_label *mainloop; sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); -OP1(SLJIT_MOV, TMP1, 0, STACK_TOP, 0); -GET_LOCAL_BASE(TMP3, 0, 0); +OP1(SLJIT_MOV, TMP3, 0, STACK_TOP, 0); +GET_LOCAL_BASE(TMP1, 0, 0); /* Drop frames until we reach STACK_TOP. */ mainloop = LABEL(); -OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), 0); -OP2(SLJIT_SUB | SLJIT_SET_S, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0); -jump = JUMP(SLJIT_SIG_LESS_EQUAL); - -OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP3, 0); -OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(TMP1), sizeof(sljit_sw)); -OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), sizeof(sljit_sw), SLJIT_MEM1(TMP1), 2 * sizeof(sljit_sw)); -OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 3 * sizeof(sljit_sw)); +OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), -sizeof(sljit_sw)); +jump = CMP(SLJIT_SIG_LESS_EQUAL, TMP2, 0, SLJIT_IMM, 0); + +OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); +OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(STACK_TOP), -2 * sizeof(sljit_sw)); +OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), sizeof(sljit_sw), SLJIT_MEM1(STACK_TOP), -3 * sizeof(sljit_sw)); +OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 3 * sizeof(sljit_sw)); JUMPTO(SLJIT_JUMP, mainloop); JUMPHERE(jump); -jump = JUMP(SLJIT_SIG_LESS); -/* End of dropping frames. */ +jump = CMP(SLJIT_NOT_ZERO /* SIG_LESS */, TMP2, 0, SLJIT_IMM, 0); +/* End of reverting values. */ +OP1(SLJIT_MOV, STACK_TOP, 0, TMP3, 0); sljit_emit_fast_return(compiler, RETURN_ADDR, 0); JUMPHERE(jump); OP1(SLJIT_NEG, TMP2, 0, TMP2, 0); -OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP3, 0); -OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(TMP1), sizeof(sljit_sw)); -OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 2 * sizeof(sljit_sw)); +OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); +OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(STACK_TOP), -2 * sizeof(sljit_sw)); +OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 2 * sizeof(sljit_sw)); JUMPTO(SLJIT_JUMP, mainloop); } @@ -4837,11 +4841,11 @@ if (common->use_ucp) jump = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE); add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL)); OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Ll); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Nd - ucp_Ll); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL); JUMPHERE(jump); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP2, 0); } @@ -4881,11 +4885,11 @@ if (common->use_ucp) jump = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE); add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL)); OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Ll); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Nd - ucp_Ll); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL); JUMPHERE(jump); } else @@ -4913,7 +4917,7 @@ else } set_jumps(skipread_list, LABEL()); -OP2(SLJIT_XOR | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1); +OP2(SLJIT_XOR | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1); sljit_emit_fast_return(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0); } @@ -5064,7 +5068,7 @@ switch(length) return TRUE; default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); return FALSE; } } @@ -5077,22 +5081,22 @@ DEFINE_COMPILER; sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0a); -OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a); -OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); -OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a); +OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a); +OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); +OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a); #if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32 #ifdef COMPILE_PCRE8 if (common->utf) { #endif - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a); #ifdef COMPILE_PCRE8 } #endif #endif /* SUPPORT_UTF || COMPILE_PCRE16 || COMPILE_PCRE32 */ -OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); +OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL); sljit_emit_fast_return(compiler, RETURN_ADDR, 0); } @@ -5103,34 +5107,34 @@ DEFINE_COMPILER; sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); -OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x09); -OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); -OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x20); -OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); -OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xa0); +OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x09); +OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); +OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x20); +OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); +OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xa0); #if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32 #ifdef COMPILE_PCRE8 if (common->utf) { #endif - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x1680); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x1680); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x2000); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x200A - 0x2000); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x202f - 0x2000); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x205f - 0x2000); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x3000 - 0x2000); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x200A - 0x2000); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x202f - 0x2000); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x205f - 0x2000); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x3000 - 0x2000); #ifdef COMPILE_PCRE8 } #endif #endif /* SUPPORT_UTF || COMPILE_PCRE16 || COMPILE_PCRE32 */ -OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); +OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL); sljit_emit_fast_return(compiler, RETURN_ADDR, 0); } @@ -5143,22 +5147,22 @@ DEFINE_COMPILER; sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0a); -OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a); -OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); -OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a); +OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a); +OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); +OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a); #if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32 #ifdef COMPILE_PCRE8 if (common->utf) { #endif - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a); #ifdef COMPILE_PCRE8 } #endif #endif /* SUPPORT_UTF || COMPILE_PCRE16 || COMPILE_PCRE32 */ -OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); +OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL); sljit_emit_fast_return(compiler, RETURN_ADDR, 0); } @@ -5183,7 +5187,7 @@ label = LABEL(); OP1(MOVU_UCHAR, CHAR1, 0, SLJIT_MEM1(TMP1), IN_UCHARS(1)); OP1(MOVU_UCHAR, CHAR2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); jump = CMP(SLJIT_NOT_EQUAL, CHAR1, 0, CHAR2, 0); -OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1)); +OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1)); JUMPTO(SLJIT_NOT_ZERO, label); JUMPHERE(jump); @@ -5227,7 +5231,7 @@ OP1(SLJIT_MOV_U8, CHAR2, 0, SLJIT_MEM2(LCC_TABLE, CHAR2), 0); JUMPHERE(jump); #endif jump = CMP(SLJIT_NOT_EQUAL, CHAR1, 0, CHAR2, 0); -OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1)); +OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1)); JUMPTO(SLJIT_NOT_ZERO, label); JUMPHERE(jump); @@ -5394,7 +5398,7 @@ do #endif default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); break; } context->ucharptr = 0; @@ -5568,7 +5572,7 @@ while (*cc != XCL_END) break; default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); break; } cc += 2; @@ -5592,7 +5596,7 @@ if ((cc[-1] & XCL_HASPROP) == 0) OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3); OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc); OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0); - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); add_jump(compiler, &found, JUMP(SLJIT_NOT_ZERO)); } @@ -5625,7 +5629,7 @@ else if ((cc[-1] & XCL_MAP) != 0) OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3); OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc); OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0); - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); add_jump(compiler, list, JUMP(SLJIT_NOT_ZERO)); #ifdef COMPILE_PCRE8 @@ -5644,6 +5648,15 @@ if (needstype || needsscript) if (needschar && !charsaved) OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0); +#ifdef COMPILE_PCRE32 + if (!common->utf) + { + jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x10ffff + 1); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); + JUMPHERE(jump); + } +#endif + OP2(SLJIT_LSHR, TMP2, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_SHIFT); OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_stage1)); OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_MASK); @@ -5735,14 +5748,14 @@ while (*cc != XCL_END) if (numberofcmps < 3 && (*cc == XCL_SINGLE || *cc == XCL_RANGE)) { - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); - OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, numberofcmps == 0 ? SLJIT_UNUSED : TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); + OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_EQUAL); numberofcmps++; } else if (numberofcmps > 0) { - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL); jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); numberofcmps = 0; } @@ -5761,14 +5774,14 @@ while (*cc != XCL_END) if (numberofcmps < 3 && (*cc == XCL_SINGLE || *cc == XCL_RANGE)) { - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); - OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, numberofcmps == 0 ? SLJIT_UNUSED : TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); + OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL); numberofcmps++; } else if (numberofcmps > 0) { - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_LESS_EQUAL); jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); numberofcmps = 0; } @@ -5793,12 +5806,12 @@ while (*cc != XCL_END) break; case PT_LAMP: - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - typeoffset); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Ll - typeoffset); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lt - typeoffset); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - typeoffset); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Ll - typeoffset); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lt - typeoffset); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL); jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); break; @@ -5820,33 +5833,33 @@ while (*cc != XCL_END) case PT_SPACE: case PT_PXSPACE: SET_CHAR_OFFSET(9); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd - 0x9); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd - 0x9); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x9); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x9); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e - 0x9); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e - 0x9); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); SET_TYPE_OFFSET(ucp_Zl); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Zl); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Zl); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_LESS_EQUAL); jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); break; case PT_WORD: - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_UNDERSCORE - charoffset)); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_UNDERSCORE - charoffset)); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); /* Fall through. */ case PT_ALNUM: SET_TYPE_OFFSET(ucp_Ll); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - ucp_Ll); - OP_FLAGS((*cc == PT_ALNUM) ? SLJIT_MOV : SLJIT_OR, TMP2, 0, (*cc == PT_ALNUM) ? SLJIT_UNUSED : TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - ucp_Ll); + OP_FLAGS((*cc == PT_ALNUM) ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL); SET_TYPE_OFFSET(ucp_Nd); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_No - ucp_Nd); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_No - ucp_Nd); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_LESS_EQUAL); jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); break; @@ -5868,8 +5881,8 @@ while (*cc != XCL_END) OP2(SLJIT_ADD, TMP2, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)charoffset); OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, other_cases[1] ^ other_cases[0]); } - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, other_cases[1]); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, other_cases[1]); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); other_cases += 2; } else if (is_powerof2(other_cases[2] ^ other_cases[1])) @@ -5881,63 +5894,63 @@ while (*cc != XCL_END) OP2(SLJIT_ADD, TMP2, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)charoffset); OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, other_cases[1] ^ other_cases[0]); } - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, other_cases[2]); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, other_cases[2]); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(other_cases[0] - charoffset)); - OP_FLAGS(SLJIT_OR | ((other_cases[3] == NOTACHAR) ? SLJIT_SET_E : 0), TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(other_cases[0] - charoffset)); + OP_FLAGS(SLJIT_OR | ((other_cases[3] == NOTACHAR) ? SLJIT_SET_Z : 0), TMP2, 0, SLJIT_EQUAL); other_cases += 3; } else { - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(*other_cases++ - charoffset)); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(*other_cases++ - charoffset)); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); } while (*other_cases != NOTACHAR) { - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(*other_cases++ - charoffset)); - OP_FLAGS(SLJIT_OR | ((*other_cases == NOTACHAR) ? SLJIT_SET_E : 0), TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(*other_cases++ - charoffset)); + OP_FLAGS(SLJIT_OR | ((*other_cases == NOTACHAR) ? SLJIT_SET_Z : 0), TMP2, 0, SLJIT_EQUAL); } jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); break; case PT_UCNC: - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_DOLLAR_SIGN - charoffset)); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_COMMERCIAL_AT - charoffset)); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_GRAVE_ACCENT - charoffset)); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_DOLLAR_SIGN - charoffset)); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_COMMERCIAL_AT - charoffset)); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_GRAVE_ACCENT - charoffset)); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); SET_CHAR_OFFSET(0xa0); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(0xd7ff - charoffset)); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(0xd7ff - charoffset)); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL); SET_CHAR_OFFSET(0); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xe000 - 0); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_GREATER_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xe000 - 0); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_GREATER_EQUAL); jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); break; case PT_PXGRAPH: /* C and Z groups are the farthest two groups. */ SET_TYPE_OFFSET(ucp_Ll); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Ll); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_GREATER); + OP2(SLJIT_SUB | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Ll); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_GREATER); jump = CMP(SLJIT_NOT_EQUAL, typereg, 0, SLJIT_IMM, ucp_Cf - ucp_Ll); /* In case of ucp_Cf, we overwrite the result. */ SET_CHAR_OFFSET(0x2066); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2069 - 0x2066); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2069 - 0x2066); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x061c - 0x2066); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x061c - 0x2066); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e - 0x2066); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e - 0x2066); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); JUMPHERE(jump); jump = CMP(SLJIT_ZERO ^ invertcmp, TMP2, 0, SLJIT_IMM, 0); @@ -5946,21 +5959,21 @@ while (*cc != XCL_END) case PT_PXPRINT: /* C and Z groups are the farthest two groups. */ SET_TYPE_OFFSET(ucp_Ll); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Ll); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_GREATER); + OP2(SLJIT_SUB | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Ll); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_GREATER); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Ll); - OP_FLAGS(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_NOT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Ll); + OP_FLAGS(SLJIT_AND, TMP2, 0, SLJIT_NOT_EQUAL); jump = CMP(SLJIT_NOT_EQUAL, typereg, 0, SLJIT_IMM, ucp_Cf - ucp_Ll); /* In case of ucp_Cf, we overwrite the result. */ SET_CHAR_OFFSET(0x2066); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2069 - 0x2066); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2069 - 0x2066); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x061c - 0x2066); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x061c - 0x2066); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); JUMPHERE(jump); jump = CMP(SLJIT_ZERO ^ invertcmp, TMP2, 0, SLJIT_IMM, 0); @@ -5968,21 +5981,21 @@ while (*cc != XCL_END) case PT_PXPUNCT: SET_TYPE_OFFSET(ucp_Sc); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Sc); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Sc); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); SET_CHAR_OFFSET(0); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x7f); - OP_FLAGS(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x7f); + OP_FLAGS(SLJIT_AND, TMP2, 0, SLJIT_LESS_EQUAL); SET_TYPE_OFFSET(ucp_Pc); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Ps - ucp_Pc); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Ps - ucp_Pc); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_LESS_EQUAL); jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); break; default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); break; } cc += 2; @@ -6028,6 +6041,7 @@ switch(type) case OP_NOT_WORD_BOUNDARY: case OP_WORD_BOUNDARY: add_jump(compiler, &common->wordboundary, JUMP(SLJIT_FAST_CALL)); + sljit_set_current_flags(compiler, SLJIT_SET_Z); add_jump(compiler, backtracks, JUMP(type == OP_NOT_WORD_BOUNDARY ? SLJIT_NOT_ZERO : SLJIT_ZERO)); return cc; @@ -6043,10 +6057,10 @@ switch(type) else { jump[1] = CMP(SLJIT_EQUAL, TMP2, 0, STR_END, 0); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_NOT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_NOT_EQUAL); add_jump(compiler, backtracks, JUMP(SLJIT_NOT_EQUAL)); check_partial(common, TRUE); add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); @@ -6068,9 +6082,9 @@ switch(type) OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); jump[1] = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR); OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0); + OP2(SLJIT_SUB | SLJIT_SET_Z | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0); jump[2] = JUMP(SLJIT_GREATER); - add_jump(compiler, backtracks, JUMP(SLJIT_LESS)); + add_jump(compiler, backtracks, JUMP(SLJIT_NOT_EQUAL) /* LESS */); /* Equal. */ OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); jump[3] = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL); @@ -6089,6 +6103,7 @@ switch(type) read_char_range(common, common->nlmin, common->nlmax, TRUE); add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, STR_END, 0)); add_jump(compiler, &common->anynewline, JUMP(SLJIT_FAST_CALL)); + sljit_set_current_flags(compiler, SLJIT_SET_Z); add_jump(compiler, backtracks, JUMP(SLJIT_ZERO)); OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1); } @@ -6204,7 +6219,7 @@ switch(type) label = LABEL(); add_jump(compiler, backtracks, CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP3, 0)); skip_char_back(common); - OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, TMP2, 0, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, label); } else @@ -6217,7 +6232,7 @@ switch(type) check_start_used_ptr(common); return cc + LINK_SIZE; } -SLJIT_ASSERT_STOP(); +SLJIT_UNREACHABLE(); return cc; } @@ -6250,7 +6265,7 @@ switch(type) #endif read_char8_type(common, type == OP_NOT_DIGIT); /* Flip the starting bit in the negative case. */ - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_digit); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_digit); add_jump(compiler, backtracks, JUMP(type == OP_DIGIT ? SLJIT_ZERO : SLJIT_NOT_ZERO)); return cc; @@ -6264,7 +6279,7 @@ switch(type) else #endif read_char8_type(common, type == OP_NOT_WHITESPACE); - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_space); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_space); add_jump(compiler, backtracks, JUMP(type == OP_WHITESPACE ? SLJIT_ZERO : SLJIT_NOT_ZERO)); return cc; @@ -6278,7 +6293,7 @@ switch(type) else #endif read_char8_type(common, type == OP_NOT_WORDCHAR); - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_word); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_word); add_jump(compiler, backtracks, JUMP(type == OP_WORDCHAR ? SLJIT_ZERO : SLJIT_NOT_ZERO)); return cc; @@ -6320,8 +6335,8 @@ switch(type) #elif defined COMPILE_PCRE16 jump[0] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xd800); OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800); - OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800); + OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); #endif @@ -6383,6 +6398,7 @@ switch(type) detect_partial_match(common, backtracks); read_char_range(common, 0x9, 0x3000, type == OP_NOT_HSPACE); add_jump(compiler, &common->hspace, JUMP(SLJIT_FAST_CALL)); + sljit_set_current_flags(compiler, SLJIT_SET_Z); add_jump(compiler, backtracks, JUMP(type == OP_NOT_HSPACE ? SLJIT_NOT_ZERO : SLJIT_ZERO)); return cc; @@ -6392,6 +6408,7 @@ switch(type) detect_partial_match(common, backtracks); read_char_range(common, 0xa, 0x2029, type == OP_NOT_VSPACE); add_jump(compiler, &common->vspace, JUMP(SLJIT_FAST_CALL)); + sljit_set_current_flags(compiler, SLJIT_SET_Z); add_jump(compiler, backtracks, JUMP(type == OP_NOT_VSPACE ? SLJIT_NOT_ZERO : SLJIT_ZERO)); return cc; @@ -6418,7 +6435,7 @@ switch(type) OP1(SLJIT_MOV_U32, TMP1, 0, SLJIT_MEM1(STACK_TOP), (sljit_sw)PRIV(ucp_gbtable)); OP1(SLJIT_MOV, STACK_TOP, 0, TMP2, 0); OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0); - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); JUMPTO(SLJIT_NOT_ZERO, label); OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0); @@ -6587,7 +6604,7 @@ switch(type) OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3); OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc); OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0); - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); add_jump(compiler, backtracks, JUMP(SLJIT_ZERO)); #if defined SUPPORT_UTF || !defined COMPILE_PCRE8 @@ -6604,7 +6621,7 @@ switch(type) return cc + GET(cc, 0) - 1; #endif } -SLJIT_ASSERT_STOP(); +SLJIT_UNREACHABLE(); return cc; } @@ -6790,9 +6807,9 @@ else #endif /* SUPPORT_UTF && SUPPORT_UCP */ { if (ref) - OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), TMP1, 0); + OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), TMP1, 0); else - OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw), TMP1, 0); + OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw), TMP1, 0); if (withchecks) jump = JUMP(SLJIT_ZERO); @@ -6883,7 +6900,7 @@ switch(type) cc += 1 + IMM2_SIZE + 1 + 2 * IMM2_SIZE; break; default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); break; } @@ -6897,7 +6914,7 @@ if (!minimize) OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0); /* Temporary release of STR_PTR. */ - OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); + OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); /* Handles both invalid and empty cases. Since the minimum repeat, is zero the invalid case is basically the same as an empty case. */ if (ref) @@ -6910,7 +6927,7 @@ if (!minimize) zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw)); } /* Restore if not zero length. */ - OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); } else { @@ -7157,7 +7174,7 @@ return (*PUBL(callout))(callout_block); (((int)sizeof(PUBL(callout_block)) + 7) & ~7) #define CALLOUT_ARG_OFFSET(arg) \ - (-CALLOUT_ARG_SIZE + SLJIT_OFFSETOF(PUBL(callout_block), arg)) + SLJIT_OFFSETOF(PUBL(callout_block), arg) static SLJIT_INLINE pcre_uchar *compile_callout_matchingpath(compiler_common *common, pcre_uchar *cc, backtrack_common *parent) { @@ -7187,7 +7204,8 @@ OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(mark), (common->mark_pt /* Needed to save important temporary registers. */ OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STACK_TOP, 0); -OP2(SLJIT_SUB, SLJIT_R1, 0, STACK_TOP, 0, SLJIT_IMM, CALLOUT_ARG_SIZE); +/* SLJIT_R0 = arguments */ +OP1(SLJIT_MOV, SLJIT_R1, 0, STACK_TOP, 0); GET_LOCAL_BASE(SLJIT_R2, 0, OVECTOR_START); sljit_emit_ijump(compiler, SLJIT_CALL3, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_callout)); OP1(SLJIT_MOV_S32, SLJIT_RETURN_REG, 0, SLJIT_RETURN_REG, 0); @@ -7195,12 +7213,12 @@ OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); free_stack(common, CALLOUT_ARG_SIZE / sizeof(sljit_sw)); /* Check return value. */ -OP2(SLJIT_SUB | SLJIT_SET_S, SLJIT_UNUSED, 0, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0); +OP2(SLJIT_SUB | SLJIT_SET_Z | SLJIT_SET_SIG_GREATER, SLJIT_UNUSED, 0, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0); add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_SIG_GREATER)); if (common->forced_quit_label == NULL) - add_jump(compiler, &common->forced_quit, JUMP(SLJIT_SIG_LESS)); + add_jump(compiler, &common->forced_quit, JUMP(SLJIT_NOT_EQUAL) /* SIG_LESS */); else - JUMPTO(SLJIT_SIG_LESS, common->forced_quit_label); + JUMPTO(SLJIT_NOT_EQUAL /* SIG_LESS */, common->forced_quit_label); return cc + 2 + 2 * LINK_SIZE; } @@ -7321,7 +7339,7 @@ else allocate_stack(common, framesize + extrasize); OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + extrasize) * sizeof(sljit_sw)); + OP2(SLJIT_ADD, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + extrasize) * sizeof(sljit_sw)); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP2, 0); if (needs_control_head) OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); @@ -7392,22 +7410,22 @@ while (1) free_stack(common, extrasize); if (needs_control_head) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(-1)); } else { if ((opcode != OP_ASSERT_NOT && opcode != OP_ASSERTBACK_NOT) || conditional) { /* We don't need to keep the STR_PTR, only the previous private_data_ptr. */ - OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_sw)); if (needs_control_head) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(-1)); } else { OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); if (needs_control_head) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), (framesize + 1) * sizeof(sljit_sw)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(-framesize - 2)); add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); } } @@ -7418,25 +7436,25 @@ while (1) if (conditional) { if (extrasize > 0) - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), needs_control_head ? sizeof(sljit_sw) : 0); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), needs_control_head ? STACK(-2) : STACK(-1)); } else if (bra == OP_BRAZERO) { if (framesize < 0) - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (extrasize - 1) * sizeof(sljit_sw)); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(-extrasize)); else { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_sw)); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (framesize + extrasize - 1) * sizeof(sljit_sw)); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(-framesize - 1)); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(-framesize - extrasize)); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP1, 0); } - OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); } else if (framesize >= 0) { /* For OP_BRA and OP_BRAMINZERO. */ - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_sw)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-framesize - 1)); } } add_jump(compiler, found, JUMP(SLJIT_JUMP)); @@ -7480,12 +7498,12 @@ if (common->positive_assert_quit != NULL) set_jumps(common->positive_assert_quit, LABEL()); SLJIT_ASSERT(framesize != no_stack); if (framesize < 0) - OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, extrasize * sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, extrasize * sizeof(sljit_sw)); else { OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); - OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + extrasize) * sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + extrasize) * sizeof(sljit_sw)); } JUMPHERE(jump); } @@ -7534,18 +7552,18 @@ if (opcode == OP_ASSERT || opcode == OP_ASSERTBACK) { /* We know that STR_PTR was stored on the top of the stack. */ if (extrasize > 0) - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (extrasize - 1) * sizeof(sljit_sw)); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(-extrasize)); /* Keep the STR_PTR on the top of the stack. */ if (bra == OP_BRAZERO) { - OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); if (extrasize == 2) OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); } else if (bra == OP_BRAMINZERO) { - OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); } } @@ -7554,13 +7572,13 @@ if (opcode == OP_ASSERT || opcode == OP_ASSERTBACK) if (bra == OP_BRA) { /* We don't need to keep the STR_PTR, only the previous private_data_ptr. */ - OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_sw)); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (extrasize - 2) * sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_sw)); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(-extrasize + 1)); } else { /* We don't need to keep the STR_PTR, only the previous private_data_ptr. */ - OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 2) * sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 2) * sizeof(sljit_sw)); if (extrasize == 2) { OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); @@ -7588,7 +7606,7 @@ if (opcode == OP_ASSERT || opcode == OP_ASSERTBACK) { OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_sw)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-framesize - 1)); } set_jumps(backtrack->common.topbacktracks, LABEL()); } @@ -7675,23 +7693,23 @@ if (framesize < 0) } if (needs_control_head) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), (ket != OP_KET || has_alternatives) ? sizeof(sljit_sw) : 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), (ket != OP_KET || has_alternatives) ? STACK(-2) : STACK(-1)); /* TMP2 which is set here used by OP_KETRMAX below. */ if (ket == OP_KETRMAX) - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), 0); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(-1)); else if (ket == OP_KETRMIN) { /* Move the STR_PTR to the private_data_ptr. */ - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-1)); } } else { stacksize = (ket != OP_KET || has_alternatives) ? 2 : 1; - OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + stacksize) * sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + stacksize) * sizeof(sljit_sw)); if (needs_control_head) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(-1)); if (ket == OP_KETRMAX) { @@ -7927,7 +7945,7 @@ if (bra == OP_BRAMINZERO) { /* Except when the whole stack frame must be saved. */ OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - braminzero = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), (BACKTRACK_AS(bracket_backtrack)->u.framesize + 1) * sizeof(sljit_sw)); + braminzero = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), STACK(-BACKTRACK_AS(bracket_backtrack)->u.framesize - 2)); } JUMPHERE(skip); } @@ -8000,7 +8018,7 @@ if (opcode == OP_ONCE) OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0); if (BACKTRACK_AS(bracket_backtrack)->u.framesize == no_frame) - OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0, SLJIT_IMM, needs_control_head ? (2 * sizeof(sljit_sw)) : sizeof(sljit_sw)); + OP2(SLJIT_ADD, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0, SLJIT_IMM, needs_control_head ? (2 * sizeof(sljit_sw)) : sizeof(sljit_sw)); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize + 1), TMP2, 0); } else if (ket == OP_KETRMAX || has_alternatives) @@ -8018,7 +8036,7 @@ if (opcode == OP_ONCE) OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0); OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw)); + OP2(SLJIT_ADD, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw)); stacksize = needs_control_head ? 1 : 0; if (ket != OP_KET || has_alternatives) @@ -8090,13 +8108,13 @@ if (opcode == OP_COND || opcode == OP_SCOND) slot = common->name_table + GET2(matchingpath, 1) * common->name_entry_size; OP1(SLJIT_MOV, TMP3, 0, STR_PTR, 0); OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1)); - OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(GET2(slot, 0) << 1), TMP1, 0); + OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(GET2(slot, 0) << 1), TMP1, 0); slot += common->name_entry_size; i--; while (i-- > 0) { OP2(SLJIT_SUB, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(GET2(slot, 0) << 1), TMP1, 0); - OP2(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, STR_PTR, 0); + OP2(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, TMP2, 0, STR_PTR, 0); slot += common->name_entry_size; } OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0); @@ -8111,7 +8129,7 @@ if (opcode == OP_COND || opcode == OP_SCOND) if (*matchingpath == OP_FAIL) stacksize = 0; - if (*matchingpath == OP_RREF) + else if (*matchingpath == OP_RREF) { stacksize = GET2(matchingpath, 1); if (common->currententry == NULL) @@ -8244,7 +8262,7 @@ if (ket == OP_KETRMAX) { if (has_alternatives) BACKTRACK_AS(bracket_backtrack)->alternative_matchingpath = LABEL(); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, rmax_label); /* Drop STR_PTR for greedy plus quantifier. */ if (opcode != OP_ONCE) @@ -8274,7 +8292,7 @@ if (ket == OP_KETRMAX) if (repeat_type == OP_EXACT) { count_match(common); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, rmax_label); } else if (repeat_type == OP_UPTO) @@ -8374,7 +8392,7 @@ switch(opcode) break; default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); break; } @@ -8452,7 +8470,7 @@ else OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); if (needs_control_head) OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); - OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0, SLJIT_IMM, -STACK(stacksize - 1)); + OP2(SLJIT_ADD, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw)); stack = 0; if (!zero) @@ -8524,7 +8542,7 @@ while (*cc != OP_KETRPOS) { if (offset != 0) { - OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, stacksize * sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, stacksize * sizeof(sljit_sw)); OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), cbraprivptr); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), STR_PTR, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), cbraprivptr, STR_PTR, 0); @@ -8535,10 +8553,10 @@ while (*cc != OP_KETRPOS) else { OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - OP2(SLJIT_ADD, STACK_TOP, 0, TMP2, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, TMP2, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw)); if (opcode == OP_SBRAPOS) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_sw)); - OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_sw), STR_PTR, 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), STACK(-framesize - 2)); + OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), STACK(-framesize - 2), STR_PTR, 0); } /* Even if the match is empty, we need to reset the control head. */ @@ -8584,7 +8602,7 @@ while (*cc != OP_KETRPOS) else { OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_sw)); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(TMP2), STACK(-framesize - 2)); } } @@ -8601,7 +8619,7 @@ if (!zero) if (framesize < 0) add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 0)); else /* TMP2 is set to [private_data_ptr] above. */ - add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(TMP2), (stacksize - 1) * sizeof(sljit_sw), SLJIT_IMM, 0)); + add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(TMP2), STACK(-stacksize), SLJIT_IMM, 0)); } /* None of them matched. */ @@ -8824,7 +8842,7 @@ if (exact > 1) OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, exact); label = LABEL(); compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, FALSE); - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, label); } else @@ -8832,7 +8850,7 @@ if (exact > 1) OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, exact); label = LABEL(); compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, TRUE); - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, label); } } @@ -8862,7 +8880,7 @@ switch(opcode) if (opcode == OP_UPTO) { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0); - OP2(SLJIT_SUB | SLJIT_SET_E, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); jump = JUMP(SLJIT_ZERO); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0, TMP1, 0); } @@ -8924,7 +8942,7 @@ switch(opcode) label = LABEL(); if (opcode == OP_UPTO) { - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_ZERO)); } compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, FALSE); @@ -8944,7 +8962,7 @@ switch(opcode) OP1(SLJIT_MOV, base, offset1, STR_PTR, 0); if (opcode == OP_UPTO) { - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); add_jump(compiler, &no_match, JUMP(SLJIT_ZERO)); } @@ -8971,7 +8989,7 @@ switch(opcode) if (opcode == OP_UPTO) { - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, label); } else @@ -9000,7 +9018,7 @@ switch(opcode) if (opcode == OP_UPTO) { - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, label); } else @@ -9026,7 +9044,7 @@ switch(opcode) compile_char1_matchingpath(common, type, cc, &no_char1_match, FALSE); if (opcode == OP_UPTO) { - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, label); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); } @@ -9113,7 +9131,7 @@ switch(opcode) label = LABEL(); compile_char1_matchingpath(common, type, cc, &no_match, TRUE); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1, STR_PTR, 0); - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, label); set_jumps(no_match, LABEL()); OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1); @@ -9124,7 +9142,7 @@ switch(opcode) label = LABEL(); detect_partial_match(common, &no_match); compile_char1_matchingpath(common, type, cc, &no_char1_match, FALSE); - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, label); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); set_jumps(no_char1_match, LABEL()); @@ -9142,7 +9160,7 @@ switch(opcode) break; default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); break; } @@ -9264,7 +9282,7 @@ size = 3 + (size < 0 ? 0 : size); OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); allocate_stack(common, size); if (size > 3) - OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, STACK_TOP, 0, SLJIT_IMM, (size - 3) * sizeof(sljit_sw)); + OP2(SLJIT_ADD, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, STACK_TOP, 0, SLJIT_IMM, (size - 3) * sizeof(sljit_sw)); else OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, STACK_TOP, 0); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(size - 1), SLJIT_IMM, BACKTRACK_AS(then_trap_backtrack)->start); @@ -9569,7 +9587,7 @@ while (cc < ccend) break; default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); return; } if (cc == NULL) @@ -9677,7 +9695,7 @@ switch(opcode) case OP_MINUPTO: OP1(SLJIT_MOV, TMP1, 0, base, offset1); OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); - OP2(SLJIT_SUB | SLJIT_SET_E, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); add_jump(compiler, &jumplist, JUMP(SLJIT_ZERO)); OP1(SLJIT_MOV, base, offset1, TMP1, 0); @@ -9723,7 +9741,7 @@ switch(opcode) break; default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); break; } @@ -9831,7 +9849,7 @@ if (*cc == OP_ASSERT || *cc == OP_ASSERTBACK) { OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(assert_backtrack)->private_data_ptr); add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(assert_backtrack)->private_data_ptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(assert_backtrack)->framesize * sizeof(sljit_sw)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(assert_backtrack)->private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-CURRENT_AS(assert_backtrack)->framesize - 1)); set_jumps(current->topbacktracks, LABEL()); } @@ -9841,7 +9859,7 @@ else if (bra == OP_BRAZERO) { /* We know there is enough place on the stack. */ - OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); JUMPTO(SLJIT_JUMP, CURRENT_AS(assert_backtrack)->matchingpath); JUMPHERE(brajump); @@ -9954,7 +9972,7 @@ else if (ket == OP_KETRMIN) else { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), (CURRENT_AS(bracket_backtrack)->u.framesize + 1) * sizeof(sljit_sw), CURRENT_AS(bracket_backtrack)->recursive_matchingpath); + CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), STACK(-CURRENT_AS(bracket_backtrack)->u.framesize - 2), CURRENT_AS(bracket_backtrack)->recursive_matchingpath); } /* Drop STR_PTR for non-greedy plus quantifier. */ if (opcode != OP_ONCE) @@ -10060,7 +10078,7 @@ if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND)) { OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr); add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr, SLJIT_MEM1(STACK_TOP), assert->framesize * sizeof(sljit_sw)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-assert->framesize - 1)); } cond = JUMP(SLJIT_JUMP); set_jumps(CURRENT_AS(bracket_backtrack)->u.assert->condfailed, LABEL()); @@ -10201,7 +10219,7 @@ if (has_alternatives) { OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr); add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr, SLJIT_MEM1(STACK_TOP), assert->framesize * sizeof(sljit_sw)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-assert->framesize - 1)); } JUMPHERE(cond); } @@ -10256,7 +10274,7 @@ else if (opcode == OP_ONCE) JUMPHERE(once); /* Restore previous private_data_ptr */ if (CURRENT_AS(bracket_backtrack)->u.framesize >= 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(bracket_backtrack)->u.framesize * sizeof(sljit_sw)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-CURRENT_AS(bracket_backtrack)->u.framesize - 1)); else if (ket == OP_KETRMIN) { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); @@ -10346,7 +10364,7 @@ if (current->topbacktracks) free_stack(common, CURRENT_AS(bracketpos_backtrack)->stacksize); JUMPHERE(jump); } -OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(bracketpos_backtrack)->private_data_ptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(bracketpos_backtrack)->framesize * sizeof(sljit_sw)); +OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(bracketpos_backtrack)->private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-CURRENT_AS(bracketpos_backtrack)->framesize - 1)); } static SLJIT_INLINE void compile_braminzero_backtrackingpath(compiler_common *common, struct backtrack_common *current) @@ -10392,10 +10410,10 @@ if (opcode == OP_THEN || opcode == OP_THEN_ARG) jump = JUMP(SLJIT_JUMP); loop = LABEL(); - OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(STACK_TOP), -(int)sizeof(sljit_sw)); + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); JUMPHERE(jump); - CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), -(int)(2 * sizeof(sljit_sw)), TMP1, 0, loop); - CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), -(int)(3 * sizeof(sljit_sw)), TMP2, 0, loop); + CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0, loop); + CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(2), TMP2, 0, loop); add_jump(compiler, &common->then_trap->quit, JUMP(SLJIT_JUMP)); return; } @@ -10645,7 +10663,7 @@ while (current) break; default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); break; } current = current->prev; @@ -10684,7 +10702,7 @@ sljit_emit_fast_enter(compiler, TMP2, 0); count_match(common); allocate_stack(common, private_data_size + framesize + alternativesize); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(private_data_size + framesize + alternativesize - 1), TMP2, 0); -copy_private_data(common, ccbegin, ccend, TRUE, private_data_size + framesize + alternativesize, framesize + alternativesize, needs_control_head); +copy_private_data(common, ccbegin, ccend, TRUE, framesize + alternativesize, private_data_size + framesize + alternativesize, needs_control_head); if (needs_control_head) OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_IMM, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr, STACK_TOP, 0); @@ -10737,9 +10755,9 @@ if (common->quit != NULL) OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr); if (needs_frame) { - OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw)); - add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw)); + add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw)); } OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, 0); common->quit = NULL; @@ -10750,32 +10768,32 @@ set_jumps(common->accept, LABEL()); OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr); if (needs_frame) { - OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw)); - add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw)); + add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw)); } OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, 1); JUMPHERE(jump); if (common->quit != NULL) set_jumps(common->quit, LABEL()); -copy_private_data(common, ccbegin, ccend, FALSE, private_data_size + framesize + alternativesize, framesize + alternativesize, needs_control_head); +copy_private_data(common, ccbegin, ccend, FALSE, framesize + alternativesize, private_data_size + framesize + alternativesize, needs_control_head); free_stack(common, private_data_size + framesize + alternativesize); if (needs_control_head) { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), 2 * sizeof(sljit_sw)); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), sizeof(sljit_sw)); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(-3)); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(-2)); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr, TMP1, 0); OP1(SLJIT_MOV, TMP1, 0, TMP3, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, TMP2, 0); } else { - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), sizeof(sljit_sw)); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(-2)); OP1(SLJIT_MOV, TMP1, 0, TMP3, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr, TMP2, 0); } -sljit_emit_fast_return(compiler, SLJIT_MEM1(STACK_TOP), 0); +sljit_emit_fast_return(compiler, SLJIT_MEM1(STACK_TOP), STACK(-1)); } #undef COMPILE_BACKTRACKINGPATH @@ -11237,7 +11255,7 @@ OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP2, 0); OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack)); OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, top), STACK_TOP, 0); -OP2(SLJIT_ADD, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, limit), SLJIT_IMM, STACK_GROWTH_RATE); +OP2(SLJIT_SUB, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, limit), SLJIT_IMM, STACK_GROWTH_RATE); sljit_emit_ijump(compiler, SLJIT_CALL2, SLJIT_IMM, SLJIT_FUNC_OFFSET(sljit_stack_resize)); jump = CMP(SLJIT_NOT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0); @@ -11391,10 +11409,10 @@ union { sljit_u8 local_space[MACHINE_STACK_SIZE]; struct sljit_stack local_stack; -local_stack.top = (sljit_sw)&local_space; -local_stack.base = local_stack.top; -local_stack.limit = local_stack.base + MACHINE_STACK_SIZE; -local_stack.max_limit = local_stack.limit; +local_stack.max_limit = local_space; +local_stack.limit = local_space; +local_stack.base = local_space + MACHINE_STACK_SIZE; +local_stack.top = local_space + MACHINE_STACK_SIZE; arguments->stack = &local_stack; convert_executable_func.executable_func = executable_func; return convert_executable_func.call_executable_func(arguments); diff --git a/erts/emulator/pcre/pcre_tables.c b/erts/emulator/pcre/pcre_tables.c index 2f6302e2e1..08e31f1460 100644 --- a/erts/emulator/pcre/pcre_tables.c +++ b/erts/emulator/pcre/pcre_tables.c @@ -6,7 +6,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge + Copyright (c) 1997-2017 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -162,7 +162,7 @@ const pcre_uint32 PRIV(ucp_gbtable[]) = { (1<<ucp_gbExtend)|(1<<ucp_gbSpacingMark), /* 5 SpacingMark */ (1<<ucp_gbExtend)|(1<<ucp_gbSpacingMark)|(1<<ucp_gbL)| /* 6 L */ - (1<<ucp_gbL)|(1<<ucp_gbV)|(1<<ucp_gbLV)|(1<<ucp_gbLVT), + (1<<ucp_gbV)|(1<<ucp_gbLV)|(1<<ucp_gbLVT), (1<<ucp_gbExtend)|(1<<ucp_gbSpacingMark)|(1<<ucp_gbV)| /* 7 V */ (1<<ucp_gbT), diff --git a/erts/emulator/pcre/pcre_ucd.c b/erts/emulator/pcre/pcre_ucd.c index 9b700c0785..2dd4b05751 100644 --- a/erts/emulator/pcre/pcre_ucd.c +++ b/erts/emulator/pcre/pcre_ucd.c @@ -38,6 +38,20 @@ const pcre_uint16 PRIV(ucd_stage2)[] = {0}; const pcre_uint32 PRIV(ucd_caseless_sets)[] = {0}; #else +/* If the 32-bit library is run in non-32-bit mode, character values +greater than 0x10ffff may be encountered. For these we set up a +special record. */ + +#ifdef COMPILE_PCRE32 +const ucd_record PRIV(dummy_ucd_record)[] = {{ + ucp_Common, /* script */ + ucp_Cn, /* type unassigned */ + ucp_gbOther, /* grapheme break property */ + 0, /* case set */ + 0, /* other case */ + }}; +#endif + /* When recompiling tables with a new Unicode version, please check the types in this structure definition from pcre_internal.h (the actual field names will be different): diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index 834b77eb58..f93d4c1557 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -29,7 +29,6 @@ #endif #define ERL_CHECK_IO_C__ -#define ERTS_WANT_BREAK_HANDLING #ifndef WANT_NONBLOCKING # define WANT_NONBLOCKING #endif @@ -44,71 +43,79 @@ #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" -#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS -# include "safe_hash.h" -# define DRV_EV_STATE_HTAB_SIZE 1024 +#if 0 +#define DEBUG_PRINT(FMT, ...) erts_printf(FMT "\r\n", ##__VA_ARGS__) +#define DEBUG_PRINT_FD(FMT, STATE, ...) \ + DEBUG_PRINT("%d: " FMT " (ev=%s, ac=%s, flg=%d)", \ + (STATE) ? (STATE)->fd : (ErtsSysFdType)-1, ##__VA_ARGS__, \ + ev2str((STATE) ? (STATE)->events : ERTS_POLL_EV_NONE), \ + ev2str((STATE) ? (STATE)->active_events : ERTS_POLL_EV_NONE), \ + (STATE) ? (STATE)->flags : ERTS_EV_FLAG_CLEAR) +#define DEBUG_PRINT_MODE +#else +#define DEBUG_PRINT(...) #endif -typedef char EventStateType; -#define ERTS_EV_TYPE_NONE ((EventStateType) 0) -#define ERTS_EV_TYPE_DRV_SEL ((EventStateType) 1) /* driver_select */ -#define ERTS_EV_TYPE_DRV_EV ((EventStateType) 2) /* driver_event */ -#define ERTS_EV_TYPE_STOP_USE ((EventStateType) 3) /* pending stop_select */ -#define ERTS_EV_TYPE_NIF ((EventStateType) 4) /* enif_select */ -#define ERTS_EV_TYPE_STOP_NIF ((EventStateType) 5) /* pending nif stop */ - -typedef char EventStateFlags; -#define ERTS_EV_FLAG_USED ((EventStateFlags) 1) /* ERL_DRV_USE has been turned on */ -#define ERTS_EV_FLAG_DEFER_IN_EV ((EventStateFlags) 2) -#define ERTS_EV_FLAG_DEFER_OUT_EV ((EventStateFlags) 4) - -#ifdef DEBUG -# define ERTS_ACTIVE_FD_INC 2 -#else -# define ERTS_ACTIVE_FD_INC 128 +#ifndef DEBUG_PRINT_FD +#define DEBUG_PRINT_FD(...) #endif -#define ERTS_CHECK_IO_POLL_RES_LEN 512 +#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS +# include "safe_hash.h" +# define DRV_EV_STATE_HTAB_SIZE 1024 +#endif -#if defined(ERTS_KERNEL_POLL_VERSION) -# define ERTS_CIO_EXPORT(FUNC) FUNC ## _kp -#elif defined(ERTS_NO_KERNEL_POLL_VERSION) -# define ERTS_CIO_EXPORT(FUNC) FUNC ## _nkp +typedef enum { + ERTS_EV_TYPE_NONE = 0, + ERTS_EV_TYPE_DRV_SEL = 1, /* driver_select */ + ERTS_EV_TYPE_STOP_USE = 2, /* pending stop_select */ + ERTS_EV_TYPE_NIF = 3, /* enif_select */ + ERTS_EV_TYPE_STOP_NIF = 4 /* pending nif stop */ +} EventStateType; + +typedef enum { + ERTS_EV_FLAG_CLEAR = 0, + ERTS_EV_FLAG_USED = 1, /* ERL_DRV_USE has been turned on */ +#ifdef ERTS_ENABLE_KERNEL_POLL + ERTS_EV_FLAG_FALLBACK = 2, /* Set when kernel poll rejected fd + and it was put in the nkp version */ #else -# define ERTS_CIO_EXPORT(FUNC) FUNC + ERTS_EV_FLAG_FALLBACK = ERTS_EV_FLAG_CLEAR, #endif -#define ERTS_CIO_HAVE_DRV_EVENT \ - (ERTS_POLL_USE_POLL && !ERTS_POLL_USE_KERNEL_POLL) + /* Combinations */ + ERTS_EV_FLAG_USED_FALLBACK = ERTS_EV_FLAG_USED | ERTS_EV_FLAG_FALLBACK +} EventStateFlags; -#define ERTS_CIO_POLL_CTL ERTS_POLL_EXPORT(erts_poll_control) -#define ERTS_CIO_POLL_CTLV ERTS_POLL_EXPORT(erts_poll_controlv) -#define ERTS_CIO_POLL_WAIT ERTS_POLL_EXPORT(erts_poll_wait) -#define ERTS_CIO_POLL_INTR ERTS_POLL_EXPORT(erts_poll_interrupt) -#define ERTS_CIO_POLL_INTR_TMD ERTS_POLL_EXPORT(erts_poll_interrupt_timed) -#define ERTS_CIO_NEW_POLLSET ERTS_POLL_EXPORT(erts_poll_create_pollset) -#define ERTS_CIO_FREE_POLLSET ERTS_POLL_EXPORT(erts_poll_destroy_pollset) -#define ERTS_CIO_POLL_MAX_FDS ERTS_POLL_EXPORT(erts_poll_max_fds) -#define ERTS_CIO_POLL_INIT ERTS_POLL_EXPORT(erts_poll_init) -#define ERTS_CIO_POLL_INFO ERTS_POLL_EXPORT(erts_poll_info) +#define flag2str(flags) \ + ((flags) == ERTS_EV_FLAG_CLEAR ? "CLEAR" : \ + ((flags) == ERTS_EV_FLAG_USED ? "USED" : \ + ((flags) == ERTS_EV_FLAG_FALLBACK ? "FLBK" : \ + ((flags) == ERTS_EV_FLAG_USED_FALLBACK ? "USED|FLBK" : "ERROR")))) -#define GET_FD(fd) fd +/* How many events that can be handled at once by one erts_poll_wait call */ +#define ERTS_CHECK_IO_POLL_RES_LEN 512 -static struct pollset_info +/* Each I/O Poll Thread has one ErtsPollThread each. The ps field + can point to either a private ErtsPollSet or a shared one. + At the moment only kqueue and epoll pollsets can be + shared across threads. +*/ +typedef struct erts_poll_thread { - ErtsPollSet ps; - erts_atomic_t in_poll_wait; /* set while doing poll */ - struct { - int six; /* start index */ - int eix; /* end index */ - erts_atomic32_t no; - int size; - ErtsSysFdType *array; - } active_fd; - struct removed_fd* removed_list; /* list of deselected fd's*/ - erts_spinlock_t removed_list_lock; -}pollset; -#define NUM_OF_POLLSETS 1 + ErtsPollSet *ps; + ErtsPollResFd *pollres; + int pollres_len; +} ErtsPollThread; + +/* pollsetv contains pointers to the ErtsPollSets that are in use. + * Which pollset to use is determined by hashing the fd. + */ +static ErtsPollSet **pollsetv; +#if ERTS_POLL_USE_FALLBACK +static ErtsPollSet *flbk_pollset; +#endif +static ErtsPollThread *psiv; typedef struct { #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS @@ -117,92 +124,151 @@ typedef struct { ErtsSysFdType fd; struct { ErtsDrvSelectDataState *select; /* ERTS_EV_TYPE_DRV_SEL */ -#if ERTS_CIO_HAVE_DRV_EVENT - ErtsDrvEventDataState *event; /* ERTS_EV_TYPE_DRV_EV */ -#endif ErtsNifSelectDataState *nif; /* ERTS_EV_TYPE_NIF */ union { erts_driver_t* drv_ptr; /* ERTS_EV_TYPE_STOP_USE */ ErtsResource* resource; /* ERTS_EV_TYPE_STOP_NIF */ - }stop; + } stop; } driver; - ErtsPollEvents events; - unsigned short remove_cnt; /* number of removed_fd's referring to this fd */ + ErtsPollEvents events; /* The events that have been selected upon */ + ErtsPollEvents active_events; /* The events currently active in the pollset */ EventStateType type; EventStateFlags flags; } ErtsDrvEventState; -struct removed_fd { - struct removed_fd *next; +struct drv_ev_state_shared { + + union { + erts_mtx_t lck; + byte _cache_line_alignment[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_mtx_t))]; + } locks[ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT]; + #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - ErtsSysFdType fd; + int max_fds; + erts_atomic_t len; + ErtsDrvEventState *v; + erts_mtx_t grow_lock; /* prevent lock-hogging of racing growers */ #else - ErtsDrvEventState* state; - #ifdef DEBUG - ErtsSysFdType fd; - #endif + SafeHash tab; + int num_prealloc; + ErtsDrvEventState *prealloc_first; + erts_spinlock_t prealloc_lock; #endif - }; -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS -static int max_fds = -1; -#endif -#define DRV_EV_STATE_LOCK_CNT 16 -static union { - erts_mtx_t lck; - byte _cache_line_alignment[64]; -}drv_ev_state_locks[DRV_EV_STATE_LOCK_CNT]; +int ERTS_WRITE_UNLIKELY(erts_no_pollsets) = 1; +int ERTS_WRITE_UNLIKELY(erts_no_poll_threads) = 1; +struct drv_ev_state_shared drv_ev_state; -static ERTS_INLINE erts_mtx_t* fd_mtx(ErtsSysFdType fd) -{ +static ERTS_INLINE int fd_hash(ErtsSysFdType fd) { int hash = (int)fd; # ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS hash ^= (hash >> 9); # endif - return &drv_ev_state_locks[hash % DRV_EV_STATE_LOCK_CNT].lck; + return hash; +} + +static ERTS_INLINE erts_mtx_t* fd_mtx(ErtsSysFdType fd) +{ + return &drv_ev_state.locks[fd_hash(fd) % ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT].lck; } #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS -static erts_atomic_t drv_ev_state_len; -static ErtsDrvEventState *drv_ev_state; -static erts_mtx_t drv_ev_state_grow_lock; /* prevent lock-hogging of racing growers */ +static ERTS_INLINE ErtsDrvEventState *get_drv_ev_state(ErtsSysFdType fd) +{ + return &drv_ev_state.v[(int) fd]; +} -#else -static SafeHash drv_ev_state_tab; -static int num_state_prealloc; -static ErtsDrvEventState *state_prealloc_first; -erts_spinlock_t state_prealloc_lock; +#define new_drv_ev_state(State, fd) (State) +#define erase_drv_ev_state(State) + +static ERTS_INLINE int grow_drv_ev_state(ErtsSysFdType fd) { + int i; + int old_len; + int new_len; + + if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state.len)) { + + if (fd < 0 || fd >= drv_ev_state.max_fds) + return 0; + + erts_mtx_lock(&drv_ev_state.grow_lock); + old_len = erts_atomic_read_nob(&drv_ev_state.len); + if (fd >= old_len) { + new_len = erts_poll_new_table_len(old_len, fd + 1); + if (new_len > drv_ev_state.max_fds) + new_len = drv_ev_state.max_fds; + + for (i=0; i<ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT; i++) { /* lock all fd's */ + erts_mtx_lock(&drv_ev_state.locks[i].lck); + } + drv_ev_state.v = (drv_ev_state.v + ? erts_realloc(ERTS_ALC_T_DRV_EV_STATE, + drv_ev_state.v, + sizeof(ErtsDrvEventState)*new_len) + : erts_alloc(ERTS_ALC_T_DRV_EV_STATE, + sizeof(ErtsDrvEventState)*new_len)); + ERTS_CT_ASSERT(ERTS_EV_TYPE_NONE == 0); + sys_memzero(drv_ev_state.v+old_len, + sizeof(ErtsDrvEventState) * (new_len - old_len)); + for (i = old_len; i < new_len; i++) { + drv_ev_state.v[i].fd = (ErtsSysFdType) i; + } + erts_atomic_set_nob(&drv_ev_state.len, new_len); + for (i=0; i<ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT; i++) { + erts_mtx_unlock(&drv_ev_state.locks[i].lck); + } + } + /*else already grown by racing thread */ + + erts_mtx_unlock(&drv_ev_state.grow_lock); + } + return 1; +} -static ERTS_INLINE ErtsDrvEventState *hash_get_drv_ev_state(ErtsSysFdType fd) +static int drv_ev_state_len(void) +{ + return erts_atomic_read_nob(&drv_ev_state.len); +} + +#else /* !ERTS_SYS_CONTINOUS_FD_NUMBERS */ + +static ERTS_INLINE ErtsDrvEventState *get_drv_ev_state(ErtsSysFdType fd) { ErtsDrvEventState tmpl; tmpl.fd = fd; - return (ErtsDrvEventState *) safe_hash_get(&drv_ev_state_tab, (void *) &tmpl); + return (ErtsDrvEventState *) safe_hash_get(&drv_ev_state.tab, (void *) &tmpl); } -static ERTS_INLINE ErtsDrvEventState* hash_new_drv_ev_state(ErtsSysFdType fd) +static ERTS_INLINE ErtsDrvEventState* new_drv_ev_state(ErtsDrvEventState *state, + ErtsSysFdType fd) { ErtsDrvEventState tmpl; + + if (state) + return state; + tmpl.fd = fd; tmpl.driver.select = NULL; -#if ERTS_CIO_HAVE_DRV_EVENT - tmpl.driver.event = NULL; -#endif tmpl.driver.nif = NULL; tmpl.driver.stop.drv_ptr = NULL; tmpl.events = 0; - tmpl.remove_cnt = 0; + tmpl.active_events = 0; tmpl.type = ERTS_EV_TYPE_NONE; tmpl.flags = 0; - return (ErtsDrvEventState *) safe_hash_put(&drv_ev_state_tab, (void *) &tmpl); + + return (ErtsDrvEventState *) safe_hash_put(&drv_ev_state.tab, (void *) &tmpl); +} + +static ERTS_INLINE void erase_drv_ev_state(ErtsDrvEventState *state) +{ + safe_hash_erase(&drv_ev_state.tab, (void *) state); } -static ERTS_INLINE void hash_erase_drv_ev_state(ErtsDrvEventState *state) +static int drv_ev_state_len(void) { - ASSERT(state->remove_cnt == 0); - safe_hash_erase(&drv_ev_state_tab, (void *) state); + return erts_atomic_read_nob(&drv_ev_state.tab.nitems); } #endif /* !ERTS_SYS_CONTINOUS_FD_NUMBERS */ @@ -222,50 +288,47 @@ static void print_nif_select_op(erts_dsprintf_buf_t*, ErtsSysFdType, static void drv_select_large_fd_error(ErlDrvPort, ErtsSysFdType, int, int); static void nif_select_large_fd_error(ErtsSysFdType, int, ErtsResource*,Eterm ref); #endif -#if ERTS_CIO_HAVE_DRV_EVENT -static void drv_event_steal(ErlDrvPort ix, ErtsDrvEventState *state, - ErlDrvEventData event_data); -static void print_drv_event_op(erts_dsprintf_buf_t *dsbufp, - ErlDrvPort, ErtsSysFdType, ErlDrvEventData); -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS -static void event_large_fd_error(ErlDrvPort, ErtsSysFdType, ErlDrvEventData); -#endif -#endif static void steal_pending_stop_use(erts_dsprintf_buf_t*, ErlDrvPort, ErtsDrvEventState*, int mode, int on); static void steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErtsResource*, ErtsDrvEventState *state, int mode, int on); - -ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(removed_fd, struct removed_fd, 64, ERTS_ALC_T_FD_LIST) +static ERTS_INLINE void +check_fd_cleanup(ErtsDrvEventState *state, + ErtsDrvSelectDataState **free_select, + ErtsNifSelectDataState **free_nif); +static ERTS_INLINE void iready(Eterm id, ErtsDrvEventState *state); +static ERTS_INLINE void oready(Eterm id, ErtsDrvEventState *state); +#ifdef DEBUG_PRINT_MODE +static char *drvmode2str(int mode); +static char *nifmode2str(enum ErlNifSelectFlags mode); +#endif static ERTS_INLINE void -init_iotask(ErtsIoTask *io_task) +init_iotask(ErtsIoTask *io_task, ErtsSysFdType fd) { erts_port_task_handle_init(&io_task->task); - erts_atomic_init_nob(&io_task->executed_time, ~((erts_aint_t) 0)); + io_task->fd = fd; } static ERTS_INLINE int -is_iotask_active(ErtsIoTask *io_task, erts_aint_t current_cio_time) -{ +is_iotask_active(ErtsIoTask *io_task) +{ if (erts_port_task_is_scheduled(&io_task->task)) return 1; - if (erts_atomic_read_nob(&io_task->executed_time) == current_cio_time) - return 1; return 0; } static ERTS_INLINE ErtsDrvSelectDataState * -alloc_drv_select_data(void) +alloc_drv_select_data(ErtsSysFdType fd) { ErtsDrvSelectDataState *dsp = erts_alloc(ERTS_ALC_T_DRV_SEL_D_STATE, sizeof(ErtsDrvSelectDataState)); dsp->inport = NIL; dsp->outport = NIL; - init_iotask(&dsp->iniotask); - init_iotask(&dsp->outiotask); + init_iotask(&dsp->iniotask, fd); + init_iotask(&dsp->outiotask, fd); return dsp; } @@ -276,8 +339,6 @@ alloc_nif_select_data(void) sizeof(ErtsNifSelectDataState)); dsp->in.pid = NIL; dsp->out.pid = NIL; - dsp->in.ddeselect_cnt = 0; - dsp->out.ddeselect_cnt = 0; return dsp; } @@ -295,201 +356,136 @@ free_nif_select_data(ErtsNifSelectDataState *dsp) erts_free(ERTS_ALC_T_NIF_SEL_D_STATE, dsp); } -#if ERTS_CIO_HAVE_DRV_EVENT - -static ERTS_INLINE ErtsDrvEventDataState * -alloc_drv_event_data(void) +static ERTS_INLINE int +get_pollset_id(ErtsSysFdType fd) { - ErtsDrvEventDataState *dep = erts_alloc(ERTS_ALC_T_DRV_EV_D_STATE, - sizeof(ErtsDrvEventDataState)); - dep->port = NIL; - dep->data = NULL; - dep->removed_events = 0; -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - dep->deferred_events = 0; -#endif - init_iotask(&dep->iotask); - return dep; + return fd_hash(fd) % erts_no_pollsets; } -static ERTS_INLINE void -free_drv_event_data(ErtsDrvEventDataState *dep) +static ERTS_INLINE ErtsPollSet * +get_pollset(ErtsSysFdType fd) { - ASSERT(!erts_port_task_is_scheduled(&dep->iotask.task)); - erts_free(ERTS_ALC_T_DRV_EV_D_STATE, dep); + return pollsetv[get_pollset_id(fd)]; } -#endif /* ERTS_CIO_HAVE_DRV_EVENT */ +#if ERTS_POLL_USE_FALLBACK +static ERTS_INLINE ErtsPollSet * +get_fallback(void) +{ + return flbk_pollset; +} +#endif -static ERTS_INLINE void -remember_removed(ErtsDrvEventState *state, struct pollset_info* psi) +/* + * Place a fd within a pollset. This will automatically use + * the fallback ps if needed. + */ +static ERTS_INLINE ErtsPollEvents +erts_io_control_wakeup(ErtsDrvEventState *state, ErtsPollOp op, + ErtsPollEvents pe, int *wake_poller) { - struct removed_fd *fdlp; + ErtsSysFdType fd = state->fd; + ErtsPollEvents res = 0; + EventStateFlags flags = state->flags; + ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd))); - if (erts_atomic_read_nob(&psi->in_poll_wait)) { - state->remove_cnt++; - ASSERT(state->remove_cnt > 0); - fdlp = removed_fd_alloc(); - #if defined(ERTS_SYS_CONTINOUS_FD_NUMBERS) || defined(DEBUG) - fdlp->fd = state->fd; - #endif - #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS - fdlp->state = state; - #endif - erts_spin_lock(&psi->removed_list_lock); - fdlp->next = psi->removed_list; - psi->removed_list = fdlp; - erts_spin_unlock(&psi->removed_list_lock); + + if (!(flags & ERTS_EV_FLAG_FALLBACK)) { + res = erts_poll_control(get_pollset(fd), fd, op, pe, wake_poller); + +#if ERTS_POLL_USE_FALLBACK + if (op == ERTS_POLL_OP_ADD && res == ERTS_POLL_EV_NVAL) { + /* When an add fails with NVAL, the poll/kevent operation could not + put that fd in the pollset, so we instead put it into a fallback pollset */ + state->flags |= ERTS_EV_FLAG_FALLBACK; + res = erts_poll_control_flbk(get_fallback(), fd, op, pe, wake_poller); + } + } else { + ASSERT(op != ERTS_POLL_OP_ADD); + res = erts_poll_control_flbk(get_fallback(), fd, op, pe, wake_poller); +#endif } -} + return res; +} -static ERTS_INLINE int -is_removed(ErtsDrvEventState *state) +static ERTS_INLINE ErtsPollEvents +erts_io_control(ErtsDrvEventState *state, ErtsPollOp op, ErtsPollEvents pe) { - /* Note that there is a possible race here, where an fd is removed - (increasing remove_cnt) and then added again just before erts_poll_wait - is called by erts_check_io. Any polled event on the re-added fd will then - be falsely ignored. But that does not matter, as the event will trigger - again next time erl_check_io is called. */ - return state->remove_cnt > 0; + int wake_poller = 0; + return erts_io_control_wakeup(state, op, pe, &wake_poller); } -static void -forget_removed(struct pollset_info* psi) +/* ToDo: Was inline in erl_check_io.h but now need struct erts_poll_thread */ +void +erts_io_notify_port_task_executed(ErtsPortTaskType type, + ErtsPortTaskHandle *pthp, + void (*reset_handle)(ErtsPortTaskHandle *)) { - struct removed_fd* fdlp; - struct removed_fd* tofree; + ErtsIoTask *itp = ErtsContainerStruct(pthp, ErtsIoTask, task); + ErtsSysFdType fd = itp->fd; + erts_mtx_t *mtx = fd_mtx(fd); + int active_events; + ErtsDrvEventState *state; + ErtsDrvSelectDataState *free_select = NULL; + ErtsNifSelectDataState *free_nif = NULL; - /* Fast track: if (atomic_ptr(removed_list)==NULL) return; */ + erts_mtx_lock(mtx); + state = get_drv_ev_state(fd); - erts_spin_lock(&psi->removed_list_lock); - fdlp = psi->removed_list; - psi->removed_list = NULL; - erts_spin_unlock(&psi->removed_list_lock); + active_events = state->active_events; - while (fdlp) { - ErtsResource* resource = NULL; - erts_driver_t* drv_ptr = NULL; - erts_mtx_t* mtx; - ErtsSysFdType fd; - ErtsDrvEventState *state; + switch (type) { + case ERTS_PORT_TASK_INPUT: -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - fd = fdlp->fd; - mtx = fd_mtx(fd); - erts_mtx_lock(mtx); - state = &drv_ev_state[(int) fd]; -#else - state = fdlp->state; - fd = state->fd; - ASSERT(fd == fdlp->fd); - mtx = fd_mtx(fd); - erts_mtx_lock(mtx); -#endif - ASSERT(state->remove_cnt > 0); - if (--state->remove_cnt == 0) { - switch (state->type) { - case ERTS_EV_TYPE_STOP_NIF: - /* Now we can call stop */ - resource = state->driver.stop.resource; - state->driver.stop.resource = NULL; - ASSERT(resource); - state->type = ERTS_EV_TYPE_NONE; - state->flags &= ~ERTS_EV_FLAG_USED; - goto case_ERTS_EV_TYPE_NONE; - - case ERTS_EV_TYPE_STOP_USE: - /* Now we can call stop_select */ - drv_ptr = state->driver.stop.drv_ptr; - ASSERT(drv_ptr); - state->type = ERTS_EV_TYPE_NONE; - state->flags &= ~ERTS_EV_FLAG_USED; - state->driver.stop.drv_ptr = NULL; - /* Fall through */ - case ERTS_EV_TYPE_NONE: - case_ERTS_EV_TYPE_NONE: -#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS - hash_erase_drv_ev_state(state); -#endif - break; - case ERTS_EV_TYPE_DRV_SEL: - case ERTS_EV_TYPE_DRV_EV: - break; - default: - ASSERT(0); - } - } - erts_mtx_unlock(mtx); - if (drv_ptr) { - int was_unmasked = erts_block_fpe(); - DTRACE1(driver_stop_select, drv_ptr->name); - LTTNG1(driver_stop_select, drv_ptr->name); - (*drv_ptr->stop_select) ((ErlDrvEvent) fd, NULL); - erts_unblock_fpe(was_unmasked); - if (drv_ptr->handle) { - erts_ddll_dereference_driver(drv_ptr->handle); - } - } - if (resource) { - erts_resource_stop(resource, (ErlNifEvent)fd, 0); - enif_release_resource(resource->data); - } + DEBUG_PRINT_FD("executed ready_input", state); + + ASSERT(!(state->active_events & ERTS_POLL_EV_IN)); + if (state->events & ERTS_POLL_EV_IN) + active_events |= ERTS_POLL_EV_IN; + break; + case ERTS_PORT_TASK_OUTPUT: - tofree = fdlp; - fdlp = fdlp->next; - removed_fd_free(tofree); + DEBUG_PRINT_FD("executed ready_output", state); + + ASSERT(!(state->active_events & ERTS_POLL_EV_OUT)); + if (state->events & ERTS_POLL_EV_OUT) + active_events |= ERTS_POLL_EV_OUT; + break; + default: + erts_exit(ERTS_ABORT_EXIT, "Invalid IO port task type"); + break; } -} -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS -static void -grow_drv_ev_state(int min_ix) -{ - int i; - int old_len; - int new_len; + reset_handle(pthp); - erts_mtx_lock(&drv_ev_state_grow_lock); - old_len = erts_atomic_read_nob(&drv_ev_state_len); - if (min_ix >= old_len) { - new_len = erts_poll_new_table_len(old_len, min_ix + 1); - if (new_len > max_fds) - new_len = max_fds; + if (active_events) { + /* This is not needed if active_events has not changed */ + if (state->active_events != active_events) { + ErtsPollEvents new_events; + state->active_events = active_events; + new_events = erts_io_control(state, ERTS_POLL_OP_MOD, active_events); - for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { /* lock all fd's */ - erts_mtx_lock(&drv_ev_state_locks[i].lck); - } - drv_ev_state = (drv_ev_state - ? erts_realloc(ERTS_ALC_T_DRV_EV_STATE, - drv_ev_state, - sizeof(ErtsDrvEventState)*new_len) - : erts_alloc(ERTS_ALC_T_DRV_EV_STATE, - sizeof(ErtsDrvEventState)*new_len)); - for (i = old_len; i < new_len; i++) { - drv_ev_state[i].fd = (ErtsSysFdType) i; - drv_ev_state[i].driver.select = NULL; -#if ERTS_CIO_HAVE_DRV_EVENT - drv_ev_state[i].driver.event = NULL; -#endif - drv_ev_state[i].driver.stop.drv_ptr = NULL; - drv_ev_state[i].driver.nif = NULL; - drv_ev_state[i].events = 0; - drv_ev_state[i].remove_cnt = 0; - drv_ev_state[i].type = ERTS_EV_TYPE_NONE; - drv_ev_state[i].flags = 0; - } - erts_atomic_set_nob(&drv_ev_state_len, new_len); - for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { - erts_mtx_unlock(&drv_ev_state_locks[i].lck); - } + /* We were unable to re-insert the fd into the pollset, signal the callback. */ + if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) { + if (active_events & ERTS_POLL_EV_IN) + iready(state->driver.select->inport, state); + if (active_events & ERTS_POLL_EV_OUT) + oready(state->driver.select->outport, state); + state->active_events = 0; + } + } + } else { + check_fd_cleanup(state, &free_select, &free_nif); } - /*else already grown by racing thread */ - erts_mtx_unlock(&drv_ev_state_grow_lock); -} -#endif /* ERTS_SYS_CONTINOUS_FD_NUMBERS */ + erts_mtx_unlock(mtx); + if (free_select) + free_drv_select_data(free_select); + if (free_nif) + free_nif_select_data(free_nif); +} static ERTS_INLINE void abort_task(Eterm id, ErtsPortTaskHandle *pthp, EventStateType type) @@ -506,13 +502,6 @@ abort_tasks(ErtsDrvEventState *state, int mode) switch (mode) { case 0: check_type: switch (state->type) { -#if ERTS_CIO_HAVE_DRV_EVENT - case ERTS_EV_TYPE_DRV_EV: - abort_task(state->driver.event->port, - &state->driver.event->iotask.task, - ERTS_EV_TYPE_DRV_EV); - return; -#endif case ERTS_EV_TYPE_NIF: case ERTS_EV_TYPE_NONE: return; @@ -542,16 +531,14 @@ abort_tasks(ErtsDrvEventState *state, int mode) static void deselect(ErtsDrvEventState *state, int mode) { - int do_wake = 0; ErtsPollEvents rm_events; ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd))); - ASSERT(state->events); abort_tasks(state, mode); - if (!mode) + if (!mode) { rm_events = state->events; - else { + } else { rm_events = 0; ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL); if (mode & ERL_DRV_READ) { @@ -564,67 +551,63 @@ deselect(ErtsDrvEventState *state, int mode) } } - state->events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, rm_events, 0, &do_wake); + state->events &= ~rm_events; + state->active_events &= ~rm_events; if (!(state->events)) { + erts_io_control(state, ERTS_POLL_OP_DEL, 0); switch (state->type) { case ERTS_EV_TYPE_NIF: state->driver.nif->in.pid = NIL; state->driver.nif->out.pid = NIL; - state->driver.nif->in.ddeselect_cnt = 0; - state->driver.nif->out.ddeselect_cnt = 0; - enif_release_resource(state->driver.stop.resource); + enif_release_resource(state->driver.stop.resource->data); state->driver.stop.resource = NULL; break; case ERTS_EV_TYPE_DRV_SEL: state->driver.select->inport = NIL; state->driver.select->outport = NIL; break; -#if ERTS_CIO_HAVE_DRV_EVENT - case ERTS_EV_TYPE_DRV_EV: - state->driver.event->port = NIL; - state->driver.event->data = NULL; - state->driver.event->removed_events = (ErtsPollEvents) 0; - break; -#endif case ERTS_EV_TYPE_NONE: break; default: ASSERT(0); break; } - state->type = ERTS_EV_TYPE_NONE; - state->flags &= ~ERTS_EV_FLAG_USED; - remember_removed(state, &pollset); + state->flags = 0; + } else { + ErtsPollEvents new_events = + erts_io_control(state, ERTS_POLL_OP_MOD, state->active_events); + + /* We were unable to re-insert the fd into the pollset, signal the callback. */ + if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) { + if (state->active_events & ERTS_POLL_EV_IN) + iready(state->driver.select->inport, state); + if (state->active_events & ERTS_POLL_EV_OUT) + oready(state->driver.select->outport, state); + state->active_events = 0; + } } } #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS -# define IS_FD_UNKNOWN(state) ((state)->type == ERTS_EV_TYPE_NONE && (state)->remove_cnt == 0) +# define IS_FD_UNKNOWN(state) ((state)->type == ERTS_EV_TYPE_NONE) #else # define IS_FD_UNKNOWN(state) ((state) == NULL) #endif static ERTS_INLINE void check_fd_cleanup(ErtsDrvEventState *state, -#if ERTS_CIO_HAVE_DRV_EVENT - ErtsDrvEventDataState **free_event, -#endif ErtsDrvSelectDataState **free_select, ErtsNifSelectDataState **free_nif) { - erts_aint_t current_cio_time; - ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd))); - - current_cio_time = erts_atomic_read_acqb(&erts_check_io_time); *free_select = NULL; if (state->driver.select && (state->type != ERTS_EV_TYPE_DRV_SEL) - && !is_iotask_active(&state->driver.select->iniotask, current_cio_time) - && !is_iotask_active(&state->driver.select->outiotask, current_cio_time)) { - + && !is_iotask_active(&state->driver.select->iniotask) + && !is_iotask_active(&state->driver.select->outiotask)) { + *free_select = state->driver.select; state->driver.select = NULL; } @@ -635,382 +618,85 @@ check_fd_cleanup(ErtsDrvEventState *state, state->driver.nif = NULL; } -#if ERTS_CIO_HAVE_DRV_EVENT - *free_event = NULL; - if (state->driver.event - && (state->type != ERTS_EV_TYPE_DRV_EV) - && !is_iotask_active(&state->driver.event->iotask, current_cio_time)) { - - *free_event = state->driver.event; - state->driver.event = NULL; - } -#endif - -#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS if (((state->type != ERTS_EV_TYPE_NONE) - | state->remove_cnt -#if ERTS_CIO_HAVE_DRV_EVENT - | (state->driver.event != NULL) -#endif + | (state->driver.nif != NULL) | (state->driver.select != NULL)) == 0) { - hash_erase_drv_ev_state(state); - + erase_drv_ev_state(state); } -#endif } -static ERTS_INLINE int -check_cleanup_active_fd(ErtsSysFdType fd, -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - ErtsPollControlEntry *pce, - int *pce_ix, -#endif - erts_aint_t current_cio_time, - int may_sleep) -{ - ErtsDrvEventState *state; - int active = 0; - erts_mtx_t *mtx = fd_mtx(fd); - void *free_select = NULL; - void *free_nif = NULL; -#if ERTS_CIO_HAVE_DRV_EVENT - void *free_event = NULL; -#endif -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - ErtsPollEvents evon = 0, evoff = 0; -#endif - - erts_mtx_lock(mtx); - -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - state = &drv_ev_state[(int) fd]; +#ifdef __WIN32__ +# define MUST_DEFER(MAY_SLEEP) 1 #else - state = hash_get_drv_ev_state(fd); /* may be NULL! */ - if (state) -#endif - { - if (state->driver.select) { -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - if (is_iotask_active(&state->driver.select->iniotask, current_cio_time)) { - active = 1; - if ((state->events & ERTS_POLL_EV_IN) - && !(state->flags & ERTS_EV_FLAG_DEFER_IN_EV)) { - evoff |= ERTS_POLL_EV_IN; - state->flags |= ERTS_EV_FLAG_DEFER_IN_EV; - } - } - else if (state->flags & ERTS_EV_FLAG_DEFER_IN_EV) { - if (state->events & ERTS_POLL_EV_IN) - evon |= ERTS_POLL_EV_IN; - state->flags &= ~ERTS_EV_FLAG_DEFER_IN_EV; - } - if (is_iotask_active(&state->driver.select->outiotask, current_cio_time)) { - active = 1; - if ((state->events & ERTS_POLL_EV_OUT) - && !(state->flags & ERTS_EV_FLAG_DEFER_OUT_EV)) { - evoff |= ERTS_POLL_EV_OUT; - state->flags |= ERTS_EV_FLAG_DEFER_OUT_EV; - } - } - else if (state->flags & ERTS_EV_FLAG_DEFER_OUT_EV) { - if (state->events & ERTS_POLL_EV_OUT) - evon |= ERTS_POLL_EV_OUT; - state->flags &= ~ERTS_EV_FLAG_DEFER_OUT_EV; - } - if (active) - (void) 0; - else -#else - if (is_iotask_active(&state->driver.select->iniotask, current_cio_time) - || is_iotask_active(&state->driver.select->outiotask, current_cio_time)) - active = 1; - else -#endif - if (state->type != ERTS_EV_TYPE_DRV_SEL) { - free_select = state->driver.select; - state->driver.select = NULL; - } - } - - if (state->driver.nif) { - ErtsPollEvents rm_events = 0; - if (state->driver.nif->in.ddeselect_cnt) { - ASSERT(state->type == ERTS_EV_TYPE_NIF); - ASSERT(state->events & ERTS_POLL_EV_IN); - ASSERT(is_nil(state->driver.nif->in.pid)); - if (may_sleep || state->driver.nif->in.ddeselect_cnt == 1) { - rm_events = ERTS_POLL_EV_IN; - state->driver.nif->in.ddeselect_cnt = 0; - } - } - if (state->driver.nif->out.ddeselect_cnt) { - ASSERT(state->type == ERTS_EV_TYPE_NIF); - ASSERT(state->events & ERTS_POLL_EV_OUT); - ASSERT(is_nil(state->driver.nif->out.pid)); - if (may_sleep || state->driver.nif->out.ddeselect_cnt == 1) { - rm_events |= ERTS_POLL_EV_OUT; - state->driver.nif->out.ddeselect_cnt = 0; - } - } - if (rm_events) { - int do_wake = 0; - state->events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, - rm_events, 0, &do_wake); - } - if (state->events) - active = 1; - else if (state->type != ERTS_EV_TYPE_NIF) { - free_nif = state->driver.nif; - state->driver.nif = NULL; - } - } - -#if ERTS_CIO_HAVE_DRV_EVENT - if (state->driver.event) { - if (is_iotask_active(&state->driver.event->iotask, current_cio_time)) { -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - ErtsPollEvents evs = state->events & ~state->driver.event->deferred_events; - if (evs) { - evoff |= evs; - state->driver.event->deferred_events |= evs; - } -#endif - active = 1; - } - else if (state->type != ERTS_EV_TYPE_DRV_EV) { - free_event = state->driver.event; - state->driver.event = NULL; - } -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - else { - ErtsPollEvents evs = state->events & state->driver.event->deferred_events; - if (evs) { - evon |= evs; - state->driver.event->deferred_events = 0; - } - } -#endif - - } -#endif - -#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS - if (((state->type != ERTS_EV_TYPE_NONE) | state->remove_cnt | active) == 0) - hash_erase_drv_ev_state(state); -#endif - - } - - erts_mtx_unlock(mtx); - - if (free_select) - free_drv_select_data(free_select); - if (free_nif) - free_nif_select_data(free_nif); -#if ERTS_CIO_HAVE_DRV_EVENT - if (free_event) - free_drv_event_data(free_event); +# define MUST_DEFER(MAY_SLEEP) (MAY_SLEEP) #endif -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - if (evoff) { - ErtsPollControlEntry *pcep = &pce[(*pce_ix)++]; - pcep->fd = fd; - pcep->events = evoff; - pcep->on = 0; - } - if (evon) { - ErtsPollControlEntry *pcep = &pce[(*pce_ix)++]; - pcep->fd = fd; - pcep->events = evon; - pcep->on = 1; - } -#endif - - return active; -} - -static void -check_cleanup_active_fds(erts_aint_t current_cio_time, int may_sleep) -{ - int six = pollset.active_fd.six; - int eix = pollset.active_fd.eix; - erts_aint32_t no = erts_atomic32_read_dirty(&pollset.active_fd.no); - int size = pollset.active_fd.size; - int ix = six; -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - /* every fd might add two entries */ - Uint pce_sz = 2*sizeof(ErtsPollControlEntry)*no; - ErtsPollControlEntry *pctrl_entries = (pce_sz - ? erts_alloc(ERTS_ALC_T_TMP, pce_sz) - : NULL); - int pctrl_ix = 0; -#endif - - while (ix != eix) { - ErtsSysFdType fd = pollset.active_fd.array[ix]; - int nix = ix + 1; - if (nix >= size) - nix = 0; - ASSERT(fd != ERTS_SYS_FD_INVALID); - if (!check_cleanup_active_fd(fd, -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - pctrl_entries, - &pctrl_ix, -#endif - current_cio_time, - may_sleep)) { - no--; - if (ix == six) { -#ifdef DEBUG - pollset.active_fd.array[ix] = ERTS_SYS_FD_INVALID; -#endif - six = nix; - } - else { - pollset.active_fd.array[ix] = pollset.active_fd.array[six]; -#ifdef DEBUG - pollset.active_fd.array[six] = ERTS_SYS_FD_INVALID; -#endif - six++; - if (six >= size) - six = 0; - } - } - ix = nix; - } - -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - ASSERT(pctrl_ix <= pce_sz/sizeof(ErtsPollControlEntry)); - if (pctrl_ix) - ERTS_CIO_POLL_CTLV(pollset.ps, pctrl_entries, pctrl_ix); - if (pctrl_entries) - erts_free(ERTS_ALC_T_TMP, pctrl_entries); -#endif - - pollset.active_fd.six = six; - pollset.active_fd.eix = eix; - erts_atomic32_set_relb(&pollset.active_fd.no, no); -} - -static void grow_active_fds(void) -{ - ASSERT(pollset.active_fd.six == pollset.active_fd.eix); - pollset.active_fd.six = 0; - pollset.active_fd.eix = pollset.active_fd.size; - pollset.active_fd.size += ERTS_ACTIVE_FD_INC; - pollset.active_fd.array = erts_realloc(ERTS_ALC_T_ACTIVE_FD_ARR, - pollset.active_fd.array, - pollset.active_fd.size*sizeof(ErtsSysFdType)); -#ifdef DEBUG - { - int i; - for (i = pollset.active_fd.eix + 1; i < pollset.active_fd.size; i++) - pollset.active_fd.array[i] = ERTS_SYS_FD_INVALID; - } -#endif -} - -static ERTS_INLINE void -add_active_fd(ErtsSysFdType fd) -{ - int eix = pollset.active_fd.eix; - int size = pollset.active_fd.size; - - pollset.active_fd.array[eix] = fd; - - erts_atomic32_set_relb(&pollset.active_fd.no, - (erts_atomic32_read_dirty(&pollset.active_fd.no) - + 1)); - - eix++; - if (eix >= size) - eix = 0; - pollset.active_fd.eix = eix; - - if (pollset.active_fd.six == eix) { - grow_active_fds(); - } -} - int -ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, - ErlDrvEvent e, - int mode, - int on) +driver_select(ErlDrvPort ix, ErlDrvEvent e, int mode, int on) { void (*stop_select_fn)(ErlDrvEvent, void*) = NULL; Port *prt = erts_drvport2port(ix); Eterm id = erts_drvport2id(ix); ErtsSysFdType fd = (ErtsSysFdType) e; ErtsPollEvents ctl_events = (ErtsPollEvents) 0; - ErtsPollEvents new_events, old_events; + ErtsPollEvents old_events; + ErtsPollEvents new_events; + ErtsPollOp ctl_op = ERTS_POLL_OP_MOD; ErtsDrvEventState *state; - int wake_poller; + int wake_poller = 0; int ret; -#if ERTS_CIO_HAVE_DRV_EVENT - ErtsDrvEventDataState *free_event = NULL; -#endif ErtsDrvSelectDataState *free_select = NULL; ErtsNifSelectDataState *free_nif = NULL; #ifdef USE_VM_PROBES DTRACE_CHARBUF(name, 64); #endif + ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_CHECK_IO); - if (prt == ERTS_INVALID_ERL_DRV_PORT) + if (prt == ERTS_INVALID_ERL_DRV_PORT) { + ERTS_MSACC_POP_STATE(); return -1; + } ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state_len)) { - if (fd < 0) { - return -1; - } - if (fd >= max_fds) { - drv_select_large_fd_error(ix, fd, mode, on); - return -1; - } - grow_drv_ev_state(fd); + if (!grow_drv_ev_state(fd)) { + if (fd > 0) drv_select_large_fd_error(ix, fd, mode, on); + ERTS_MSACC_POP_STATE(); + return -1; } #endif erts_mtx_lock(fd_mtx(fd)); -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - state = &drv_ev_state[(int) fd]; -#else - state = hash_get_drv_ev_state(fd); /* may be NULL! */ -#endif + state = get_drv_ev_state(fd); /* may be NULL! */ - if (!on && (mode&ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) { - if (IS_FD_UNKNOWN(state)) { - /* fast track to stop_select callback */ - stop_select_fn = prt->drv_ptr->stop_select; -#ifdef USE_VM_PROBES - strncpy(name, prt->drv_ptr->name, - sizeof(DTRACE_CHARBUF_NAME(name))-1); - name[sizeof(name)-1] = '\0'; -#endif - ret = 0; - goto done_unknown; - } - mode |= (ERL_DRV_READ | ERL_DRV_WRITE); - wake_poller = 1; /* to eject fd from pollset (if needed) */ - } - else wake_poller = 0; + DEBUG_PRINT_FD("driver_select(%T, %p, %s, %d)", + state, id, fd, drvmode2str(mode), on); -#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS - if (state == NULL) { - state = hash_new_drv_ev_state(fd); + if (!on) { + if (IS_FD_UNKNOWN(state)) { + if ((mode&ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) { + /* fast track to stop_select callback */ + stop_select_fn = prt->drv_ptr->stop_select; + #ifdef USE_VM_PROBES + strncpy(name, prt->drv_ptr->name, + sizeof(DTRACE_CHARBUF_NAME(name))-1); + name[sizeof(name)-1] = '\0'; + #endif + } + ret = 0; + goto done_unknown; + } + else if ((mode&ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) { + mode |= (ERL_DRV_READ | ERL_DRV_WRITE); + } } -#endif + + state = new_drv_ev_state(state, fd); switch (state->type) { -#if ERTS_CIO_HAVE_DRV_EVENT - case ERTS_EV_TYPE_DRV_EV: -#endif case ERTS_EV_TYPE_NIF: drv_select_steal(ix, state, mode, on); break; @@ -1032,7 +718,9 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, ASSERT(state->type == ERTS_EV_TYPE_NONE); break; - }} + } + default: break; + } if (mode & ERL_DRV_READ) { if (state->type == ERTS_EV_TYPE_DRV_SEL) { @@ -1040,7 +728,7 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, if (owner != id && is_not_nil(owner)) drv_select_steal(ix, state, mode, on); } - ctl_events |= ERTS_POLL_EV_IN; + ctl_events = ERTS_POLL_EV_IN; } if (mode & ERL_DRV_WRITE) { if (state->type == ERTS_EV_TYPE_DRV_SEL) { @@ -1049,100 +737,105 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, drv_select_steal(ix, state, mode, on); } ctl_events |= ERTS_POLL_EV_OUT; - } + } + ASSERT((state->type == ERTS_EV_TYPE_DRV_SEL) || (state->type == ERTS_EV_TYPE_NONE && !state->events)); - if (!on && !(state->flags & ERTS_EV_FLAG_USED) - && state->events && !(state->events & ~ctl_events)) { - /* Old driver removing all events. At least wake poller. - It will not make close() 100% safe but it will prevent - actions delayed by poll timeout. */ - wake_poller = 1; - } + old_events = state->events; - new_events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, ctl_events, on, &wake_poller); + if (on) { + ctl_events &= ~old_events; + state->events |= ctl_events; + if (ctl_events & ERTS_POLL_EV_IN && (!state->driver.select || !is_iotask_active(&state->driver.select->iniotask))) + state->active_events |= ERTS_POLL_EV_IN; + if (ctl_events & ERTS_POLL_EV_OUT && (!state->driver.select || !is_iotask_active(&state->driver.select->outiotask))) + state->active_events |= ERTS_POLL_EV_OUT; + if (old_events == 0 && !(state->flags & ERTS_EV_FLAG_USED)) { + ctl_op = ERTS_POLL_OP_ADD; + } + } + else { + ctl_events &= old_events; + state->events &= ~ctl_events; + state->active_events &= ~ctl_events; - if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) { - if (state->type == ERTS_EV_TYPE_DRV_SEL && !state->events) { - state->type = ERTS_EV_TYPE_NONE; - state->flags &= ~ERTS_EV_FLAG_USED; - state->driver.select->inport = NIL; - state->driver.select->outport = NIL; - } - ret = -1; - goto done; + if (!state->events) { + if (!(state->flags & ERTS_EV_FLAG_USED) || mode & ERL_DRV_USE) + ctl_op = ERTS_POLL_OP_DEL; + } } - old_events = state->events; + if (ctl_events || ctl_op == ERTS_POLL_OP_DEL) { - ASSERT(on - ? (new_events == (state->events | ctl_events)) - : (new_events == (state->events & ~ctl_events))); + new_events = erts_io_control_wakeup(state, ctl_op, + state->active_events, + &wake_poller); - ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL - || state->type == ERTS_EV_TYPE_NONE); + ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL || state->type == ERTS_EV_TYPE_NONE); + } - state->events = new_events; - if (ctl_events) { - if (on) { + if (on) { + if (ctl_events) { if (!state->driver.select) - state->driver.select = alloc_drv_select_data(); + state->driver.select = alloc_drv_select_data(state->fd); if (state->type == ERTS_EV_TYPE_NONE) state->type = ERTS_EV_TYPE_DRV_SEL; ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL); - if (ctl_events & ERTS_POLL_EV_IN) + if (ctl_events & ERTS_POLL_EV_IN) { state->driver.select->inport = id; - if (ctl_events & ERTS_POLL_EV_OUT) + if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) + iready(id, state); + } + if (ctl_events & ERTS_POLL_EV_OUT) { state->driver.select->outport = id; - if (mode & ERL_DRV_USE) { + if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) + oready(id, state); + } + if (mode & ERL_DRV_USE) state->flags |= ERTS_EV_FLAG_USED; - } - } - else { /* off */ - if (state->type == ERTS_EV_TYPE_DRV_SEL) { - if (ctl_events & ERTS_POLL_EV_IN) { - abort_tasks(state, ERL_DRV_READ); - state->driver.select->inport = NIL; - } - if (ctl_events & ERTS_POLL_EV_OUT) { - abort_tasks(state, ERL_DRV_WRITE); - state->driver.select->outport = NIL; - } - if (new_events == 0) { - if (old_events != 0) { - remember_removed(state, &pollset); - } - if ((mode & ERL_DRV_USE) || !(state->flags & ERTS_EV_FLAG_USED)) { - state->type = ERTS_EV_TYPE_NONE; - state->flags &= ~ERTS_EV_FLAG_USED; - } - /*else keep it, as fd will probably be selected upon again */ - } - } - if ((mode & ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) { - erts_driver_t* drv_ptr = prt->drv_ptr; - ASSERT(new_events==0); - if (state->remove_cnt == 0 || !wake_poller) { - /* Safe to close fd now as it is not in pollset - or there was no need to eject fd (kernel poll) */ - stop_select_fn = drv_ptr->stop_select; + } + } + else { /* off */ + if (state->type == ERTS_EV_TYPE_DRV_SEL) { + if (ctl_events & ERTS_POLL_EV_IN) { + abort_tasks(state, ERL_DRV_READ); + state->driver.select->inport = NIL; + } + if (ctl_events & ERTS_POLL_EV_OUT) { + abort_tasks(state, ERL_DRV_WRITE); + state->driver.select->outport = NIL; + } + if (state->events == 0) { + if ((mode & ERL_DRV_USE) || !(state->flags & ERTS_EV_FLAG_USED)) { + state->type = ERTS_EV_TYPE_NONE; + state->flags = 0; + } + /*else keep it, as fd will probably be selected upon again */ + } + } + if ((mode & ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) { + erts_driver_t* drv_ptr = prt->drv_ptr; + ASSERT(state->events==0); + if (!wake_poller) { + /* Safe to close fd now as it is not in pollset + or there was no need to eject fd (kernel poll) */ + stop_select_fn = drv_ptr->stop_select; #ifdef USE_VM_PROBES - strncpy(name, prt->drv_ptr->name, sizeof(name)-1); - name[sizeof(name)-1] = '\0'; + strncpy(name, prt->drv_ptr->name, sizeof(name)-1); + name[sizeof(name)-1] = '\0'; #endif - } - else { - /* Not safe to close fd, postpone stop_select callback. */ - state->type = ERTS_EV_TYPE_STOP_USE; - state->driver.stop.drv_ptr = drv_ptr; - if (drv_ptr->handle) { - erts_ddll_reference_referenced_driver(drv_ptr->handle); - } - } - } - } + } + else { + /* Not safe to close fd, postpone stop_select callback. */ + state->type = ERTS_EV_TYPE_STOP_USE; + state->driver.stop.drv_ptr = drv_ptr; + if (drv_ptr->handle) { + erts_ddll_reference_referenced_driver(drv_ptr->handle); + } + } + } } ret = 0; @@ -1150,9 +843,6 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, done: check_fd_cleanup(state, -#if ERTS_CIO_HAVE_DRV_EVENT - &free_event, -#endif &free_select, &free_nif); @@ -1170,61 +860,47 @@ done_unknown: if (free_nif) free_nif_select_data(free_nif); -#if ERTS_CIO_HAVE_DRV_EVENT - if (free_event) - free_drv_event_data(free_event); -#endif + ERTS_MSACC_POP_STATE(); + return ret; } int -ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, - ErlNifEvent e, - enum ErlNifSelectFlags mode, - void* obj, - const ErlNifPid* pid, - Eterm ref) +enif_select(ErlNifEnv* env, + ErlNifEvent e, + enum ErlNifSelectFlags mode, + void* obj, + const ErlNifPid* pid, + Eterm ref) { int on; ErtsResource* resource = DATA_TO_RESOURCE(obj); ErtsSysFdType fd = (ErtsSysFdType) e; ErtsPollEvents ctl_events = (ErtsPollEvents) 0; - ErtsPollEvents new_events, old_events; + ErtsPollEvents old_events; + ErtsPollOp ctl_op = ERTS_POLL_OP_MOD; ErtsDrvEventState *state; - int wake_poller; - int ret; + int ret, wake_poller = 0; enum { NO_STOP=0, CALL_STOP, CALL_STOP_AND_RELEASE } call_stop = NO_STOP; -#if ERTS_CIO_HAVE_DRV_EVENT - ErtsDrvEventDataState *free_event = NULL; -#endif ErtsDrvSelectDataState *free_select = NULL; ErtsNifSelectDataState *free_nif = NULL; -#ifdef USE_VM_PROBES - DTRACE_CHARBUF(name, 64); -#endif ASSERT(!(resource->monitors && resource->monitors->is_dying)); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state_len)) { - if (fd < 0) { - return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT; - } - if (fd >= max_fds) { - nif_select_large_fd_error(fd, mode, resource, ref); - return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT; - } - grow_drv_ev_state(fd); + if (!grow_drv_ev_state(fd)) { + if (fd > 0) nif_select_large_fd_error(fd, mode, resource, ref); + return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT; } #endif erts_mtx_lock(fd_mtx(fd)); -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - state = &drv_ev_state[(int) fd]; -#else - state = hash_get_drv_ev_state(fd); /* may be NULL! */ -#endif + state = get_drv_ev_state(fd); /* may be NULL! */ + + DEBUG_PRINT_FD("enif_select(%T, %d, %s, %p, %T, %T)", + state, env->proc->common.id, fd, nifmode2str(mode), resource, + pid ? pid->pid : THE_NON_VALUE, ref); if (mode & ERL_NIF_SELECT_STOP) { ASSERT(resource->type->stop); @@ -1236,13 +912,12 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, } on = 0; mode = ERL_DRV_READ | ERL_DRV_WRITE | ERL_DRV_USE; - wake_poller = 1; /* to eject fd from pollset (if needed) */ ctl_events = ERTS_POLL_EV_IN | ERTS_POLL_EV_OUT; + ctl_op = ERTS_POLL_OP_DEL; } else { on = 1; ASSERT(mode); - wake_poller = 0; if (mode & ERL_DRV_READ) { ctl_events |= ERTS_POLL_EV_IN; } @@ -1251,11 +926,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, } } -#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS - if (state == NULL) { - state = hash_new_drv_ev_state(fd); - } -#endif + state = new_drv_ev_state(state,fd); switch (state->type) { case ERTS_EV_TYPE_NIF: @@ -1266,9 +937,6 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, if (state->driver.stop.resource != resource) nif_select_steal(state, ERL_DRV_READ | ERL_DRV_WRITE, resource, ref); break; -#if ERTS_CIO_HAVE_DRV_EVENT - case ERTS_EV_TYPE_DRV_EV: -#endif case ERTS_EV_TYPE_DRV_SEL: nif_select_steal(state, mode, resource, ref); break; @@ -1289,37 +957,52 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, } ASSERT(state->type == ERTS_EV_TYPE_NONE); break; - }} + } + default: break; + } ASSERT((state->type == ERTS_EV_TYPE_NIF) || (state->type == ERTS_EV_TYPE_NONE && !state->events)); - new_events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, ctl_events, on, &wake_poller); + old_events = state->events; - if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) { - if (state->type == ERTS_EV_TYPE_NIF && !state->events) { - state->type = ERTS_EV_TYPE_NONE; - state->flags &= ~ERTS_EV_FLAG_USED; - state->driver.nif->in.pid = NIL; - state->driver.nif->out.pid = NIL; - state->driver.nif->in.ddeselect_cnt = 0; - state->driver.nif->out.ddeselect_cnt = 0; - state->driver.stop.resource = NULL; - } - ret = INT_MIN | ERL_NIF_SELECT_FAILED; - goto done; + if (on) { + ctl_events &= ~old_events; + state->events |= ctl_events; + state->active_events |= ctl_events; + if (state->type == ERTS_EV_TYPE_NONE) + ctl_op = ERTS_POLL_OP_ADD; + } + else { + ctl_events &= old_events; + state->events &= ~ctl_events; + state->active_events &= ~ctl_events; } - old_events = state->events; + if (ctl_events || ctl_op == ERTS_POLL_OP_DEL) { + ErtsPollEvents new_events; - ASSERT(on - ? (new_events == (state->events | ctl_events)) - : (new_events == (state->events & ~ctl_events))); + new_events = erts_io_control_wakeup(state, ctl_op, + state->active_events, + &wake_poller); + + if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) { + if (state->type == ERTS_EV_TYPE_NIF && !old_events) { + state->type = ERTS_EV_TYPE_NONE; + state->flags = 0; + state->driver.nif->in.pid = NIL; + state->driver.nif->out.pid = NIL; + state->driver.stop.resource = NULL; + } + ret = INT_MIN | ERL_NIF_SELECT_FAILED; + goto done; + } + ASSERT(new_events == state->events); + } ASSERT(state->type == ERTS_EV_TYPE_NIF || state->type == ERTS_EV_TYPE_NONE); - state->events = new_events; if (on) { const Eterm recipient = pid ? pid->pid : env->proc->common.id; Uint32* refn; @@ -1332,7 +1015,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, } ASSERT(state->type == ERTS_EV_TYPE_NIF); ASSERT(state->driver.stop.resource == resource); - if (ctl_events & ERTS_POLL_EV_IN) { + if (mode & ERL_DRV_READ) { state->driver.nif->in.pid = recipient; if (is_immed(ref)) { state->driver.nif->in.immed = ref; @@ -1340,13 +1023,11 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, ASSERT(is_internal_ref(ref)); refn = internal_ref_numbers(ref); state->driver.nif->in.immed = THE_NON_VALUE; - state->driver.nif->in.refn[0] = refn[0]; - state->driver.nif->in.refn[1] = refn[1]; - state->driver.nif->in.refn[2] = refn[2]; + sys_memcpy(state->driver.nif->in.refn, refn, + sizeof(state->driver.nif->in.refn)); } - state->driver.nif->in.ddeselect_cnt = 0; } - if (ctl_events & ERTS_POLL_EV_OUT) { + if (mode & ERL_DRV_WRITE) { state->driver.nif->out.pid = recipient; if (is_immed(ref)) { state->driver.nif->out.immed = ref; @@ -1354,11 +1035,9 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, ASSERT(is_internal_ref(ref)); refn = internal_ref_numbers(ref); state->driver.nif->out.immed = THE_NON_VALUE; - state->driver.nif->out.refn[0] = refn[0]; - state->driver.nif->out.refn[1] = refn[1]; - state->driver.nif->out.refn[2] = refn[2]; + sys_memcpy(state->driver.nif->out.refn, refn, + sizeof(state->driver.nif->out.refn)); } - state->driver.nif->out.ddeselect_cnt = 0; } ret = 0; } @@ -1366,14 +1045,9 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, if (state->type == ERTS_EV_TYPE_NIF) { state->driver.nif->in.pid = NIL; state->driver.nif->out.pid = NIL; - state->driver.nif->in.ddeselect_cnt = 0; - state->driver.nif->out.ddeselect_cnt = 0; - if (old_events != 0) { - remember_removed(state, &pollset); - } } - ASSERT(new_events==0); - if (state->remove_cnt == 0 || !wake_poller) { + ASSERT(state->events==0); + if (!wake_poller) { /* * Safe to close fd now as it is not in pollset * or there was no need to eject fd (kernel poll) @@ -1405,9 +1079,6 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, done: check_fd_cleanup(state, -#if ERTS_CIO_HAVE_DRV_EVENT - &free_event, -#endif &free_select, &free_nif); @@ -1424,160 +1095,7 @@ done_unknown: if (free_nif) free_nif_select_data(free_nif); -#if ERTS_CIO_HAVE_DRV_EVENT - if (free_event) - free_drv_event_data(free_event); -#endif - return ret; -} - - -int -ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix, - ErlDrvEvent e, - ErlDrvEventData event_data) -{ -#if !ERTS_CIO_HAVE_DRV_EVENT - return -1; -#else - ErtsSysFdType fd = (ErtsSysFdType) e; - ErtsPollEvents events; - ErtsPollEvents add_events; - ErtsPollEvents remove_events; - Eterm id = erts_drvport2id(ix); - ErtsDrvEventState *state; - int do_wake = 0; - int ret; -#if ERTS_CIO_HAVE_DRV_EVENT - ErtsDrvEventDataState *free_event; -#endif - ErtsDrvSelectDataState *free_select; - ErtsNifSelectDataState *free_nif; - Port *prt = erts_drvport2port(ix); - - if (prt == ERTS_INVALID_ERL_DRV_PORT) - return -1; - - ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); - -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state_len)) { - if (fd < 0) - return -1; - if (fd >= max_fds) { - event_large_fd_error(ix, fd, event_data); - return -1; - } - grow_drv_ev_state(fd); - } -#endif - - erts_mtx_lock(fd_mtx(fd)); - -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - state = &drv_ev_state[(int) fd]; -#else - /* Could use hash_new directly, but want to keep the normal case fast */ - state = hash_get_drv_ev_state(fd); - if (state == NULL) { - state = hash_new_drv_ev_state(fd); - } -#endif - - switch (state->type) { - case ERTS_EV_TYPE_DRV_EV: - if (state->driver.event->port == id) break; - /*fall through*/ - case ERTS_EV_TYPE_DRV_SEL: - drv_event_steal(ix, state, event_data); - break; - case ERTS_EV_TYPE_STOP_USE: { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - print_drv_event_op(dsbufp, ix, fd, event_data); - steal_pending_stop_use(dsbufp, ix, state, 0, 1); - break; - } - } - - ASSERT(state->type == ERTS_EV_TYPE_DRV_EV - || state->type == ERTS_EV_TYPE_NONE); - - events = state->events; - - if (!event_data) { - remove_events = events; - add_events = 0; - } - else { - remove_events = ~event_data->events & events; - add_events = ~events & event_data->events; - } - - if (add_events) { - events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, add_events, 1, &do_wake); - if (events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) { - ret = -1; - goto done; - } - } - if (remove_events) { - events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, remove_events, 0, &do_wake); - if (events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) { - ret = -1; - goto done; - } - } - if (event_data && event_data->events != 0) { - if (state->type == ERTS_EV_TYPE_DRV_EV) { - state->driver.event->removed_events &= ~add_events; - state->driver.event->removed_events |= remove_events; - } - else { - if (!state->driver.event) - state->driver.event = alloc_drv_event_data(); - state->driver.event->port = id; - state->driver.event->removed_events = (ErtsPollEvents) 0; - state->type = ERTS_EV_TYPE_DRV_EV; - } - state->driver.event->data = event_data; - } - else { - if (state->type == ERTS_EV_TYPE_DRV_EV) { - abort_tasks(state, 0); - state->driver.event->port = NIL; - state->driver.event->data = NULL; - state->driver.event->removed_events = (ErtsPollEvents) 0; - } - state->type = ERTS_EV_TYPE_NONE; - remember_removed(state, &pollset); - } - state->events = events; - ASSERT(event_data ? events == event_data->events : events == 0); - - ret = 0; - -done: - - check_fd_cleanup(state, -#if ERTS_CIO_HAVE_DRV_EVENT - &free_event, -#endif - &free_select, - &free_nif); - - erts_mtx_unlock(fd_mtx(fd)); - - if (free_select) - free_drv_select_data(free_select); - if (free_nif) - free_nif_select_data(free_nif); -#if ERTS_CIO_HAVE_DRV_EVENT - if (free_event) - free_drv_event_data(free_event); -#endif - return ret; -#endif } static ERTS_INLINE int @@ -1611,11 +1129,6 @@ need2steal(ErtsDrvEventState *state, int mode) do_steal = 1; break; -#if ERTS_CIO_HAVE_DRV_EVENT - case ERTS_EV_TYPE_DRV_EV: - do_steal |= chk_stale(state->driver.event->port, state, 0); - break; -#endif case ERTS_EV_TYPE_STOP_USE: case ERTS_EV_TYPE_STOP_NIF: ASSERT(0); @@ -1649,7 +1162,7 @@ print_driver_name(erts_dsprintf_buf_t *dsbufp, Eterm id) static void steal(erts_dsprintf_buf_t *dsbufp, ErtsDrvEventState *state, int mode) { - erts_dsprintf(dsbufp, "stealing control of fd=%d from ", (int) GET_FD(state->fd)); + erts_dsprintf(dsbufp, "stealing control of fd=%d from ", (int) state->fd); switch (state->type) { case ERTS_EV_TYPE_DRV_SEL: { int deselect_mode = 0; @@ -1673,7 +1186,7 @@ steal(erts_dsprintf_buf_t *dsbufp, ErtsDrvEventState *state, int mode) if (deselect_mode) deselect(state, deselect_mode); else { - erts_dsprintf(dsbufp, "no one", (int) GET_FD(state->fd)); + erts_dsprintf(dsbufp, "no one", (int) state->fd); ASSERT(0); } erts_dsprintf(dsbufp, "\n"); @@ -1698,30 +1211,13 @@ steal(erts_dsprintf_buf_t *dsbufp, ErtsDrvEventState *state, int mode) erts_dsprintf(dsbufp, "\n"); break; } -#if ERTS_CIO_HAVE_DRV_EVENT - case ERTS_EV_TYPE_DRV_EV: { - Eterm eid = state->driver.event->port; - if (is_nil(eid)) { - erts_dsprintf(dsbufp, "no one", (int) state->fd); - ASSERT(0); - } - else { - erts_dsprintf(dsbufp, "event driver "); - print_driver_name(dsbufp, eid); - erts_dsprintf(dsbufp, "%T ", eid); - } - erts_dsprintf(dsbufp, "\n"); - deselect(state, 0); - break; - } -#endif case ERTS_EV_TYPE_STOP_USE: case ERTS_EV_TYPE_STOP_NIF: { ASSERT(0); break; } default: - erts_dsprintf(dsbufp, "no one\n", (int) GET_FD(state->fd)); + erts_dsprintf(dsbufp, "no one\n", (int) state->fd); ASSERT(0); } } @@ -1735,7 +1231,7 @@ print_drv_select_op(erts_dsprintf_buf_t *dsbufp, "driver_select(%p, %d,%s%s%s%s, %d) " "by ", ix, - (int) GET_FD(fd), + (int) fd, mode & ERL_DRV_READ ? " ERL_DRV_READ" : "", mode & ERL_DRV_WRITE ? " ERL_DRV_WRITE" : "", mode & ERL_DRV_USE ? " ERL_DRV_USE" : "", @@ -1752,7 +1248,7 @@ print_nif_select_op(erts_dsprintf_buf_t *dsbufp, { erts_dsprintf(dsbufp, "enif_select(_, %d,%s%s%s, %T:%T, %T) ", - (int) GET_FD(fd), + (int) fd, mode & ERL_NIF_SELECT_READ ? " READ" : "", mode & ERL_NIF_SELECT_WRITE ? " WRITE" : "", mode & ERL_NIF_SELECT_STOP ? " STOP" : "", @@ -1791,7 +1287,7 @@ large_fd_error_common(erts_dsprintf_buf_t *dsbufp, ErtsSysFdType fd) { erts_dsprintf(dsbufp, "fd=%d is larger than the largest allowed fd=%d\n", - (int) fd, max_fds - 1); + (int) fd, drv_ev_state.max_fds - 1); } static void @@ -1847,7 +1343,7 @@ steal_pending_stop_use(erts_dsprintf_buf_t *dsbufp, ErlDrvPort ix, erts_ddll_dereference_driver(state->driver.stop.drv_ptr->handle); } state->type = ERTS_EV_TYPE_NONE; - state->flags &= ~ERTS_EV_FLAG_USED; + state->flags = 0; state->driver.stop.drv_ptr = NULL; } else { @@ -1884,9 +1380,9 @@ steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErtsResource* resource, erts_dsprintf(dsbufp, "called before stop was called for NIF resource %T:%T\n", rt->module, rt->name); - enif_release_resource(state->driver.stop.resource); + enif_release_resource(state->driver.stop.resource->data); state->type = ERTS_EV_TYPE_NONE; - state->flags &= ~ERTS_EV_FLAG_USED; + state->flags = 0; state->driver.stop.resource = NULL; } else { @@ -1896,59 +1392,9 @@ steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErtsResource* resource, } - -#if ERTS_CIO_HAVE_DRV_EVENT - -static void -print_drv_event_op(erts_dsprintf_buf_t *dsbufp, - ErlDrvPort ix, ErtsSysFdType fd, ErlDrvEventData event_data) -{ - Port *pp = erts_drvport2port(ix); - erts_dsprintf(dsbufp, "driver_event(%p, %d, ", ix, (int) fd); - if (!event_data) - erts_dsprintf(dsbufp, "NULL"); - else - erts_dsprintf(dsbufp, "{0x%x, 0x%x}", - (unsigned int) event_data->events, - (unsigned int) event_data->revents); - erts_dsprintf(dsbufp, ") by "); - if (pp != ERTS_INVALID_ERL_DRV_PORT) - print_driver_name(dsbufp, pp->common.id); - erts_dsprintf(dsbufp, "driver %T ", pp != ERTS_INVALID_ERL_DRV_PORT ? pp->common.id : NIL); -} - -static void -drv_event_steal(ErlDrvPort ix, ErtsDrvEventState *state, ErlDrvEventData event_data) -{ - if (need2steal(state, ERL_DRV_READ|ERL_DRV_WRITE)) { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - print_drv_event_op(dsbufp, ix, state->fd, event_data); - steal(dsbufp, state, ERL_DRV_READ|ERL_DRV_WRITE); - erts_send_error_to_logger_nogl(dsbufp); - } - else if (state->type == ERTS_EV_TYPE_DRV_SEL) { - ASSERT(state->flags & ERTS_EV_FLAG_USED); - deselect(state, 0); - } -} - -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS -static void -event_large_fd_error(ErlDrvPort ix, ErtsSysFdType fd, ErlDrvEventData event_data) -{ - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - print_drv_event_op(dsbufp, ix, fd, event_data); - erts_dsprintf(dsbufp, "failed: "); - large_fd_error_common(dsbufp, fd); - erts_send_error_to_logger_nogl(dsbufp); -} -#endif -#endif - static ERTS_INLINE int io_task_schedule_allowed(ErtsDrvEventState *state, - ErtsPortTaskType type, - erts_aint_t current_cio_time) + ErtsPortTaskType type) { ErtsIoTask *io_task; @@ -1956,71 +1402,53 @@ io_task_schedule_allowed(ErtsDrvEventState *state, case ERTS_PORT_TASK_INPUT: if (!state->driver.select) return 0; -#if ERTS_CIO_HAVE_DRV_EVENT - if (state->driver.event) - return 0; -#endif io_task = &state->driver.select->iniotask; break; case ERTS_PORT_TASK_OUTPUT: if (!state->driver.select) return 0; -#if ERTS_CIO_HAVE_DRV_EVENT - if (state->driver.event) - return 0; -#endif io_task = &state->driver.select->outiotask; break; -#if ERTS_CIO_HAVE_DRV_EVENT - case ERTS_PORT_TASK_EVENT: - if (!state->driver.event) - return 0; - if (state->driver.select) - return 0; - io_task = &state->driver.event->iotask; - break; -#endif default: ERTS_INTERNAL_ERROR("Invalid I/O-task type"); return 0; } - return !is_iotask_active(io_task, current_cio_time); + return !is_iotask_active(io_task); } static ERTS_INLINE void -iready(Eterm id, ErtsDrvEventState *state, erts_aint_t current_cio_time) +iready(Eterm id, ErtsDrvEventState *state) { if (io_task_schedule_allowed(state, - ERTS_PORT_TASK_INPUT, - current_cio_time)) { + ERTS_PORT_TASK_INPUT)) { ErtsIoTask *iotask = &state->driver.select->iniotask; - erts_atomic_set_nob(&iotask->executed_time, current_cio_time); if (erts_port_task_schedule(id, &iotask->task, ERTS_PORT_TASK_INPUT, (ErlDrvEvent) state->fd) != 0) { stale_drv_select(id, state, ERL_DRV_READ); - } - add_active_fd(state->fd); + } else { + DEBUG_PRINT_FD("schedule ready_input(%T, %d)", + state, id, state->fd); + } } } static ERTS_INLINE void -oready(Eterm id, ErtsDrvEventState *state, erts_aint_t current_cio_time) +oready(Eterm id, ErtsDrvEventState *state) { if (io_task_schedule_allowed(state, - ERTS_PORT_TASK_OUTPUT, - current_cio_time)) { + ERTS_PORT_TASK_OUTPUT)) { ErtsIoTask *iotask = &state->driver.select->outiotask; - erts_atomic_set_nob(&iotask->executed_time, current_cio_time); if (erts_port_task_schedule(id, &iotask->task, ERTS_PORT_TASK_OUTPUT, (ErlDrvEvent) state->fd) != 0) { stale_drv_select(id, state, ERL_DRV_WRITE); - } - add_active_fd(state->fd); + } else { + DEBUG_PRINT_FD("schedule ready_output(%T, %d)", state, id, state->fd); + } } } @@ -2072,131 +1500,59 @@ send_event_tuple(struct erts_nif_select_event* e, ErtsResource* resource, erts_proc_unlock(rp, rp_locks); } - -#if ERTS_CIO_HAVE_DRV_EVENT -static ERTS_INLINE void -eready(Eterm id, ErtsDrvEventState *state, ErlDrvEventData event_data, - erts_aint_t current_cio_time) -{ - if (io_task_schedule_allowed(state, - ERTS_PORT_TASK_EVENT, - current_cio_time)) { - ErtsIoTask *iotask = &state->driver.event->iotask; - erts_atomic_set_nob(&iotask->executed_time, current_cio_time); - if (erts_port_task_schedule(id, - &iotask->task, - ERTS_PORT_TASK_EVENT, - (ErlDrvEvent) state->fd, - event_data) != 0) { - stale_drv_select(id, state, 0); - } - add_active_fd(state->fd); - } -} -#endif - static void bad_fd_in_pollset(ErtsDrvEventState *, Eterm inport, Eterm outport); - void -ERTS_CIO_EXPORT(erts_check_io_interrupt)(int set) +erts_check_io_interrupt(ErtsPollThread *psi, int set) { - ERTS_CIO_POLL_INTR(pollset.ps, set); + if (psi) { +#if ERTS_POLL_USE_FALLBACK + if (psi->ps == get_fallback()) { + erts_poll_interrupt_flbk(psi->ps, set); + return; + } +#endif + erts_poll_interrupt(psi->ps, set); + } } -void -ERTS_CIO_EXPORT(erts_check_io_interrupt_timed)(int set, - ErtsMonotonicTime timeout_time) -{ - ERTS_CIO_POLL_INTR_TMD(pollset.ps, set, timeout_time); +ErtsPollThread * +erts_create_pollset_thread(int id) { + return psiv+id; } -#if !ERTS_CIO_DEFER_ACTIVE_EVENTS -/* - * Number of ignored events, for a lingering fd added by enif_select(), - * until we deselect fd-event from pollset. - */ -# define ERTS_NIF_DELAYED_DESELECT 20 -#else -/* Disable delayed deselect as pollset cannot handle active events */ -# define ERTS_NIF_DELAYED_DESELECT 1 -#endif - void -ERTS_CIO_EXPORT(erts_check_io)(int do_wait) +erts_check_io(ErtsPollThread *psi) { - ErtsPollResFd *pollres; int pollres_len; - ErtsMonotonicTime timeout_time; int poll_ret, i; - erts_aint_t current_cio_time; - ErtsSchedulerData *esdp = erts_get_scheduler_data(); - - ASSERT(esdp); + ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_CHECK_IO); restart: -#ifdef ERTS_BREAK_REQUESTED - if (ERTS_BREAK_REQUESTED) - erts_do_break_handling(); -#endif - -#ifdef ERTS_SIGNAL_STATE /* ifndef ERTS_SMP */ - if (ERTS_SIGNAL_STATE) { - erts_handle_signal_state(); - } -#endif - - /* Figure out timeout value */ - timeout_time = (do_wait - ? erts_check_next_timeout_time(esdp) - : ERTS_POLL_NO_TIMEOUT /* poll only */); - - /* - * No need for an atomic inc op when incrementing - * erts_check_io_time, since only one thread can - * check io at a time. - */ - current_cio_time = erts_atomic_read_dirty(&erts_check_io_time); - current_cio_time++; - erts_atomic_set_relb(&erts_check_io_time, current_cio_time); - - check_cleanup_active_fds(current_cio_time, - timeout_time != ERTS_POLL_NO_TIMEOUT); - #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_check_exact(NULL, 0); /* No locks should be locked */ #endif - pollres_len = erts_atomic32_read_dirty(&pollset.active_fd.no) + ERTS_CHECK_IO_POLL_RES_LEN; + pollres_len = psi->pollres_len; - pollres = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollResFd)*pollres_len); +#if ERTS_POLL_USE_FALLBACK + if (psi->ps == get_fallback()) { - erts_atomic_set_nob(&pollset.in_poll_wait, 1); + poll_ret = erts_poll_wait_flbk(psi->ps, psi->pollres, &pollres_len); - poll_ret = ERTS_CIO_POLL_WAIT(pollset.ps, pollres, &pollres_len, timeout_time); + } else +#endif + { + poll_ret = erts_poll_wait(psi->ps, psi->pollres, &pollres_len); + } #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_check_exact(NULL, 0); /* No locks should be locked */ #endif -#ifdef ERTS_BREAK_REQUESTED - if (ERTS_BREAK_REQUESTED) - erts_do_break_handling(); -#endif - - -#ifdef ERTS_SIGNAL_STATE /* ifndef ERTS_SMP */ - if (ERTS_SIGNAL_STATE) { - erts_handle_signal_state(); - } -#endif - - if (poll_ret != 0) { - erts_atomic_set_nob(&pollset.in_poll_wait, 0); - forget_removed(&pollset); - erts_free(ERTS_ALC_T_TMP, pollres); + if (poll_ret == EAGAIN) { goto restart; } @@ -2212,65 +1568,86 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) erl_errno_id(poll_ret), poll_ret); erts_send_error_to_logger_nogl(dsbufp); } + ERTS_MSACC_POP_STATE(); return; } for (i = 0; i < pollres_len; i++) { - ErtsSysFdType fd = (ErtsSysFdType) pollres[i].fd; + erts_driver_t* drv_ptr = NULL; + ErtsResource* resource = NULL; + ErtsDrvSelectDataState *free_select = NULL; + ErtsNifSelectDataState *free_nif = NULL; + ErtsSysFdType fd = (ErtsSysFdType) ERTS_POLL_RES_GET_FD(&psi->pollres[i]); ErtsDrvEventState *state; + ErtsPollEvents revents; erts_mtx_lock(fd_mtx(fd)); -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - state = &drv_ev_state[ (int) fd]; -#else - state = hash_get_drv_ev_state(fd); + state = get_drv_ev_state(fd); + if (!state) { - goto next_pollres; + erts_mtx_unlock(fd_mtx(fd)); + continue; } -#endif - /* Skip this fd if it was removed from pollset */ - if (is_removed(state)) { - goto next_pollres; - } + revents = ERTS_POLL_RES_GET_EVTS(&psi->pollres[i]); + + DEBUG_PRINT_FD("triggered %s", state, ev2str(revents)); + + if (revents & ERTS_POLL_EV_ERR) { + /* + * Handle error events by triggering all in/out events + * that has been selected on. + * We *do not* want to call a callback that corresponds + * to an event not selected. + */ + revents = state->active_events; + state->active_events = 0; + } else { + + /* Disregard any events that are not active at the moment, + for instance this could happen if the driver/nif does + select/deselect in rapid succession. */ + revents &= state->active_events | ERTS_POLL_EV_NVAL; + state->active_events &= ~revents; + + /* Reactivate the poll op if there are still active events */ + if (state->active_events) { + ErtsPollEvents new_events; + DEBUG_PRINT_FD("re-enable %s", state, ev2str(state->active_events)); + + new_events = erts_io_control(state, ERTS_POLL_OP_MOD, state->active_events); + + /* Unable to re-enable the fd, signal all callbacks */ + if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) { + revents |= state->active_events; + state->active_events = 0; + } + } + } switch (state->type) { case ERTS_EV_TYPE_DRV_SEL: { /* Requested via driver_select()... */ - ErtsPollEvents revents = pollres[i].events; - - if (revents & ERTS_POLL_EV_ERR) { - /* - * Handle error events by triggering all in/out events - * that the driver has selected. - * We *do not* want to call a callback that corresponds - * to an event not selected. - */ - revents = state->events; - } - else { - revents &= (state->events | ERTS_POLL_EV_NVAL); - } if (revents & (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) { if (revents & ERTS_POLL_EV_OUT) { - oready(state->driver.select->outport, state, current_cio_time); + oready(state->driver.select->outport, state); } /* Someone might have deselected input since revents was read (true also on the non-smp emulator since oready() may have been called); therefore, update revents... */ - revents &= state->events; + revents &= state->events; if (revents & ERTS_POLL_EV_IN) { - iready(state->driver.select->inport, state, current_cio_time); + iready(state->driver.select->inport, state); } } else if (revents & ERTS_POLL_EV_NVAL) { bad_fd_in_pollset(state, state->driver.select->inport, state->driver.select->outport); - add_active_fd(state->fd); + check_fd_cleanup(state, &free_select, &free_nif); } break; } @@ -2279,108 +1656,116 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) struct erts_nif_select_event in = {NIL}; struct erts_nif_select_event out = {NIL}; ErtsResource* resource = NULL; - ErtsPollEvents revents = pollres[i].events; - - if (revents & ERTS_POLL_EV_ERR) { - /* - * Handle error events by triggering all in/out events - * that the NIF has selected. - * We *do not* want to send a message that corresponds - * to an event not selected. - */ - revents = state->events; - } - else { - revents &= (state->events | ERTS_POLL_EV_NVAL); - } if (revents & (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) { if (revents & ERTS_POLL_EV_OUT) { if (is_not_nil(state->driver.nif->out.pid)) { out = state->driver.nif->out; resource = state->driver.stop.resource; - state->driver.nif->out.ddeselect_cnt = ERTS_NIF_DELAYED_DESELECT; state->driver.nif->out.pid = NIL; - add_active_fd(state->fd); - } - else { - ASSERT(state->driver.nif->out.ddeselect_cnt >= 2); - state->driver.nif->out.ddeselect_cnt--; } } if (revents & ERTS_POLL_EV_IN) { if (is_not_nil(state->driver.nif->in.pid)) { in = state->driver.nif->in; resource = state->driver.stop.resource; - state->driver.nif->in.ddeselect_cnt = ERTS_NIF_DELAYED_DESELECT; state->driver.nif->in.pid = NIL; - add_active_fd(state->fd); - } - else { - ASSERT(state->driver.nif->in.ddeselect_cnt >= 2); - state->driver.nif->in.ddeselect_cnt--; } } + state->events &= ~revents; } else if (revents & ERTS_POLL_EV_NVAL) { bad_fd_in_pollset(state, NIL, NIL); - add_active_fd(state->fd); + check_fd_cleanup(state, &free_select, &free_nif); } erts_mtx_unlock(fd_mtx(fd)); + if (is_not_nil(in.pid)) { send_event_tuple(&in, resource, am_ready_input); } if (is_not_nil(out.pid)) { send_event_tuple(&out, resource, am_ready_output); } - goto next_pollres_unlocked; + continue; } -#if ERTS_CIO_HAVE_DRV_EVENT - case ERTS_EV_TYPE_DRV_EV: { /* Requested via driver_event()... */ - ErlDrvEventData event_data; - ErtsPollEvents revents; - ASSERT(state->driver.event); - ASSERT(state->driver.event->data); - event_data = state->driver.event->data; - revents = pollres[i].events; - revents &= ~state->driver.event->removed_events; - - if (revents) { - event_data->events = state->events; - event_data->revents = revents; - eready(state->driver.event->port, state, event_data, current_cio_time); - } - break; - } -#endif + case ERTS_EV_TYPE_STOP_NIF: { + resource = state->driver.stop.resource; + state->type = ERTS_EV_TYPE_NONE; + goto case_ERTS_EV_TYPE_NONE; + } + case ERTS_EV_TYPE_STOP_USE: { +#if ERTS_POLL_USE_FALLBACK + ASSERT(psi->ps == get_fallback()); +#endif + drv_ptr = state->driver.stop.drv_ptr; + state->type = ERTS_EV_TYPE_NONE; + /* fallthrough */ case ERTS_EV_TYPE_NONE: /* Deselected ... */ + case_ERTS_EV_TYPE_NONE: + ASSERT(!state->events && !state->active_events && !state->flags); + check_fd_cleanup(state, &free_select, &free_nif); break; + } default: { /* Error */ erts_dsprintf_buf_t *dsbufp; dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Invalid event request type for fd in erts_poll()! " - "fd=%d, event request type=%sd\n", (int) state->fd, + "fd=%d, event request type=%d\n", (int) state->fd, (int) state->type); ASSERT(0); deselect(state, 0); - add_active_fd(state->fd); break; } } - next_pollres:; erts_mtx_unlock(fd_mtx(fd)); - next_pollres_unlocked:; - } - erts_atomic_set_nob(&pollset.in_poll_wait, 0); - erts_free(ERTS_ALC_T_TMP, pollres); - forget_removed(&pollset); + if (drv_ptr) { + int was_unmasked = erts_block_fpe(); + DTRACE1(driver_stop_select, drv_ptr->name); + LTTNG1(driver_stop_select, drv_ptr->name); + (*drv_ptr->stop_select)((ErlDrvEvent) fd, NULL); + erts_unblock_fpe(was_unmasked); + if (drv_ptr->handle) { + erts_ddll_dereference_driver(drv_ptr->handle); + } + } + if (resource) { + erts_resource_stop(resource, (ErlNifEvent)fd, 1); + enif_release_resource(resource->data); + } + if (free_select) + free_drv_select_data(free_select); + if (free_nif) + free_nif_select_data(free_nif); + } + + /* The entire pollres array was filled with events, + * grow it for the next call. We do this for two reasons: + * 1. Pulling out more events in on go will increase throughput + * 2. If the polling implementation is not fair, this will make + * sure that we get all fds that we can. i.e. if 12 fds are + * constantly active, but we only have a pollres_len of 10, + * two of the fds may never be triggered depending on what the + * kernel decides to do. + **/ + if (pollres_len == psi->pollres_len) { + int ev_state_len = drv_ev_state_len(); + erts_free(ERTS_ALC_T_POLLSET, psi->pollres); + psi->pollres_len *= 2; + /* Never grow it larger than the current drv_ev_state.len size */ + if (psi->pollres_len > ev_state_len) + psi->pollres_len = ev_state_len; + psi->pollres = erts_alloc(ERTS_ALC_T_POLLSET, + sizeof(ErtsPollResFd) * psi->pollres_len); + } + + ERTS_MSACC_POP_STATE(); } static void @@ -2474,16 +1859,16 @@ static int drv_ev_state_cmp(void *des1, void *des2) static void *drv_ev_state_alloc(void *des_tmpl) { ErtsDrvEventState *evstate; - erts_spin_lock(&state_prealloc_lock); - if (state_prealloc_first == NULL) { - erts_spin_unlock(&state_prealloc_lock); + erts_spin_lock(&drv_ev_state.prealloc_lock); + if (drv_ev_state.prealloc_first == NULL) { + erts_spin_unlock(&drv_ev_state.prealloc_lock); evstate = (ErtsDrvEventState *) erts_alloc(ERTS_ALC_T_DRV_EV_STATE, sizeof(ErtsDrvEventState)); } else { - evstate = state_prealloc_first; - state_prealloc_first = (ErtsDrvEventState *) evstate->hb.next; - --num_state_prealloc; - erts_spin_unlock(&state_prealloc_lock); + evstate = drv_ev_state.prealloc_first; + drv_ev_state.prealloc_first = (ErtsDrvEventState *) evstate->hb.next; + --drv_ev_state.num_prealloc; + erts_spin_unlock(&drv_ev_state.prealloc_lock); } /* XXX: Already valid data if prealloced, could ignore template! */ *evstate = *((ErtsDrvEventState *) des_tmpl); @@ -2493,58 +1878,200 @@ static void *drv_ev_state_alloc(void *des_tmpl) static void drv_ev_state_free(void *des) { - erts_spin_lock(&state_prealloc_lock); - ((ErtsDrvEventState *) des)->hb.next = &state_prealloc_first->hb; - state_prealloc_first = (ErtsDrvEventState *) des; - ++num_state_prealloc; - erts_spin_unlock(&state_prealloc_lock); + erts_spin_lock(&drv_ev_state.prealloc_lock); + ((ErtsDrvEventState *) des)->hb.next = &drv_ev_state.prealloc_first->hb; + drv_ev_state.prealloc_first = (ErtsDrvEventState *) des; + ++drv_ev_state.num_prealloc; + erts_spin_unlock(&drv_ev_state.prealloc_lock); } #endif +#define ERTS_MAX_NO_OF_POLL_THREADS ERTS_MAX_NO_OF_SCHEDULERS + +static char * +get_arg(char* rest, char** argv, int* ip) +{ + int i = *ip; + if (*rest == '\0') { + if (argv[i+1] == NULL) { + erts_fprintf(stderr, "too few arguments\n"); + erts_usage(); + } + argv[i++] = NULL; + rest = argv[i]; + } + argv[i] = NULL; + *ip = i; + return rest; +} + +static void +parse_args(int *argc, char **argv, int concurrent_waiters) +{ + int i = 0, j; + int no_pollsets = 0, no_poll_threads = 0, + no_pollsets_percentage = 0, + no_poll_threads_percentage = 0; + ASSERT(argc && argv); + while (i < *argc) { + if(argv[i][0] == '-') { + switch (argv[i][1]) { + case 'I': { + if (strncmp(argv[i]+2, "Ot", 2) == 0) { + char *arg = get_arg(argv[i]+4, argv, &i); + if (sscanf(arg, "%d", &no_poll_threads) != 1 || + no_poll_threads < 1 || + ERTS_MAX_NO_OF_POLL_THREADS < no_poll_threads) { + erts_fprintf(stderr,"bad I/O poll threads number: %s\n", arg); + erts_usage(); + } + } else if (strncmp(argv[i]+2, "Op", 3) == 0) { + char *arg = get_arg(argv[i]+4, argv, &i); + if (sscanf(arg, "%d", &no_pollsets) != 1 || + no_pollsets < 1) { + erts_fprintf(stderr,"bad I/O pollset number: %s\n", arg); + erts_usage(); + } + } else if (strncmp(argv[i]+2, "OPt", 4) == 0) { + char *arg = get_arg(argv[i]+5, argv, &i); + if (sscanf(arg, "%d", &no_poll_threads_percentage) != 1 || + no_poll_threads_percentage < 0 || + no_poll_threads_percentage > 100) { + erts_fprintf(stderr,"bad I/O poll thread percentage number: %s\n", arg); + erts_usage(); + } + } else if (strncmp(argv[i]+2, "OPp", 4) == 0) { + char *arg = get_arg(argv[i]+5, argv, &i); + if (sscanf(arg, "%d", &no_pollsets_percentage) != 1 || + no_pollsets_percentage < 0 || + no_pollsets_percentage > 100) { + erts_fprintf(stderr,"bad I/O pollset percentage number: %s\n", arg); + erts_usage(); + } + } else { + break; + } + break; + } + case 'K': + (void)get_arg(argv[i]+2, argv, &i); + break; + case '-': + goto args_parsed; + default: + break; + } + } + i++; + } + +args_parsed: + + if (!concurrent_waiters) { + no_pollsets = no_poll_threads; + no_pollsets_percentage = 100; + } + + if (no_poll_threads == 0) { + if (no_poll_threads_percentage == 0) + no_poll_threads = 1; /* This is the default */ + else { + no_poll_threads = erts_no_schedulers * no_poll_threads_percentage / 100; + if (no_poll_threads < 1) + no_poll_threads = 1; + } + } + + if (no_pollsets == 0) { + if (no_pollsets_percentage == 0) + no_pollsets = 1; /* This is the default */ + else { + no_pollsets = no_poll_threads * no_pollsets_percentage / 100; + if (no_pollsets < 1) + no_pollsets = 1; + } + } + + if (no_poll_threads < no_pollsets) { + erts_fprintf(stderr, + "number of IO poll threads has to be greater or equal to " + "the number of \nIO pollsets. Current values are set to: \n" + " -IOt %d -IOp %d\n", + no_poll_threads, no_pollsets); + erts_usage(); + } + + /* Handled arguments have been marked with NULL. Slide arguments + not handled towards the beginning of argv. */ + for (i = 0, j = 0; i < *argc; i++) { + if (argv[i]) + argv[j++] = argv[i]; + } + *argc = j; + + erts_no_pollsets = no_pollsets; + erts_no_poll_threads = no_poll_threads; +} + void -ERTS_CIO_EXPORT(erts_init_check_io)(void) +erts_init_check_io(int *argc, char **argv) { + int j, concurrent_waiters, no_poll_threads; ERTS_CT_ASSERT((INT_MIN & (ERL_NIF_SELECT_STOP_CALLED | ERL_NIF_SELECT_STOP_SCHEDULED | ERL_NIF_SELECT_INVALID_EVENT | ERL_NIF_SELECT_FAILED)) == 0); - erts_atomic_init_nob(&erts_check_io_time, 0); - erts_atomic_init_nob(&pollset.in_poll_wait, 0); - - ERTS_CIO_POLL_INIT(); - pollset.ps = ERTS_CIO_NEW_POLLSET(); - - pollset.active_fd.six = 0; - pollset.active_fd.eix = 0; - erts_atomic32_init_nob(&pollset.active_fd.no, 0); - pollset.active_fd.size = ERTS_ACTIVE_FD_INC; - pollset.active_fd.array = erts_alloc(ERTS_ALC_T_ACTIVE_FD_ARR, - sizeof(ErtsSysFdType)*ERTS_ACTIVE_FD_INC); -#ifdef DEBUG - { - int i; - for (i = 0; i < ERTS_ACTIVE_FD_INC; i++) - pollset.active_fd.array[i] = ERTS_SYS_FD_INVALID; - } + + + erts_poll_init(&concurrent_waiters); +#if ERTS_POLL_USE_FALLBACK + erts_poll_init_flbk(NULL); #endif + parse_args(argc, argv, concurrent_waiters); - init_removed_fd_alloc(); - pollset.removed_list = NULL; - erts_spinlock_init(&pollset.removed_list_lock, "pollset_rm_list", NIL, - ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); - { - int i; - for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { - erts_mtx_init(&drv_ev_state_locks[i].lck, "drv_ev_state", make_small(i), - ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); - } + /* Create the actual pollsets */ + pollsetv = erts_alloc(ERTS_ALC_T_POLLSET,sizeof(ErtsPollSet *) * erts_no_pollsets); + + for (j=0; j < erts_no_pollsets; j++) + pollsetv[j] = erts_poll_create_pollset(j); + +#if ERTS_POLL_USE_FALLBACK + flbk_pollset = erts_poll_create_pollset_flbk(-1); +#endif + + no_poll_threads = erts_no_poll_threads; +#if ERTS_POLL_USE_FALLBACK + no_poll_threads++; +#endif + + psiv = erts_alloc(ERTS_ALC_T_POLLSET, sizeof(ErtsPollThread) * no_poll_threads); + +#if ERTS_POLL_USE_FALLBACK + psiv[0].pollres_len = ERTS_CHECK_IO_POLL_RES_LEN; + psiv[0].pollres = erts_alloc(ERTS_ALC_T_POLLSET, + sizeof(ErtsPollResFd) * ERTS_CHECK_IO_POLL_RES_LEN); + psiv[0].ps = get_fallback(); + psiv++; +#endif + + for (j = 0; j < erts_no_poll_threads; j++) { + psiv[j].pollres_len = ERTS_CHECK_IO_POLL_RES_LEN; + psiv[j].pollres = erts_alloc(ERTS_ALC_T_POLLSET, + sizeof(ErtsPollResFd) * ERTS_CHECK_IO_POLL_RES_LEN); + psiv[j].ps = pollsetv[j % erts_no_pollsets]; } + + for (j=0; j < ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT; j++) { + erts_mtx_init(&drv_ev_state.locks[j].lck, "drv_ev_state", make_small(j), + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); + } + #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - max_fds = ERTS_CIO_POLL_MAX_FDS(); - erts_atomic_init_nob(&drv_ev_state_len, 0); - drv_ev_state = NULL; - erts_mtx_init(&drv_ev_state_grow_lock, "drv_ev_state_grow", NIL, + drv_ev_state.max_fds = erts_poll_max_fds(); + erts_atomic_init_nob(&drv_ev_state.len, 0); + drv_ev_state.v = NULL; + erts_mtx_init(&drv_ev_state.grow_lock, "drv_ev_state_grow", NIL, ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); #else { @@ -2553,153 +2080,182 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void) hf.cmp = &drv_ev_state_cmp; hf.alloc = &drv_ev_state_alloc; hf.free = &drv_ev_state_free; - num_state_prealloc = 0; - state_prealloc_first = NULL; - erts_spinlock_init(&state_prealloc_lock,"state_prealloc", NIL, - ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); - - safe_hash_init(ERTS_ALC_T_DRV_EV_STATE, &drv_ev_state_tab, "drv_ev_state_tab", + drv_ev_state.num_prealloc = 0; + drv_ev_state.prealloc_first = NULL; + erts_spinlock_init(&drv_ev_state.prealloc_lock, "state_prealloc", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); + safe_hash_init(ERTS_ALC_T_DRV_EV_STATE, &drv_ev_state.tab, "drv_ev_state_tab", ERTS_LOCK_FLAGS_CATEGORY_IO, DRV_EV_STATE_HTAB_SIZE, hf); } #endif } int -ERTS_CIO_EXPORT(erts_check_io_max_files)(void) +erts_check_io_max_files(void) { #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - return max_fds; + return drv_ev_state.max_fds; #else - return ERTS_POLL_EXPORT(erts_poll_max_fds)(); + return erts_poll_max_fds(); #endif } Uint -ERTS_CIO_EXPORT(erts_check_io_size)(void) +erts_check_io_size(void) { - Uint res; + Uint res = 0; ErtsPollInfo pi; - ERTS_CIO_POLL_INFO(pollset.ps, &pi); - res = pi.memory_size; + int i; + +#if ERTS_POLL_USE_FALLBACK + erts_poll_info(get_fallback(), &pi); + res += pi.memory_size; +#endif + + for (i = 0; i < erts_no_pollsets; i++) { + erts_poll_info(pollsetv[i], &pi); + res += pi.memory_size; + } #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - res += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state_len); + res += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state.len); #else - res += safe_hash_table_sz(&drv_ev_state_tab); + res += safe_hash_table_sz(&drv_ev_state.tab); { SafeHashInfo hi; - safe_hash_get_info(&hi, &drv_ev_state_tab); + safe_hash_get_info(&hi, &drv_ev_state.tab); res += hi.objs * sizeof(ErtsDrvEventState); } - erts_spin_lock(&state_prealloc_lock); - res += num_state_prealloc * sizeof(ErtsDrvEventState); - erts_spin_unlock(&state_prealloc_lock); + erts_spin_lock(&drv_ev_state.prealloc_lock); + res += drv_ev_state.num_prealloc * sizeof(ErtsDrvEventState); + erts_spin_unlock(&drv_ev_state.prealloc_lock); #endif return res; } Eterm -ERTS_CIO_EXPORT(erts_check_io_info)(void *proc) +erts_check_io_info(void *proc) { Process *p = (Process *) proc; - Eterm tags[16], values[16], res; - Uint sz, *szp, *hp, **hpp, memory_size; - Sint i; - ErtsPollInfo pi; - erts_aint_t cio_time = erts_atomic_read_acqb(&erts_check_io_time); - int active_fds = (int) erts_atomic32_read_acqb(&pollset.active_fd.no); + Eterm tags[16], values[16], res, list = NIL; + Uint sz, *szp, *hp, **hpp; + ErtsPollInfo *piv; + Sint i, j = 0, len; + int no_pollsets = erts_no_pollsets + ERTS_POLL_USE_FALLBACK; + ERTS_CT_ASSERT(ERTS_POLL_USE_FALLBACK == 0 || ERTS_POLL_USE_FALLBACK == 1); - while (1) { - erts_aint_t post_cio_time; - int post_active_fds; + piv = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollInfo) * no_pollsets); - ERTS_CIO_POLL_INFO(pollset.ps, &pi); +#if ERTS_POLL_USE_FALLBACK + erts_poll_info_flbk(get_fallback(), &piv[0]); + piv[0].poll_threads = 1; + piv[0].active_fds = 0; + piv++; +#endif - post_cio_time = erts_atomic_read_mb(&erts_check_io_time); - post_active_fds = (int) erts_atomic32_read_acqb(&pollset.active_fd.no); - if (cio_time == post_cio_time && active_fds == post_active_fds) - break; - cio_time = post_cio_time; - active_fds = post_active_fds; + for (j = 0; j < erts_no_pollsets; j++) { + erts_poll_info(pollsetv[j], &piv[j]); + piv[j].active_fds = 0; + piv[j].poll_threads = erts_no_poll_threads / erts_no_pollsets; + if (erts_no_poll_threads % erts_no_pollsets > j) + piv[j].poll_threads++; } - memory_size = pi.memory_size; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - memory_size += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state_len); + i = 0; + erts_mtx_lock(&drv_ev_state.grow_lock); + len = erts_atomic_read_nob(&drv_ev_state.len); + for (i = 0; i < ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT; i++) { + erts_mtx_lock(&drv_ev_state.locks[i].lck); + for (j = i; j < len; j+=ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT) { + ErtsDrvEventState *state = get_drv_ev_state(j); + int pollsetid = get_pollset_id(j); + ASSERT(fd_mtx(j) == &drv_ev_state.locks[i].lck); + if (state->flags & ERTS_EV_FLAG_FALLBACK) + pollsetid = -1; + if (state->driver.select + && (state->type == ERTS_EV_TYPE_DRV_SEL) + && (is_iotask_active(&state->driver.select->iniotask) + || is_iotask_active(&state->driver.select->outiotask))) + piv[pollsetid].active_fds++; + } + erts_mtx_unlock(&drv_ev_state.locks[i].lck); + } + erts_mtx_unlock(&drv_ev_state.grow_lock); + + piv[0].memory_size += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state.len); #else - memory_size += safe_hash_table_sz(&drv_ev_state_tab); + piv[0].memory_size += safe_hash_table_sz(&drv_ev_state.tab); { - SafeHashInfo hi; - safe_hash_get_info(&hi, &drv_ev_state_tab); - memory_size += hi.objs * sizeof(ErtsDrvEventState); + SafeHashInfo hi; + safe_hash_get_info(&hi, &drv_ev_state.tab); + piv[0].memory_size += hi.objs * sizeof(ErtsDrvEventState); } - erts_spin_lock(&state_prealloc_lock); - memory_size += num_state_prealloc * sizeof(ErtsDrvEventState); - erts_spin_unlock(&state_prealloc_lock); + erts_spin_lock(&drv_ev_state.prealloc_lock); + piv[0].memory_size += drv_ev_state.num_prealloc * sizeof(ErtsDrvEventState); + erts_spin_unlock(&drv_ev_state.prealloc_lock); #endif hpp = NULL; szp = &sz; sz = 0; - bld_it: - i = 0; + piv -= ERTS_POLL_USE_FALLBACK; - tags[i] = erts_bld_atom(hpp, szp, "name"); - values[i++] = erts_bld_atom(hpp, szp, "erts_poll"); + bld_it: - tags[i] = erts_bld_atom(hpp, szp, "primary"); - values[i++] = erts_bld_atom(hpp, szp, pi.primary); + for (j = no_pollsets-1; j >= 0; j--) { + i = 0; - tags[i] = erts_bld_atom(hpp, szp, "fallback"); - values[i++] = erts_bld_atom(hpp, szp, pi.fallback ? pi.fallback : "false"); + tags[i] = erts_bld_atom(hpp, szp, "name"); + values[i++] = erts_bld_atom(hpp, szp, "erts_poll"); - tags[i] = erts_bld_atom(hpp, szp, "kernel_poll"); - values[i++] = erts_bld_atom(hpp, szp, - pi.kernel_poll ? pi.kernel_poll : "false"); + tags[i] = erts_bld_atom(hpp, szp, "primary"); + values[i++] = erts_bld_atom(hpp, szp, piv[j].primary); - tags[i] = erts_bld_atom(hpp, szp, "memory_size"); - values[i++] = erts_bld_uint(hpp, szp, memory_size); + tags[i] = erts_bld_atom(hpp, szp, "kernel_poll"); + values[i++] = erts_bld_atom(hpp, szp, + piv[j].kernel_poll ? piv[j].kernel_poll : "false"); - tags[i] = erts_bld_atom(hpp, szp, "total_poll_set_size"); - values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.poll_set_size); + tags[i] = erts_bld_atom(hpp, szp, "memory_size"); + values[i++] = erts_bld_uint(hpp, szp, piv[j].memory_size); - if (pi.fallback) { - tags[i] = erts_bld_atom(hpp, szp, "fallback_poll_set_size"); - values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.fallback_poll_set_size); - } + tags[i] = erts_bld_atom(hpp, szp, "total_poll_set_size"); + values[i++] = erts_bld_uint(hpp, szp, piv[j].poll_set_size); - tags[i] = erts_bld_atom(hpp, szp, "lazy_updates"); - values[i++] = pi.lazy_updates ? am_true : am_false; + tags[i] = erts_bld_atom(hpp, szp, "lazy_updates"); + values[i++] = piv[j].lazy_updates ? am_true : am_false; - if (pi.lazy_updates) { - tags[i] = erts_bld_atom(hpp, szp, "pending_updates"); - values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.pending_updates); - } + tags[i] = erts_bld_atom(hpp, szp, "pending_updates"); + values[i++] = erts_bld_uint(hpp, szp, piv[j].pending_updates); - tags[i] = erts_bld_atom(hpp, szp, "batch_updates"); - values[i++] = pi.batch_updates ? am_true : am_false; + tags[i] = erts_bld_atom(hpp, szp, "batch_updates"); + values[i++] = piv[j].batch_updates ? am_true : am_false; - tags[i] = erts_bld_atom(hpp, szp, "concurrent_updates"); - values[i++] = pi.concurrent_updates ? am_true : am_false; + tags[i] = erts_bld_atom(hpp, szp, "concurrent_updates"); + values[i++] = piv[j].concurrent_updates ? am_true : am_false; - tags[i] = erts_bld_atom(hpp, szp, "max_fds"); - values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.max_fds); + tags[i] = erts_bld_atom(hpp, szp, "fallback"); + values[i++] = piv[j].is_fallback ? am_true : am_false; - tags[i] = erts_bld_atom(hpp, szp, "active_fds"); - values[i++] = erts_bld_uint(hpp, szp, (Uint) active_fds); + tags[i] = erts_bld_atom(hpp, szp, "max_fds"); + values[i++] = erts_bld_uint(hpp, szp, piv[j].max_fds); -#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS - tags[i] = erts_bld_atom(hpp, szp, "no_avoided_wakeups"); - values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.no_avoided_wakeups); + tags[i] = erts_bld_atom(hpp, szp, "active_fds"); + values[i++] = erts_bld_uint(hpp, szp, piv[j].active_fds); - tags[i] = erts_bld_atom(hpp, szp, "no_avoided_interrupts"); - values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.no_avoided_interrupts); + tags[i] = erts_bld_atom(hpp, szp, "poll_threads"); + values[i++] = erts_bld_uint(hpp, szp, piv[j].poll_threads); - tags[i] = erts_bld_atom(hpp, szp, "no_interrupt_timed"); - values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.no_interrupt_timed); -#endif + res = erts_bld_2tup_list(hpp, szp, i, tags, values); - res = erts_bld_2tup_list(hpp, szp, i, tags, values); + if (!hpp) { + *szp += 2; + } + else { + list = CONS(*hpp, res, list); + *hpp += 2; + } + } if (!hpp) { hp = HAlloc(p, sz); @@ -2708,386 +2264,442 @@ ERTS_CIO_EXPORT(erts_check_io_info)(void *proc) goto bld_it; } - return res; + erts_free(ERTS_ALC_T_TMP, piv); + + return list; } static ERTS_INLINE ErtsPollEvents -print_events(ErtsPollEvents ev) +print_events(erts_dsprintf_buf_t *dsbufp, ErtsPollEvents ev) { int first = 1; + if(ev == ERTS_POLL_EV_NONE) { + erts_dsprintf(dsbufp, "N/A"); + return 0; + } if(ev & ERTS_POLL_EV_IN) { ev &= ~ERTS_POLL_EV_IN; - erts_printf("%s%s", first ? "" : "|", "IN"); + erts_dsprintf(dsbufp, "%s%s", first ? "" : "|", "IN"); first = 0; } if(ev & ERTS_POLL_EV_OUT) { ev &= ~ERTS_POLL_EV_OUT; - erts_printf("%s%s", first ? "" : "|", "OUT"); + erts_dsprintf(dsbufp, "%s%s", first ? "" : "|", "OUT"); first = 0; } /* The following should not appear... */ if(ev & ERTS_POLL_EV_NVAL) { - erts_printf("%s%s", first ? "" : "|", "NVAL"); + erts_dsprintf(dsbufp, "%s%s", first ? "" : "|", "NVAL"); first = 0; } if(ev & ERTS_POLL_EV_ERR) { - erts_printf("%s%s", first ? "" : "|", "ERR"); + erts_dsprintf(dsbufp, "%s%s", first ? "" : "|", "ERR"); first = 0; } if (ev) - erts_printf("%s0x%b32x", first ? "" : "|", (Uint32) ev); + erts_dsprintf(dsbufp, "%s0x%b32x", first ? "" : "|", (Uint32) ev); return ev; } +static ERTS_INLINE void +print_flags(erts_dsprintf_buf_t *dsbufp, EventStateFlags f) +{ + const char* delim = ""; + if(f & ERTS_EV_FLAG_USED) { + erts_dsprintf(dsbufp, "%s","USED"); + delim = "|"; + } + if(f & ERTS_EV_FLAG_FALLBACK) { + erts_dsprintf(dsbufp, "%s%s", delim, "FLBK"); + delim = "|"; + } +} + +#ifdef DEBUG_PRINT_MODE + +static ERTS_INLINE char * +drvmode2str(int mode) { + switch (mode) { + case ERL_DRV_READ|ERL_DRV_USE: return "READ|USE"; + case ERL_DRV_WRITE|ERL_DRV_USE: return "WRITE|USE"; + case ERL_DRV_READ|ERL_DRV_WRITE|ERL_DRV_USE: return "READ|WRITE|USE"; + case ERL_DRV_USE: return "USE"; + case ERL_DRV_READ: return "READ"; + case ERL_DRV_WRITE: return "WRITE"; + case ERL_DRV_READ|ERL_DRV_WRITE: return "READ|WRITE"; + default: return "UNKNOWN"; + } +} + +static ERTS_INLINE char * +nifmode2str(enum ErlNifSelectFlags mode) { + switch (mode) { + case ERL_NIF_SELECT_READ: return "READ"; + case ERL_NIF_SELECT_WRITE: return "WRITE"; + case ERL_NIF_SELECT_STOP: return "STOP"; + default: return "UNKNOWN"; + } +} + +#endif + typedef struct { int used_fds; int num_errors; int no_driver_select_structs; - int no_driver_event_structs; + int no_enif_select_structs; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS int internal_fds; ErtsPollEvents *epep; #endif } IterDebugCounters; -static void doit_erts_check_io_debug(void *vstate, void *vcounters) +static int erts_debug_print_checkio_state(erts_dsprintf_buf_t *dsbufp, + ErtsDrvEventState *state, + ErtsPollEvents ep_events, + int internal) { - ErtsDrvEventState *state = (ErtsDrvEventState *) vstate; - IterDebugCounters *counters = (IterDebugCounters *) vcounters; - ErtsPollEvents cio_events = state->events; - ErtsSysFdType fd = state->fd; -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - int internal = 0; - ErtsPollEvents ep_events = counters->epep[(int) fd]; -#endif - int err = 0; - #if defined(HAVE_FSTAT) && !defined(NO_FSTAT_ON_SYS_FD_TYPE) struct stat stat_buf; #endif - - if (state->driver.select) - counters->no_driver_select_structs++; -#if ERTS_CIO_HAVE_DRV_EVENT - if (state->driver.event) - counters->no_driver_event_structs++; -#endif - + ErtsSysFdType fd = state->fd; + ErtsPollEvents cio_events = state->events; + int err = 0; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if (state->events || ep_events) { - if (ep_events & ERTS_POLL_EV_NVAL) { - ep_events &= ~ERTS_POLL_EV_NVAL; - internal = 1; - counters->internal_fds++; - } - else - counters->used_fds++; -#else - if (state->events) { - counters->used_fds++; + ErtsPollEvents aio_events = state->active_events; #endif - - erts_printf("fd=%d ", (int) fd); - + erts_dsprintf(dsbufp, "pollset=%d fd=%d ", + state->flags & ERTS_EV_FLAG_FALLBACK ? -1 : get_pollset_id(fd), (int) fd); + #if defined(HAVE_FSTAT) && !defined(NO_FSTAT_ON_SYS_FD_TYPE) - if (fstat((int) fd, &stat_buf) < 0) - erts_printf("type=unknown "); - else { - erts_printf("type="); + if (fstat((int) fd, &stat_buf) < 0) + erts_dsprintf(dsbufp, "type=unknown "); + else { + erts_dsprintf(dsbufp, "type="); #ifdef S_ISSOCK - if (S_ISSOCK(stat_buf.st_mode)) - erts_printf("sock "); - else + if (S_ISSOCK(stat_buf.st_mode)) + erts_dsprintf(dsbufp, "sock "); + else #endif #ifdef S_ISFIFO if (S_ISFIFO(stat_buf.st_mode)) - erts_printf("fifo "); + erts_dsprintf(dsbufp, "fifo "); else #endif #ifdef S_ISCHR - if (S_ISCHR(stat_buf.st_mode)) - erts_printf("chr "); - else + if (S_ISCHR(stat_buf.st_mode)) + erts_dsprintf(dsbufp, "chr "); + else #endif #ifdef S_ISDIR - if (S_ISDIR(stat_buf.st_mode)) - erts_printf("dir "); - else + if (S_ISDIR(stat_buf.st_mode)) + erts_dsprintf(dsbufp, "dir "); + else #endif #ifdef S_ISBLK - if (S_ISBLK(stat_buf.st_mode)) - erts_printf("blk "); - else + if (S_ISBLK(stat_buf.st_mode)) + erts_dsprintf(dsbufp, "blk "); + else #endif #ifdef S_ISREG - if (S_ISREG(stat_buf.st_mode)) - erts_printf("reg "); - else + if (S_ISREG(stat_buf.st_mode)) + erts_dsprintf(dsbufp, "reg "); + else #endif #ifdef S_ISLNK - if (S_ISLNK(stat_buf.st_mode)) - erts_printf("lnk "); - else + if (S_ISLNK(stat_buf.st_mode)) + erts_dsprintf(dsbufp, "lnk "); + else #endif #ifdef S_ISDOOR - if (S_ISDOOR(stat_buf.st_mode)) - erts_printf("door "); - else + if (S_ISDOOR(stat_buf.st_mode)) + erts_dsprintf(dsbufp, "door "); + else #endif #ifdef S_ISWHT - if (S_ISWHT(stat_buf.st_mode)) - erts_printf("wht "); - else + if (S_ISWHT(stat_buf.st_mode)) + erts_dsprintf(dsbufp, "wht "); + else #endif #ifdef S_ISXATTR - if (S_ISXATTR(stat_buf.st_mode)) - erts_printf("xattr "); - else + if (S_ISXATTR(stat_buf.st_mode)) + erts_dsprintf(dsbufp, "xattr "); + else #endif - erts_printf("unknown "); - } + erts_dsprintf(dsbufp, "unknown "); + } #else - erts_printf("type=unknown "); + erts_dsprintf(dsbufp, "type=unknown "); #endif - if (state->type == ERTS_EV_TYPE_DRV_SEL) { - erts_printf("driver_select "); - -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if (internal) { - erts_printf("internal "); - err = 1; - } - - if (cio_events == ep_events) { - erts_printf("ev="); - if (print_events(cio_events) != 0) - err = 1; - } - else { - err = 1; - erts_printf("cio_ev="); - print_events(cio_events); - erts_printf(" ep_ev="); - print_events(ep_events); - } -#else - if (print_events(cio_events) != 0) - err = 1; -#endif - erts_printf(" "); - if (cio_events & ERTS_POLL_EV_IN) { - Eterm id = state->driver.select->inport; - if (is_nil(id)) { - erts_printf("inport=none inname=none indrv=none "); - err = 1; - } - else { - ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT); - erts_printf(" inport=%T inname=%s indrv=%s ", - id, - pnp->name ? pnp->name : "unknown", - (pnp->driver_name - ? pnp->driver_name - : "unknown")); - erts_free_port_names(pnp); - } - } - if (cio_events & ERTS_POLL_EV_OUT) { - Eterm id = state->driver.select->outport; - if (is_nil(id)) { - erts_printf("outport=none outname=none outdrv=none "); - err = 1; - } - else { - ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT); - erts_printf(" outport=%T outname=%s outdrv=%s ", - id, - pnp->name ? pnp->name : "unknown", - (pnp->driver_name - ? pnp->driver_name - : "unknown")); - erts_free_port_names(pnp); - } - } - } - else if (state->type == ERTS_EV_TYPE_NIF) { - ErtsResource* r; - erts_printf("enif_select "); + if (state->type == ERTS_EV_TYPE_DRV_SEL) { + erts_dsprintf(dsbufp, "driver_select "); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if (internal) { - erts_printf("internal "); - err = 1; - } - + if (internal) { + erts_dsprintf(dsbufp, "internal "); + err = 1; + } + if (aio_events == cio_events) { if (cio_events == ep_events) { - erts_printf("ev="); - if (print_events(cio_events) != 0) + erts_dsprintf(dsbufp, "ev="); + if (print_events(dsbufp, cio_events) != 0) err = 1; } else { + ErtsPollEvents ev = cio_events; + if (ev != ep_events && ep_events != ERTS_POLL_EV_NONE) + err = 1; + erts_dsprintf(dsbufp, "cio_ev="); + print_events(dsbufp, cio_events); + erts_dsprintf(dsbufp, " ep_ev="); + print_events(dsbufp, ep_events); + } + } else { + erts_dsprintf(dsbufp, "cio_ev="); + print_events(dsbufp, cio_events); + erts_dsprintf(dsbufp, " aio_ev="); + print_events(dsbufp, aio_events); + if ((aio_events != ep_events && ep_events != ERTS_POLL_EV_NONE) || + (aio_events != 0 && ep_events == ERTS_POLL_EV_NONE)) { + erts_dsprintf(dsbufp, " ep_ev="); + print_events(dsbufp, ep_events); err = 1; - erts_printf("cio_ev="); - print_events(cio_events); - erts_printf(" ep_ev="); - print_events(ep_events); } + } #else - if (print_events(cio_events) != 0) + if (print_events(dsbufp, cio_events) != 0) + err = 1; +#endif + erts_dsprintf(dsbufp, " "); + if (cio_events & ERTS_POLL_EV_IN) { + Eterm id = state->driver.select->inport; + if (is_nil(id)) { + erts_dsprintf(dsbufp, "inport=none inname=none indrv=none "); err = 1; -#endif - erts_printf(" inpid=%T dd_cnt=%b32d", state->driver.nif->in.pid, - state->driver.nif->in.ddeselect_cnt); - erts_printf(" outpid=%T dd_cnt=%b32d", state->driver.nif->out.pid, - state->driver.nif->out.ddeselect_cnt); - r = state->driver.stop.resource; - erts_printf(" resource=%p(%T:%T)", r, r->type->module, r->type->name); + } + else { + ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT); + erts_dsprintf(dsbufp, " inport=%T inname=%s indrv=%s ", + id, + pnp->name ? pnp->name : "unknown", + (pnp->driver_name + ? pnp->driver_name + : "unknown")); + erts_free_port_names(pnp); + } } -#if ERTS_CIO_HAVE_DRV_EVENT - else if (state->type == ERTS_EV_TYPE_DRV_EV) { - Eterm id; - erts_printf("driver_event "); + if (cio_events & ERTS_POLL_EV_OUT) { + Eterm id = state->driver.select->outport; + if (is_nil(id)) { + erts_dsprintf(dsbufp, "outport=none outname=none outdrv=none "); + err = 1; + } + else { + ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT); + erts_dsprintf(dsbufp, " outport=%T outname=%s outdrv=%s ", + id, + pnp->name ? pnp->name : "unknown", + (pnp->driver_name + ? pnp->driver_name + : "unknown")); + erts_free_port_names(pnp); + } + } + } + else if (state->type == ERTS_EV_TYPE_NIF) { + ErtsResource* r; + erts_dsprintf(dsbufp, "enif_select "); + #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if (internal) { - erts_printf("internal "); - err = 1; - } - if (cio_events == ep_events) { - erts_printf("ev=0x%b32x", (Uint32) cio_events); - } - else { - err = 1; - erts_printf("cio_ev=0x%b32x", (Uint32) cio_events); - erts_printf(" ep_ev=0x%b32x", (Uint32) ep_events); - } + if (internal) { + erts_dsprintf(dsbufp, "internal "); + err = 1; + } + + if (cio_events == ep_events) { + erts_dsprintf(dsbufp, "ev="); + if (print_events(dsbufp, cio_events) != 0) + err = 1; + } + else { + err = 1; + erts_dsprintf(dsbufp, "cio_ev="); + print_events(dsbufp, cio_events); + erts_dsprintf(dsbufp, " ep_ev="); + print_events(dsbufp, ep_events); + } #else - erts_printf("ev=0x%b32x", (Uint32) cio_events); + if (print_events(dsbufp, cio_events) != 0) + err = 1; #endif - id = state->driver.event->port; - if (is_nil(id)) { - erts_printf(" port=none name=none drv=none "); - err = 1; - } - else { - ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT); - erts_printf(" port=%T name=%s drv=%s ", - id, - pnp->name ? pnp->name : "unknown", - (pnp->driver_name - ? pnp->driver_name - : "unknown")); - erts_free_port_names(pnp); - } - } + erts_dsprintf(dsbufp, " inpid=%T", state->driver.nif->in.pid); + erts_dsprintf(dsbufp, " outpid=%T", state->driver.nif->out.pid); + r = state->driver.stop.resource; + erts_dsprintf(dsbufp, " resource=%p(%T:%T)", r, r->type->module, r->type->name); + } +#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS + else if (internal) { + erts_dsprintf(dsbufp, "internal "); + if (cio_events) { + err = 1; + erts_dsprintf(dsbufp, "cio_ev="); + print_events(dsbufp, cio_events); + } + if (ep_events) { + erts_dsprintf(dsbufp, "ep_ev="); + print_events(dsbufp, ep_events); + } + } #endif + else { + err = 1; + erts_dsprintf(dsbufp, "control_type=%d ", (int)state->type); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - else if (internal) { - erts_printf("internal "); - if (cio_events) { - err = 1; - erts_printf("cio_ev="); - print_events(cio_events); - } - if (ep_events) { - erts_printf("ep_ev="); - print_events(ep_events); - } - } + if (cio_events == ep_events) { + erts_dsprintf(dsbufp, "ev="); + print_events(dsbufp, cio_events); + } + else { + erts_dsprintf(dsbufp, "cio_ev="); print_events(dsbufp, cio_events); + erts_dsprintf(dsbufp, " ep_ev="); print_events(dsbufp, ep_events); + } +#else + erts_dsprintf(dsbufp, "ev=0x%b32x", (Uint32) cio_events); #endif - else { - err = 1; - erts_printf("control_type=%d ", (int)state->type); + } + + erts_dsprintf(dsbufp, " flags="); print_flags(dsbufp, state->flags); + if (err) { + erts_dsprintf(dsbufp, " ERROR"); + } + erts_dsprintf(dsbufp, "\r\n"); + return err; +} + +static void doit_erts_check_io_debug(void *vstate, void *vcounters, + erts_dsprintf_buf_t *dsbufp) +{ + ErtsDrvEventState *state = (ErtsDrvEventState *) vstate; + IterDebugCounters *counters = (IterDebugCounters *) vcounters; + int internal = 0; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if (cio_events == ep_events) { - erts_printf("ev="); - print_events(cio_events); - } - else { - erts_printf("cio_ev="); print_events(cio_events); - erts_printf(" ep_ev="); print_events(ep_events); - } + ErtsSysFdType fd = state->fd; + ErtsPollEvents ep_events = counters->epep[(int) fd]; #else - erts_printf("ev=0x%b32x", (Uint32) cio_events); + ErtsPollEvents ep_events = ERTS_POLL_EV_NONE; #endif + + if (state->driver.select) { + counters->no_driver_select_structs++; + ASSERT(state->events || (ep_events != 0 && ep_events != ERTS_POLL_EV_NONE)); + } + if (state->driver.nif) { + counters->no_enif_select_structs++; + ASSERT(state->events || (ep_events != 0 && ep_events != ERTS_POLL_EV_NONE)); + } + +#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS + if (state->events || (ep_events != 0 && ep_events != ERTS_POLL_EV_NONE)) { + if (ep_events & ERTS_POLL_EV_NVAL) { + ep_events &= ~ERTS_POLL_EV_NVAL; + internal = 1; + counters->internal_fds++; } - - if (err) { + else + counters->used_fds++; +#else + if (state->events) { + counters->used_fds++; +#endif + if (erts_debug_print_checkio_state(dsbufp, state, ep_events, internal)) { counters->num_errors++; - erts_printf(" ERROR"); } - erts_printf("\n"); } } - + +/* ciodpi can be NULL when called from etp-commands */ int -ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip) +erts_check_io_debug(ErtsCheckIoDebugInfo *ciodip) { + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - int fd, len; + int fd, len, i; #endif - IterDebugCounters counters; + IterDebugCounters counters = {0}; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS ErtsDrvEventState null_des; null_des.driver.select = NULL; -#if ERTS_CIO_HAVE_DRV_EVENT - null_des.driver.event = NULL; -#endif + null_des.driver.nif = NULL; null_des.driver.stop.drv_ptr = NULL; null_des.events = 0; - null_des.remove_cnt = 0; null_des.type = ERTS_EV_TYPE_NONE; + null_des.flags = 0; + + counters.epep = erts_alloc(ERTS_ALC_T_TMP, + sizeof(ErtsPollEvents)*drv_ev_state.max_fds); #endif - erts_printf("--- fds in pollset --------------------------------------\n"); #if defined(ERTS_ENABLE_LOCK_CHECK) erts_lc_check_exact(NULL, 0); /* No locks should be locked */ #endif - erts_thr_progress_block(); /* stop the world to avoid messy locking */ + if (ciodip) + erts_thr_progress_block(); /* stop the world to avoid messy locking */ #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - counters.epep = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollEvents)*max_fds); - ERTS_POLL_EXPORT(erts_poll_get_selected_events)(pollset.ps, counters.epep, max_fds); - counters.internal_fds = 0; -#endif - counters.used_fds = 0; - counters.num_errors = 0; - counters.no_driver_select_structs = 0; - counters.no_driver_event_structs = 0; + len = erts_atomic_read_nob(&drv_ev_state.len); -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - len = erts_atomic_read_nob(&drv_ev_state_len); +#if ERTS_POLL_USE_FALLBACK + erts_dsprintf(dsbufp, "--- fds in flbk pollset ---------------------------------\n"); + erts_poll_get_selected_events_flbk(get_fallback(), counters.epep, + drv_ev_state.max_fds); for (fd = 0; fd < len; fd++) { - doit_erts_check_io_debug((void *) &drv_ev_state[fd], (void *) &counters); + if (drv_ev_state.v[fd].flags & ERTS_EV_FLAG_FALLBACK) + doit_erts_check_io_debug(&drv_ev_state.v[fd], &counters, dsbufp); + } +#endif + erts_dsprintf(dsbufp, "--- fds in pollset --------------------------------------\n"); + + for (i = 0; i < erts_no_pollsets; i++) { + erts_poll_get_selected_events(pollsetv[i], + counters.epep, + drv_ev_state.max_fds); + for (fd = 0; fd < len; fd++) { + if (!(drv_ev_state.v[fd].flags & ERTS_EV_FLAG_FALLBACK) + && get_pollset_id(fd) == i) + doit_erts_check_io_debug(&drv_ev_state.v[fd], &counters, dsbufp); + } } - for ( ; fd < max_fds; fd++) { - null_des.fd = fd; - doit_erts_check_io_debug((void *) &null_des, (void *) &counters); + for (fd = len ; fd < drv_ev_state.max_fds; fd++) { + null_des.fd = fd; + doit_erts_check_io_debug(&null_des, &counters, dsbufp); } #else - safe_hash_for_each(&drv_ev_state_tab, &doit_erts_check_io_debug, (void *) &counters); + safe_hash_for_each(&drv_ev_state.tab, &doit_erts_check_io_debug, + &counters, dsbufp); #endif - erts_thr_progress_unblock(); + if (ciodip) + erts_thr_progress_unblock(); - ciodip->no_used_fds = counters.used_fds; - ciodip->no_driver_select_structs = counters.no_driver_select_structs; - ciodip->no_driver_event_structs = counters.no_driver_event_structs; + if (ciodip) { + ciodip->no_used_fds = counters.used_fds; + ciodip->no_driver_select_structs = counters.no_driver_select_structs; + ciodip->no_enif_select_structs = counters.no_enif_select_structs; + } - erts_printf("\n"); - erts_printf("used fds=%d\n", counters.used_fds); - erts_printf("Number of driver_select() structures=%d\n", counters.no_driver_select_structs); -#if ERTS_CIO_HAVE_DRV_EVENT - erts_printf("Number of driver_event() structures=%d\n", counters.no_driver_event_structs); -#endif + erts_dsprintf(dsbufp, "\n"); + erts_dsprintf(dsbufp, "used fds=%d\n", counters.used_fds); + erts_dsprintf(dsbufp, "Number of driver_select() structures=%d\n", counters.no_driver_select_structs); + erts_dsprintf(dsbufp, "Number of enif_select() structures=%d\n", counters.no_enif_select_structs); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - erts_printf("internal fds=%d\n", counters.internal_fds); + erts_dsprintf(dsbufp, "internal fds=%d\n", counters.internal_fds); #endif - erts_printf("---------------------------------------------------------\n"); - fflush(stdout); + erts_dsprintf(dsbufp, "---------------------------------------------------------\n"); + erts_send_error_to_logger_nogl(dsbufp); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS erts_free(ERTS_ALC_T_TMP, (void *) counters.epep); #endif @@ -3096,11 +2708,19 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip) } #ifdef ERTS_ENABLE_LOCK_COUNT -void ERTS_CIO_EXPORT(erts_lcnt_update_cio_locks)(int enable) { +void erts_lcnt_update_cio_locks(int enable) { + int i; #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS - erts_lcnt_enable_hash_lock_count(&drv_ev_state_tab, ERTS_LOCK_FLAGS_CATEGORY_IO, enable); + erts_lcnt_enable_hash_lock_count(&drv_ev_state.tab, ERTS_LOCK_FLAGS_CATEGORY_IO, enable); #else (void)enable; #endif + +#if ERTS_POLL_USE_FALLBACK + erts_lcnt_enable_pollset_lock_count_flbk(get_fallback(), enable); +#endif + + for (i = 0; i < erts_no_pollsets; i++) + erts_lcnt_enable_pollset_lock_count(pollsetv[i], enable); } #endif /* ERTS_ENABLE_LOCK_COUNT */ diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h index 777942a473..443ef1264c 100644 --- a/erts/emulator/sys/common/erl_check_io.h +++ b/erts/emulator/sys/common/erl_check_io.h @@ -18,10 +18,11 @@ * %CopyrightEnd% */ -/* - * Description: Check I/O +/** + * @description Check I/O, a cross platform IO polling framework for ERTS * - * Author: Rickard Green + * @author Rickard Green + * @author Lukas Larsson */ #ifndef ERL_CHECK_IO_H__ @@ -30,72 +31,79 @@ #include "sys.h" #include "erl_sys_driver.h" -#ifdef ERTS_ENABLE_KERNEL_POLL - -int driver_select_kp(ErlDrvPort, ErlDrvEvent, int, int); -int driver_select_nkp(ErlDrvPort, ErlDrvEvent, int, int); -int enif_select_kp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm); -int enif_select_nkp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm); -int driver_event_kp(ErlDrvPort, ErlDrvEvent, ErlDrvEventData); -int driver_event_nkp(ErlDrvPort, ErlDrvEvent, ErlDrvEventData); -Uint erts_check_io_size_kp(void); -Uint erts_check_io_size_nkp(void); -Eterm erts_check_io_info_kp(void *); -Eterm erts_check_io_info_nkp(void *); -int erts_check_io_max_files_kp(void); -int erts_check_io_max_files_nkp(void); -void erts_check_io_interrupt_kp(int); -void erts_check_io_interrupt_nkp(int); -void erts_check_io_interrupt_timed_kp(int, ErtsMonotonicTime); -void erts_check_io_interrupt_timed_nkp(int, ErtsMonotonicTime); -void erts_check_io_kp(int); -void erts_check_io_nkp(int); -void erts_init_check_io_kp(void); -void erts_init_check_io_nkp(void); -int erts_check_io_debug_kp(ErtsCheckIoDebugInfo *); -int erts_check_io_debug_nkp(ErtsCheckIoDebugInfo *); - -#ifdef ERTS_ENABLE_LOCK_COUNT -void erts_lcnt_update_cio_locks_kp(int enable); -void erts_lcnt_update_cio_locks_nkp(int enable); -#endif - -#else /* !ERTS_ENABLE_KERNEL_POLL */ +/** @brief a structure that is used by each polling thread */ +struct erts_poll_thread; +/** + * Get the memory size of the check io framework + */ Uint erts_check_io_size(void); -Eterm erts_check_io_info(void *); +/** + * Returns an Eterm with information about all the pollsets active at the + * moment. + * + * @param proc the Process* to allocate the result on. It is passed as + * void * because of header include problems. + */ +Eterm erts_check_io_info(void *proc); +/** + * Should be called when a port IO task has been executed in order to re-enable + * or clear the information about the fd. + * + * @param type The type of event that has been completed. + * @param handle The port task handle of the event. + * @param reset A function pointer to be called when the port task handle + * should be reset. + */ +void erts_io_notify_port_task_executed(ErtsPortTaskType type, + ErtsPortTaskHandle *handle, + void (*reset)(ErtsPortTaskHandle *)); +/** + * Returns the maximum number of fds that the check io framework can handle. + */ int erts_check_io_max_files(void); -void erts_check_io_interrupt(int); -void erts_check_io_interrupt_timed(int, ErtsMonotonicTime); -void erts_check_io(int); -void erts_init_check_io(void); - +/** + * Called by any thread that should check for new IO events. This function will + * not return unless erts_check_io_interrupt(pt, 1) is called by another thread. + * + * @param pt the poll thread structure to use. + */ +void erts_check_io(struct erts_poll_thread *pt); +/** + * Initialize the check io framework. This function will parse the arguments + * and delete any entries that it is interested in. + * + * @param argc the number of arguments + * @param argv an array with the arguments + */ +void erts_init_check_io(int *argc, char **argv); +/** + * Interrupt the poll thread so that it can execute other code. + * + * Should be called with set = 0 by the waiting thread before calling + * erts_check_io. + * + * @param pt the poll thread to wake + * @param set whether to set or clear the interrupt flag + */ +void erts_check_io_interrupt(struct erts_poll_thread *pt, int set); +/** + * Create a new poll thread structure that is associated with the number no. + * It is the callers responsibility that no is unique. + */ +struct erts_poll_thread* erts_create_pollset_thread(int no); #ifdef ERTS_ENABLE_LOCK_COUNT +/** + * Toggle lock counting on all check io locks + */ void erts_lcnt_update_cio_locks(int enable); #endif -#endif - -extern erts_atomic_t erts_check_io_time; - typedef struct { ErtsPortTaskHandle task; - erts_atomic_t executed_time; + ErtsSysFdType fd; } ErtsIoTask; -ERTS_GLB_INLINE void erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -ERTS_GLB_INLINE void -erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp) -{ - ErtsIoTask *itp = (ErtsIoTask *) (((char *) pthp) - offsetof(ErtsIoTask, task)); - erts_aint_t ci_time = erts_atomic_read_acqb(&erts_check_io_time); - erts_atomic_set_relb(&itp->executed_time, ci_time); -} - -#endif #endif /* ERL_CHECK_IO_H__ */ @@ -103,6 +111,16 @@ erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp) #define ERL_CHECK_IO_INTERNAL__ #endif +#define ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT 128 + +/* Controls how many pollsets to allocate. Fd's are hashed into + each pollset based on the FD. When doing non-concurrent updates + there will be one pollset per thread. +*/ +extern int erts_no_pollsets; +extern int erts_no_poll_threads; + + #ifndef ERL_CHECK_IO_INTERNAL__ #define ERL_CHECK_IO_INTERNAL__ #include "erl_poll.h" @@ -115,24 +133,8 @@ erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp) */ # define ERTS_CIO_DEFER_ACTIVE_EVENTS 1 #else -# define ERTS_CIO_DEFER_ACTIVE_EVENTS 0 -#endif - -/* - * ErtsDrvEventDataState is used by driver_event() which is almost never - * used. We allocate ErtsDrvEventDataState separate since we dont wan't - * the size of ErtsDrvEventState to increase due to driver_event() - * information. - */ -typedef struct { - Eterm port; - ErlDrvEventData data; - ErtsPollEvents removed_events; -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - ErtsPollEvents deferred_events; +# define ERTS_CIO_DEFER_ACTIVE_EVENTS 1 #endif - ErtsIoTask iotask; -} ErtsDrvEventDataState; typedef struct { Eterm inport; diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c index 7d26839b0f..30a595c17a 100644 --- a/erts/emulator/sys/common/erl_poll.c +++ b/erts/emulator/sys/common/erl_poll.c @@ -18,9 +18,8 @@ * %CopyrightEnd% */ -/* - * Description: Poll interface suitable for ERTS with or without - * SMP support. +/** + * @description Poll interface suitable for ERTS * * The interface is currently implemented using: * - select @@ -29,12 +28,36 @@ * - epoll with poll or select as fallback * - kqueue with poll or select as fallback * - * Some time in the future it will also be - * implemented using Solaris ports. * + * @author Rickard Green + * @author Lukas Larsson + * + * There are two major different implementations off IO polling in this + * file. The concurrent and non-concurrent implementations. + * When available epoll/kqueue are used to implement the concurrent + * versions. poll, select and dev/poll use non-concurrent updates. + * + * Concurrent version: + * In the concurrent version erts_poll_control directly modifies + * the kernel pollset without waking the thread that is waiting + * on events. Also the ErtsPollResFd type is directly mapped to + * the native event type, so no extra copying is needed. Note that + * as no locking at all is done, fds can be triggered that have been + * removed from the pollset. The check_io layer has to deal with this. + * + * Non-concurrent version: + * In the non-concurrent version, the pollset has an internal representation + * of the pollset that is updated by erts_poll_control. When an fd is updated, + * its number is placed in the update request queue and then the waiting thread + * is woken in order to see the change. The internal data in the pollset is + * protected by a mutex that has to be taken by both the modifying and waiting + * thread at different times. * + * The non-concurrent pollset cannot have fd's closed in it while a thread is + * waiting on that fd. In order to fix this, when an ERTS_POLL_OP_DEL command + * is issued, the fd is marked as closing and the waiting thread is woken. The + * fd is then returned in the waiting threads results as ERTS_POLL_EV_NONE. * - * Author: Rickard Green */ #ifdef HAVE_CONFIG_H @@ -62,6 +85,8 @@ # ifdef SYS_SELECT_H # include <sys/select.h> # endif +#elif defined(_DARWIN_UNLIMITED_SELECT) +# undef _DARWIN_UNLIMITED_SELECT #endif #ifdef NO_SYSCONF # if ERTS_POLL_USE_SELECT @@ -83,20 +108,35 @@ #error "Missing implementation of erts_poll()" #endif -#if defined(ERTS_KERNEL_POLL_VERSION) && !ERTS_POLL_USE_KERNEL_POLL -#error "Missing kernel poll implementation of erts_poll()" -#endif +#if 0 +#define ERTS_POLL_DEBUG_PRINT 1 + +#define DEBUG_PRINT(FMT, PS, ...) \ + do { \ + int myerrno = errno; \ + erts_printf("%d: " FMT "\r\n", (PS)->id, ##__VA_ARGS__); \ + errno = myerrno; \ + } while(0) -#if defined(ERTS_NO_KERNEL_POLL_VERSION) && ERTS_POLL_USE_KERNEL_POLL -#error "Kernel poll used when it shouldn't be used" +/* Define to print info about modifications done to each fd */ +#define DEBUG_PRINT_FD(FMT, PS, FD, ...) DEBUG_PRINT("%d: " FMT, PS, FD, ##__VA_ARGS__) +/* Define to print entry and exit from erts_poll_wait (can be very spammy) */ +//#define DEBUG_PRINT_WAIT(FMT, PS, ...) DEBUG_PRINT(FMT, PS, ##__VA_ARGS__) + +#else +#define ERTS_POLL_DEBUG_PRINT 0 +#define DEBUG_PRINT(...) #endif -#if 0 -#define ERTS_POLL_DEBUG_PRINT +#ifndef DEBUG_PRINT_FD +#define DEBUG_PRINT_FD(...) +#endif +#ifndef DEBUG_PRINT_WAIT +#define DEBUG_PRINT_WAIT(...) #endif -#ifdef _DARWIN_UNLIMITED_SELECT +#if defined(_DARWIN_UNLIMITED_SELECT) && ERTS_POLL_USE_SELECT typedef struct { size_t sz; fd_set* ptr; @@ -142,25 +182,13 @@ int ERTS_SELECT(int nfds, ERTS_fd_set *readfds, ERTS_fd_set *writefds, # define ERTS_SELECT select #endif -#define ERTS_POLL_USE_BATCH_UPDATE_POLLSET (ERTS_POLL_USE_DEVPOLL \ - || ERTS_POLL_USE_KQUEUE) +#define ERTS_POLL_IS_FALLBACK (ERTS_POLL_USE_POLL || ERTS_POLL_USE_SELECT) && ERTS_ENABLE_KERNEL_POLL -#define ERTS_POLL_USE_CONCURRENT_UPDATE ERTS_POLL_USE_EPOLL +#define ERTS_POLL_USE_CONCURRENT_UPDATE (ERTS_POLL_USE_EPOLL || ERTS_POLL_USE_KQUEUE) -#define ERTS_POLL_COALESCE_KP_RES (ERTS_POLL_USE_KQUEUE || ERTS_POLL_USE_EPOLL) - -#define ERTS_POLLSET_LOCK(PS) \ - erts_mtx_lock(&(PS)->mtx) -#define ERTS_POLLSET_UNLOCK(PS) \ - erts_mtx_unlock(&(PS)->mtx) - -#define ERTS_POLLSET_SET_POLLED_CHK(PS) \ - ((int) erts_atomic32_xchg_nob(&(PS)->polled, (erts_aint32_t) 1)) -#define ERTS_POLLSET_UNSET_POLLED(PS) \ - erts_atomic32_set_nob(&(PS)->polled, (erts_aint32_t) 0) -#define ERTS_POLLSET_IS_POLLED(PS) \ - ((int) erts_atomic32_read_nob(&(PS)->polled)) +#define ERTS_POLL_USE_WAKEUP_PIPE (!ERTS_POLL_USE_CONCURRENT_UPDATE) +#if !ERTS_POLL_USE_CONCURRENT_UPDATE #define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) \ erts_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 1) @@ -169,17 +197,28 @@ int ERTS_SELECT(int nfds, ERTS_fd_set *readfds, ERTS_fd_set *writefds, #define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) \ ((int) erts_atomic32_read_nob(&(PS)->have_update_requests)) -#if ERTS_POLL_USE_FALLBACK -# if ERTS_POLL_USE_POLL -# define ERTS_POLL_NEED_FALLBACK(PS) ((PS)->no_poll_fds > 1) -# elif ERTS_POLL_USE_SELECT -# define ERTS_POLL_NEED_FALLBACK(PS) ((PS)->no_select_fds > 1) -# endif +#define ERTS_POLLSET_LOCK(PS) \ + erts_mtx_lock(&(PS)->mtx) +#define ERTS_POLLSET_UNLOCK(PS) \ + erts_mtx_unlock(&(PS)->mtx) + +#else + +#define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) +#define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS) +#define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) 0 + +#define ERTS_POLLSET_LOCK(PS) +#define ERTS_POLLSET_UNLOCK(PS) + #endif + /* * --- Data types ------------------------------------------------------------ */ +#if !ERTS_POLL_USE_CONCURRENT_UPDATE + #define ERTS_POLLSET_UPDATE_REQ_BLOCK_SIZE 128 typedef struct ErtsPollSetUpdateRequestsBlock_ ErtsPollSetUpdateRequestsBlock; @@ -189,237 +228,200 @@ struct ErtsPollSetUpdateRequestsBlock_ { int fds[ERTS_POLLSET_UPDATE_REQ_BLOCK_SIZE]; }; - - # define ERTS_POLL_FD_FLG_INURQ (((unsigned short) 1) << 0) -#if ERTS_POLL_USE_FALLBACK -# define ERTS_POLL_FD_FLG_INFLBCK (((unsigned short) 1) << 1) -# define ERTS_POLL_FD_FLG_USEFLBCK (((unsigned short) 1) << 2) -#endif -# define ERTS_POLL_FD_FLG_RST (((unsigned short) 1) << 3) +# define ERTS_POLL_FD_FLG_RST (((unsigned short) 1) << 1) + typedef struct { #if ERTS_POLL_USE_POLL int pix; #endif + ErtsPollEvents used_events; ErtsPollEvents events; -#if ERTS_POLL_COALESCE_KP_RES - unsigned short res_ev_ix; -#endif unsigned short flags; } ErtsFdStatus; - -#if ERTS_POLL_COALESCE_KP_RES -/* res_ev_ix max value */ -#define ERTS_POLL_MAX_RES ((1 << sizeof(unsigned short)*8) - 1) #endif -#if ERTS_POLL_USE_KQUEUE - -#define ERTS_POLL_KQ_OP_HANDLED 1 -#define ERTS_POLL_KQ_OP_DEL_R 2 -#define ERTS_POLL_KQ_OP_DEL_W 3 -#define ERTS_POLL_KQ_OP_ADD_R 4 -#define ERTS_POLL_KQ_OP_ADD_W 5 -#define ERTS_POLL_KQ_OP_ADD2_R 6 -#define ERTS_POLL_KQ_OP_ADD2_W 7 - -#endif - -struct ErtsPollSet_ { - ErtsPollSet next; +/* + * This struct is not really exported, but it's nice to + * get unique names in debugger for kp/nkp + */ +struct ERTS_POLL_EXPORT(erts_pollset) { + int id; int internal_fd_limit; - ErtsFdStatus *fds_status; erts_atomic_t no_of_user_fds; - int fds_status_len; + #if ERTS_POLL_USE_KERNEL_POLL int kp_fd; - int res_events_len; -#if ERTS_POLL_USE_EPOLL - struct epoll_event *res_events; -#elif ERTS_POLL_USE_KQUEUE - struct kevent *res_events; -#elif ERTS_POLL_USE_DEVPOLL - struct pollfd *res_events; -#endif #endif /* ERTS_POLL_USE_KERNEL_POLL */ + #if ERTS_POLL_USE_POLL int next_poll_fds_ix; int no_poll_fds; int poll_fds_len; - struct pollfd*poll_fds; + struct pollfd *poll_fds; #elif ERTS_POLL_USE_SELECT int next_sel_fd; int max_fd; -#if ERTS_POLL_USE_FALLBACK - int no_select_fds; -#endif ERTS_fd_set input_fds; ERTS_fd_set res_input_fds; ERTS_fd_set output_fds; ERTS_fd_set res_output_fds; +#elif ERTS_POLL_USE_DEVPOLL + struct pollfd *poll_fds; + int poll_fds_ix; #endif + +#if !ERTS_POLL_USE_CONCURRENT_UPDATE + ErtsFdStatus *fds_status; + int fds_status_len; ErtsPollSetUpdateRequestsBlock update_requests; ErtsPollSetUpdateRequestsBlock *curr_upd_req_block; erts_atomic32_t have_update_requests; - erts_atomic32_t polled; erts_mtx_t mtx; - int wake_fds[2]; -#if ERTS_POLL_USE_TIMERFD - int timer_fd; -#endif -#if ERTS_POLL_USE_FALLBACK - int fallback_used; -#endif erts_atomic32_t wakeup_state; - erts_atomic64_t timeout_time; -#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS - erts_atomic_t no_avoided_wakeups; - erts_atomic_t no_avoided_interrupts; - erts_atomic_t no_interrupt_timed; +#endif + +#if ERTS_POLL_USE_WAKEUP_PIPE + int wake_fds[2]; #endif }; void erts_silence_warn_unused_result(long unused); static void fatal_error(char *format, ...); -static void fatal_error_async_signal_safe(char *error_str); static int max_fds = -1; -static ErtsPollSet pollsets; -static erts_mtx_t pollsets_lock; #if ERTS_POLL_USE_POLL +#if !ERTS_POLL_IS_FALLBACK +static ERTS_INLINE short ev2pollev(ErtsPollEvents ev) +{ + return ERTS_POLL_EV_E2N(ev); +} + +static ERTS_INLINE ErtsPollEvents pollev2ev(short ev) +{ + return ERTS_POLL_EV_N2E(ev); +} + +#else /* ERTS_POLL_IS_FALLBACK */ + static ERTS_INLINE short ev2pollev(ErtsPollEvents ev) { -#if !ERTS_POLL_USE_FALLBACK || ERTS_POLL_USE_KQUEUE - return ERTS_POLL_EV_E2N(ev); -#else /* Note, we only map events we are interested in */ short res_ev = (short) 0; if (ev & ERTS_POLL_EV_IN) - res_ev |= ERTS_POLL_EV_NKP_IN; + res_ev |= ERTS_POLL_EV_NKP_IN; if (ev & ERTS_POLL_EV_OUT) - res_ev |= ERTS_POLL_EV_NKP_OUT; + res_ev |= ERTS_POLL_EV_NKP_OUT; return res_ev; -#endif } static ERTS_INLINE ErtsPollEvents pollev2ev(short ev) { -#if !ERTS_POLL_USE_FALLBACK || ERTS_POLL_USE_KQUEUE - return ERTS_POLL_EV_N2E(ev); -#else /* Note, we only map events we are interested in */ ErtsPollEvents res_ev = (ErtsPollEvents) 0; if (ev & ERTS_POLL_EV_NKP_IN) - res_ev |= ERTS_POLL_EV_IN; + res_ev |= ERTS_POLL_EV_IN; if (ev & ERTS_POLL_EV_NKP_OUT) - res_ev |= ERTS_POLL_EV_OUT; + res_ev |= ERTS_POLL_EV_OUT; if (ev & ERTS_POLL_EV_NKP_ERR) - res_ev |= ERTS_POLL_EV_ERR; + res_ev |= ERTS_POLL_EV_ERR; if (ev & ERTS_POLL_EV_NKP_NVAL) - res_ev |= ERTS_POLL_EV_NVAL; - return res_ev; -#endif + res_ev |= ERTS_POLL_EV_NVAL; + return res_ev; } -#endif +#endif /* !ERTS_POLL_IS_FALLBACK */ + +#endif /* ERTS_POLL_USE_POLL */ + #ifdef HARD_DEBUG static void check_poll_result(ErtsPollResFd pr[], int len); -#if ERTS_POLL_USE_DEVPOLL -static void check_poll_status(ErtsPollSet ps); -#endif /* ERTS_POLL_USE_DEVPOLL */ #endif /* HARD_DEBUG */ -#ifdef ERTS_POLL_DEBUG_PRINT +#if ERTS_POLL_USE_DEVPOLL && defined(DEBUG) +static void check_poll_status(ErtsPollSet *ps); +#endif /* ERTS_POLL_USE_DEVPOLL && DEBUG */ static void print_misc_debug_info(void); +#if ERTS_POLL_USE_EPOLL +uint32_t epoll_events(int kp_fd, int fd); #endif -static ERTS_INLINE void -init_timeout_time(ErtsPollSet ps) -{ - erts_atomic64_init_nob(&ps->timeout_time, - (erts_aint64_t) ERTS_MONOTONIC_TIME_MAX); -} - -static ERTS_INLINE void -set_timeout_time(ErtsPollSet ps, ErtsMonotonicTime time) -{ - erts_atomic64_set_relb(&ps->timeout_time, - (erts_aint64_t) time); -} - -static ERTS_INLINE ErtsMonotonicTime -get_timeout_time(ErtsPollSet ps) -{ - return (ErtsMonotonicTime) erts_atomic64_read_acqb(&ps->timeout_time); -} #define ERTS_POLL_NOT_WOKEN 0 #define ERTS_POLL_WOKEN -1 #define ERTS_POLL_WOKEN_INTR 1 +#if !ERTS_POLL_USE_CONCURRENT_UPDATE static ERTS_INLINE void -reset_wakeup_state(ErtsPollSet ps) +reset_wakeup_state(ErtsPollSet *ps) { erts_atomic32_set_mb(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); } +#endif static ERTS_INLINE int -is_woken(ErtsPollSet ps) +is_woken(ErtsPollSet *ps) { +#if !ERTS_POLL_USE_CONCURRENT_UPDATE return erts_atomic32_read_acqb(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN; +#else + return 0; +#endif } static ERTS_INLINE int -is_interrupted_reset(ErtsPollSet ps) +is_interrupted_reset(ErtsPollSet *ps) { +#if !ERTS_POLL_USE_CONCURRENT_UPDATE return (erts_atomic32_xchg_acqb(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN) == ERTS_POLL_WOKEN_INTR); +#else + return 0; +#endif } static ERTS_INLINE void -woke_up(ErtsPollSet ps) +woke_up(ErtsPollSet *ps) { +#if !ERTS_POLL_USE_CONCURRENT_UPDATE erts_aint32_t wakeup_state = erts_atomic32_read_acqb(&ps->wakeup_state); if (wakeup_state == ERTS_POLL_NOT_WOKEN) (void) erts_atomic32_cmpxchg_nob(&ps->wakeup_state, ERTS_POLL_WOKEN, ERTS_POLL_NOT_WOKEN); ASSERT(erts_atomic32_read_nob(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN); +#endif } /* * --- Wakeup pipe ----------------------------------------------------------- */ +#if ERTS_POLL_USE_WAKEUP_PIPE static ERTS_INLINE void -wake_poller(ErtsPollSet ps, int interrupted, int async_signal_safe) +wake_poller(ErtsPollSet *ps, int interrupted) { +#if !ERTS_POLL_USE_CONCURRENT_UPDATE int wake; - if (async_signal_safe) - wake = 1; - else { - erts_aint32_t wakeup_state; - if (!interrupted) - wakeup_state = erts_atomic32_cmpxchg_relb(&ps->wakeup_state, - ERTS_POLL_WOKEN, - ERTS_POLL_NOT_WOKEN); - else - wakeup_state = erts_atomic32_xchg_relb(&ps->wakeup_state, - ERTS_POLL_WOKEN_INTR); - wake = wakeup_state == ERTS_POLL_NOT_WOKEN; - } - /* - * NOTE: This function might be called from signal handlers in the - * non-smp case; therefore, it has to be async-signal safe in - * the non-smp case. - */ - if (wake) { + erts_aint32_t wakeup_state; + if (!interrupted) + wakeup_state = erts_atomic32_cmpxchg_relb(&ps->wakeup_state, + ERTS_POLL_WOKEN, + ERTS_POLL_NOT_WOKEN); + else + wakeup_state = erts_atomic32_xchg_relb(&ps->wakeup_state, + ERTS_POLL_WOKEN_INTR); + wake = wakeup_state == ERTS_POLL_NOT_WOKEN; + + if (wake) +#endif + { ssize_t res; if (ps->wake_fds[1] < 0) return; /* Not initialized yet */ @@ -428,29 +430,27 @@ wake_poller(ErtsPollSet ps, int interrupted, int async_signal_safe) res = write(ps->wake_fds[1], "!", 1); } while (res < 0 && errno == EINTR); if (res <= 0 && errno != ERRNO_BLOCK) { - if (async_signal_safe) - fatal_error_async_signal_safe(__FILE__ - ":XXX:wake_poller(): " - "Failed to write on wakeup pipe\n"); - else - fatal_error("%s:%d:wake_poller(): " - "Failed to write to wakeup pipe fd=%d: " - "%s (%d)\n", - __FILE__, __LINE__, - ps->wake_fds[1], - erl_errno_id(errno), errno); + fatal_error("%s:%d:wake_poller(): " + "Failed to write to wakeup pipe fd=%d: " + "%s (%d)\n", + __FILE__, __LINE__, + ps->wake_fds[1], + erl_errno_id(errno), errno); } } } static ERTS_INLINE void -cleanup_wakeup_pipe(ErtsPollSet ps) +cleanup_wakeup_pipe(ErtsPollSet *ps) { + int intr = 0; int fd = ps->wake_fds[0]; int res; do { char buf[32]; res = read(fd, buf, sizeof(buf)); + if (res > 0) + intr = 1; } while (res > 0 || (res < 0 && errno == EINTR)); if (res < 0 && errno != ERRNO_BLOCK) { fatal_error("%s:%d:cleanup_wakeup_pipe(): " @@ -460,10 +460,14 @@ cleanup_wakeup_pipe(ErtsPollSet ps) fd, erl_errno_id(errno), errno); } +#if !ERTS_POLL_USE_CONCURRENT_UPDATE + if (intr) + erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_WOKEN_INTR); +#endif } static void -create_wakeup_pipe(ErtsPollSet ps) +create_wakeup_pipe(ErtsPollSet *ps) { int do_wake = 0; int wake_fds[2]; @@ -480,20 +484,13 @@ create_wakeup_pipe(ErtsPollSet ps) SET_NONBLOCKING(wake_fds[0]); SET_NONBLOCKING(wake_fds[1]); -#ifdef ERTS_POLL_DEBUG_PRINT - erts_printf("wakeup fds = {%d, %d}\n", wake_fds[0], wake_fds[1]); -#endif + DEBUG_PRINT("wakeup fds = {%d, %d}", ps, wake_fds[0], wake_fds[1]); ERTS_POLL_EXPORT(erts_poll_control)(ps, wake_fds[0], + ERTS_POLL_OP_ADD, ERTS_POLL_EV_IN, - 1, &do_wake); -#if ERTS_POLL_USE_FALLBACK - /* We depend on the wakeup pipe being handled by kernel poll */ - if (ps->fds_status[wake_fds[0]].flags & ERTS_POLL_FD_FLG_INFLBCK) - fatal_error("%s:%d:create_wakeup_pipe(): Internal error\n", - __FILE__, __LINE__); -#endif + &do_wake); if (ps->internal_fd_limit <= wake_fds[1]) ps->internal_fd_limit = wake_fds[1] + 1; if (ps->internal_fd_limit <= wake_fds[0]) @@ -502,82 +499,16 @@ create_wakeup_pipe(ErtsPollSet ps) ps->wake_fds[1] = wake_fds[1]; } - -/* - * --- timer fd ----------------------------------------------------------- - */ - -#if ERTS_POLL_USE_TIMERFD - -/* We use the timerfd when using epoll_wait to get high accuracy - timeouts, i.e. we want to sleep with < ms accuracy. */ - -static void -create_timerfd(ErtsPollSet ps) -{ - int do_wake = 0; - int timer_fd; - timer_fd = timerfd_create(CLOCK_MONOTONIC,0); - ERTS_POLL_EXPORT(erts_poll_control)(ps, - timer_fd, - ERTS_POLL_EV_IN, - 1, &do_wake); -#if ERTS_POLL_USE_FALLBACK - /* We depend on the wakeup pipe being handled by kernel poll */ - if (ps->fds_status[timer_fd].flags & ERTS_POLL_FD_FLG_INFLBCK) - fatal_error("%s:%d:create_wakeup_pipe(): Internal error\n", - __FILE__, __LINE__); #endif - if (ps->internal_fd_limit <= timer_fd) - ps->internal_fd_limit = timer_fd + 1; - ps->timer_fd = timer_fd; -} - -static ERTS_INLINE void -timerfd_set(ErtsPollSet ps, struct itimerspec *its) -{ -#ifdef DEBUG - struct itimerspec old_its; - int res; - res = timerfd_settime(ps->timer_fd, 0, its, &old_its); - ASSERT(res == 0); - ASSERT(old_its.it_interval.tv_sec == 0 && - old_its.it_interval.tv_nsec == 0 && - old_its.it_value.tv_sec == 0 && - old_its.it_value.tv_nsec == 0); - -#else - timerfd_settime(ps->timer_fd, 0, its, NULL); -#endif -} - -static ERTS_INLINE int -timerfd_clear(ErtsPollSet ps, int res, int max_res) { - - struct itimerspec its; - /* we always have to clear the timer */ - its.it_interval.tv_sec = 0; - its.it_interval.tv_nsec = 0; - its.it_value.tv_sec = 0; - its.it_value.tv_nsec = 0; - timerfd_settime(ps->timer_fd, 0, &its, NULL); - - /* only timeout fd triggered */ - if (res == 1 && ps->res_events[0].data.fd == ps->timer_fd) - return 0; - - return res; -} - -#endif /* ERTS_POLL_USE_TIMERFD */ - /* * --- Poll set update requests ---------------------------------------------- */ +#if !ERTS_POLL_USE_CONCURRENT_UPDATE + static ERTS_INLINE void -enqueue_update_request(ErtsPollSet ps, int fd) +enqueue_update_request(ErtsPollSet *ps, int fd) { ErtsPollSetUpdateRequestsBlock *urqbp; @@ -592,13 +523,11 @@ enqueue_update_request(ErtsPollSet ps, int fd) urqbp = ps->curr_upd_req_block; if (urqbp->len == ERTS_POLLSET_UPDATE_REQ_BLOCK_SIZE) { - ASSERT(!urqbp->next); urqbp = erts_alloc(ERTS_ALC_T_POLLSET_UPDREQ, sizeof(ErtsPollSetUpdateRequestsBlock)); - ps->curr_upd_req_block->next = urqbp; - ps->curr_upd_req_block = urqbp; - urqbp->next = NULL; + urqbp->next = ps->curr_upd_req_block; urqbp->len = 0; + ps->curr_upd_req_block = urqbp; } ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_INURQ; @@ -606,28 +535,29 @@ enqueue_update_request(ErtsPollSet ps, int fd) } static ERTS_INLINE void -free_update_requests_block(ErtsPollSet ps, +free_update_requests_block(ErtsPollSet *ps, ErtsPollSetUpdateRequestsBlock *urqbp) { if (urqbp != &ps->update_requests) erts_free(ERTS_ALC_T_POLLSET_UPDREQ, (void *) urqbp); else { - urqbp->next = NULL; urqbp->len = 0; } } +#endif /* !ERTS_POLL_USE_CONCURRENT_UPDATE */ /* * --- Growing poll set structures ------------------------------------------- */ -#ifndef ERTS_KERNEL_POLL_VERSION /* only one shared implementation */ +#if !ERTS_NO_KERNEL_POLL_VERSION || !ERTS_ENABLE_KERNEL_POLL +/* only one shared implementation */ #define ERTS_FD_TABLE_MIN_LENGTH 1024 #define ERTS_FD_TABLE_EXP_THRESHOLD (2048*1024) -int erts_poll_new_table_len (int old_len, int need_len) +int erts_poll_new_table_len(int old_len, int need_len) { int new_len; @@ -637,7 +567,7 @@ int erts_poll_new_table_len (int old_len, int need_len) } else { new_len = old_len; - do { + do { if (new_len < ERTS_FD_TABLE_EXP_THRESHOLD) new_len *= 2; else @@ -650,30 +580,9 @@ int erts_poll_new_table_len (int old_len, int need_len) } #endif -#if ERTS_POLL_USE_KERNEL_POLL -static void -grow_res_events(ErtsPollSet ps, int new_len) -{ - size_t new_size = sizeof( -#if ERTS_POLL_USE_EPOLL - struct epoll_event -#elif ERTS_POLL_USE_DEVPOLL - struct pollfd -#elif ERTS_POLL_USE_KQUEUE - struct kevent -#endif - ) * erts_poll_new_table_len(ps->res_events_len, new_len); - /* We do not need to save previously stored data */ - if (ps->res_events) - erts_free(ERTS_ALC_T_POLL_RES_EVS, ps->res_events); - ps->res_events = erts_alloc(ERTS_ALC_T_POLL_RES_EVS, new_size); - ps->res_events_len = new_len; -} -#endif /* ERTS_POLL_USE_KERNEL_POLL */ - #if ERTS_POLL_USE_POLL static void -grow_poll_fds(ErtsPollSet ps, int min_ix) +grow_poll_fds(ErtsPollSet *ps, int min_ix) { int i; int new_len = erts_poll_new_table_len(ps->poll_fds_len, min_ix + 1); @@ -721,8 +630,9 @@ ensure_select_fds(int fd, ERTS_fd_set* in, ERTS_fd_set* out) # define ensure_select_fds(fd, in, out) do {} while(0) #endif /* _DARWIN_UNLIMITED_SELECT */ +#if !ERTS_POLL_USE_CONCURRENT_UPDATE static void -grow_fds_status(ErtsPollSet ps, int min_fd) +grow_fds_status(ErtsPollSet *ps, int min_fd) { int i; int new_len = erts_poll_new_table_len(ps->fds_status_len, min_fd + 1); @@ -741,428 +651,26 @@ grow_fds_status(ErtsPollSet ps, int min_fd) #endif ps->fds_status[i].used_events = (ErtsPollEvents) 0; ps->fds_status[i].events = (ErtsPollEvents) 0; -#if ERTS_POLL_COALESCE_KP_RES - ps->fds_status[i].res_ev_ix = (unsigned short) ERTS_POLL_MAX_RES; -#endif ps->fds_status[i].flags = (unsigned short) 0; } ps->fds_status_len = new_len; } +#endif /* * --- Selecting fd to poll on ----------------------------------------------- */ -#if ERTS_POLL_USE_FALLBACK -static int update_fallback_pollset(ErtsPollSet ps, int fd); -#endif - -static ERTS_INLINE int -need_update(ErtsPollSet ps, int fd) -{ -#if ERTS_POLL_USE_KERNEL_POLL - int reset; -#endif - - ASSERT(fd < ps->fds_status_len); - -#if ERTS_POLL_USE_KERNEL_POLL - reset = (int) (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST); - if (reset && !ps->fds_status[fd].used_events) { - ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_RST; - reset = 0; - } -#else - ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_RST; -#endif - - if (ps->fds_status[fd].used_events != ps->fds_status[fd].events) - return 1; - -#if ERTS_POLL_USE_KERNEL_POLL - return reset; -#else - return 0; -#endif -} - -#if ERTS_POLL_USE_BATCH_UPDATE_POLLSET - -#if ERTS_POLL_USE_KQUEUE -#define ERTS_POLL_MIN_BATCH_BUF_SIZE 128 -#else -#define ERTS_POLL_MIN_BATCH_BUF_SIZE 64 -#endif - -typedef struct { - int len; - int size; -#if ERTS_POLL_USE_DEVPOLL - struct pollfd *buf; -#elif ERTS_POLL_USE_KQUEUE - struct kevent *buf; - struct kevent *ebuf; -#endif -} ErtsPollBatchBuf; - - -static ERTS_INLINE void -setup_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp) -{ - bbp->len = 0; -#if ERTS_POLL_USE_DEVPOLL - bbp->size = ps->res_events_len; - bbp->buf = ps->res_events; -#elif ERTS_POLL_USE_KQUEUE - bbp->size = ps->res_events_len/2; - bbp->buf = ps->res_events; - bbp->ebuf = bbp->buf + bbp->size; -#endif -} - - -#if ERTS_POLL_USE_DEVPOLL - -static void -write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp) -{ - ssize_t wres; - char *buf = (char *) bbp->buf; - size_t buf_size = sizeof(struct pollfd)*bbp->len; - - while (1) { - wres = write(ps->kp_fd, (void *) buf, buf_size); - if (wres < 0) { - if (errno == EINTR) - continue; - fatal_error("%s:%d:write_batch_buf(): " - "Failed to write to /dev/poll: " - "%s (%d)\n", - __FILE__, __LINE__, - erl_errno_id(errno), errno); - } - buf_size -= wres; - if (buf_size <= 0) - break; - buf += wres; - } - - if (buf_size < 0) { - fatal_error("%s:%d:write_devpoll_buf(): Internal error\n", - __FILE__, __LINE__); - } - bbp->len = 0; -} - -#elif ERTS_POLL_USE_KQUEUE - -static void -write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp) -{ - int res; - int len = bbp->len; - struct kevent *buf = bbp->buf; - struct timespec ts = {0, 0}; - - do { - res = kevent(ps->kp_fd, buf, len, NULL, 0, &ts); - } while (res < 0 && errno == EINTR); - if (res < 0) { - int i; - struct kevent *ebuf = bbp->ebuf; - do { - res = kevent(ps->kp_fd, buf, len, ebuf, len, &ts); - } while (res < 0 && errno == EINTR); - if (res < 0) { - fatal_error("%s:%d: kevent() failed: %s (%d)\n", - __FILE__, __LINE__, erl_errno_id(errno), errno); - } - for (i = 0; i < res; i++) { - if (ebuf[i].flags & EV_ERROR) { - short filter; - int fd = (int) ebuf[i].ident; - - switch ((int) (long) ebuf[i].udata) { - - /* - * Since we use a lazy update approach EV_DELETE will - * frequently fail. This since kqueue automatically - * removes a file descriptor that is closed from the - * poll set. - */ - case ERTS_POLL_KQ_OP_DEL_R: - case ERTS_POLL_KQ_OP_DEL_W: - case ERTS_POLL_KQ_OP_HANDLED: - break; - - /* - * According to the kqueue man page EVFILT_READ support - * does not imply EVFILT_WRITE support; therefore, - * if an EV_ADD fail, we may have to remove other - * events on this fd in the kqueue pollset before - * adding fd to the fallback pollset. - */ - case ERTS_POLL_KQ_OP_ADD_W: - if (ps->fds_status[fd].used_events & ERTS_POLL_EV_IN) { - filter = EVFILT_READ; - goto rm_add_fb; - } - goto add_fb; - case ERTS_POLL_KQ_OP_ADD_R: - if (ps->fds_status[fd].used_events & ERTS_POLL_EV_OUT) { - filter = EVFILT_WRITE; - goto rm_add_fb; - } - goto add_fb; - case ERTS_POLL_KQ_OP_ADD2_W: - case ERTS_POLL_KQ_OP_ADD2_R: { - int j; - for (j = i+1; j < res; j++) { - if (fd == (int) ebuf[j].ident) { - ebuf[j].udata = (void *) ERTS_POLL_KQ_OP_HANDLED; - if (!(ebuf[j].flags & EV_ERROR)) { - switch ((int) (long) ebuf[j].udata) { - case ERTS_POLL_KQ_OP_ADD2_W: - filter = EVFILT_WRITE; - goto rm_add_fb; - case ERTS_POLL_KQ_OP_ADD2_R: - filter = EVFILT_READ; - goto rm_add_fb; - default: - fatal_error("%s:%d:write_batch_buf(): " - "Internal error", - __FILE__, __LINE__); - break; - } - } - goto add_fb; - } - } - /* The other add succeded... */ - filter = ((((int) (long) ebuf[i].udata) - == ERTS_POLL_KQ_OP_ADD2_W) - ? EVFILT_READ - : EVFILT_WRITE); - rm_add_fb: - { - struct kevent kev; - struct timespec ts = {0, 0}; - EV_SET(&kev, fd, filter, EV_DELETE, 0, 0, 0); - (void) kevent(ps->kp_fd, &kev, 1, NULL, 0, &ts); - } - - add_fb: - ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK; - ASSERT(ps->fds_status[fd].used_events); - ps->fds_status[fd].used_events = 0; - erts_atomic_dec_nob(&ps->no_of_user_fds); - update_fallback_pollset(ps, fd); - ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK); - break; - } - default: - fatal_error("%s:%d:write_batch_buf(): Internal error", - __FILE__, __LINE__); - break; - } - } - } - } - bbp->len = 0; -} - -#endif /* ERTS_POLL_USE_KQUEUE */ - -static ERTS_INLINE void -batch_update_pollset(ErtsPollSet ps, int fd, ErtsPollBatchBuf *bbp) -{ - int buf_len; -#if ERTS_POLL_USE_DEVPOLL - short events; - struct pollfd *buf; -#elif ERTS_POLL_USE_KQUEUE - struct kevent *buf; -#endif - -#ifdef ERTS_POLL_DEBUG_PRINT - erts_printf("Doing lazy update on fd=%d\n", fd); -#endif - - if (!need_update(ps, fd)) - return; - - /* Make sure we have room for at least maximum no of entries - per fd */ - if (bbp->size - bbp->len < 2) - write_batch_buf(ps, bbp); - - buf_len = bbp->len; - buf = bbp->buf; - - ASSERT(fd < ps->fds_status_len); - -#if ERTS_POLL_USE_DEVPOLL - events = ERTS_POLL_EV_E2N(ps->fds_status[fd].events); - if (!events) { - buf[buf_len].events = POLLREMOVE; - erts_atomic_dec_nob(&ps->no_of_user_fds); - } - else if (!ps->fds_status[fd].used_events) { - buf[buf_len].events = events; - erts_atomic_inc_nob(&ps->no_of_user_fds); - } - else { - if ((ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST) - || (ps->fds_status[fd].used_events & ~events)) { - /* Reset or removed events... */ - buf[buf_len].fd = fd; - buf[buf_len].events = POLLREMOVE; - buf[buf_len++].revents = 0; - } - buf[buf_len].events = events; - } - buf[buf_len].fd = fd; - buf[buf_len++].revents = 0; - -#elif ERTS_POLL_USE_KQUEUE - - if (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK) { - if (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_USEFLBCK) - update_fallback_pollset(ps, fd); - else { /* Remove from fallback and try kqueue */ - ErtsPollEvents events = ps->fds_status[fd].events; - ps->fds_status[fd].events = (ErtsPollEvents) 0; - update_fallback_pollset(ps, fd); - ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK)); - if (events) { - ps->fds_status[fd].events = events; - goto try_kqueue; - } - } - } - else { - ErtsPollEvents events, used_events; - int mod_w, mod_r; - try_kqueue: - events = ERTS_POLL_EV_E2N(ps->fds_status[fd].events); - used_events = ERTS_POLL_EV_E2N(ps->fds_status[fd].used_events); - if (!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST)) { - if (!used_events && - (events & ERTS_POLL_EV_IN) && (events & ERTS_POLL_EV_OUT)) - goto do_add_rw; - mod_r = ((events & ERTS_POLL_EV_IN) - != (used_events & ERTS_POLL_EV_IN)); - mod_w = ((events & ERTS_POLL_EV_OUT) - != (used_events & ERTS_POLL_EV_OUT)); - goto do_mod; - } - else { /* Reset */ - if ((events & ERTS_POLL_EV_IN) && (events & ERTS_POLL_EV_OUT)) { - do_add_rw: - EV_SET(&buf[buf_len], fd, EVFILT_READ, EV_ADD, - 0, 0, (void *) ERTS_POLL_KQ_OP_ADD2_R); - buf_len++; - EV_SET(&buf[buf_len], fd, EVFILT_WRITE, EV_ADD, - 0, 0, (void *) ERTS_POLL_KQ_OP_ADD2_W); - buf_len++; - - } - else { - mod_r = 1; - mod_w = 1; - do_mod: - if (mod_r) { - if (events & ERTS_POLL_EV_IN) { - EV_SET(&buf[buf_len], fd, EVFILT_READ, EV_ADD, - 0, 0, (void *) ERTS_POLL_KQ_OP_ADD_R); - buf_len++; - } - else if (used_events & ERTS_POLL_EV_IN) { - EV_SET(&buf[buf_len], fd, EVFILT_READ, EV_DELETE, - 0, 0, (void *) ERTS_POLL_KQ_OP_DEL_R); - buf_len++; - } - } - if (mod_w) { - if (events & ERTS_POLL_EV_OUT) { - EV_SET(&buf[buf_len], fd, EVFILT_WRITE, EV_ADD, - 0, 0, (void *) ERTS_POLL_KQ_OP_ADD_W); - buf_len++; - } - else if (used_events & ERTS_POLL_EV_OUT) { - EV_SET(&buf[buf_len], fd, EVFILT_WRITE, EV_DELETE, - 0, 0, (void *) ERTS_POLL_KQ_OP_DEL_W); - buf_len++; - } - } - } - } - if (used_events) { - if (!events) { - erts_atomic_dec_nob(&ps->no_of_user_fds); - } - } - else { - if (events) - erts_atomic_inc_nob(&ps->no_of_user_fds); - } - ASSERT((events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0); - ASSERT((used_events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0); - } - -#endif - - ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_RST; - ps->fds_status[fd].used_events = ps->fds_status[fd].events; - - bbp->len = buf_len; -} - -#else /* !ERTS_POLL_USE_BATCH_UPDATE_POLLSET */ - #if ERTS_POLL_USE_EPOLL static int -#if ERTS_POLL_USE_CONCURRENT_UPDATE -conc_update_pollset(ErtsPollSet ps, int fd, int *update_fallback) -#else -update_pollset(ErtsPollSet ps, int fd) -#endif +update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events) { int res; - int op; + int epoll_op = EPOLL_CTL_MOD; struct epoll_event epe_templ; struct epoll_event epe; - ASSERT(fd < ps->fds_status_len); - - if (!need_update(ps, fd)) - return 0; - -#ifdef ERTS_POLL_DEBUG_PRINT - erts_printf("Doing update on fd=%d\n", fd); -#endif - if (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK) { -#if ERTS_POLL_USE_CONCURRENT_UPDATE - if (!*update_fallback) { - *update_fallback = 1; - return 0; - } -#endif - if (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_USEFLBCK) { - return update_fallback_pollset(ps, fd); - } - else { /* Remove from fallback and try epoll */ - ErtsPollEvents events = ps->fds_status[fd].events; - ps->fds_status[fd].events = (ErtsPollEvents) 0; - res = update_fallback_pollset(ps, fd); - ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK)); - if (!events) - return res; - ps->fds_status[fd].events = events; - } - } - - epe_templ.events = ERTS_POLL_EV_E2N(ps->fds_status[fd].events); + epe_templ.events = ERTS_POLL_EV_E2N(events) | EPOLLONESHOT; epe_templ.data.fd = fd; #ifdef VALGRIND @@ -1170,30 +678,24 @@ update_pollset(ErtsPollSet ps, int fd) memset((void *) &epe.data, 0, sizeof(epoll_data_t)); #endif - if (epe_templ.events && ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST) { - do { - /* We init 'epe' every time since epoll_ctl() may modify it - (not declared const and not documented as const). */ - epe.events = epe_templ.events; - epe.data.fd = epe_templ.data.fd; - res = epoll_ctl(ps->kp_fd, EPOLL_CTL_DEL, fd, &epe); - } while (res != 0 && errno == EINTR); - erts_atomic_dec_nob(&ps->no_of_user_fds); - ps->fds_status[fd].used_events = 0; - } - - if (!epe_templ.events) { + switch (op) { + case ERTS_POLL_OP_DEL: /* A note on EPOLL_CTL_DEL: linux kernel versions before 2.6.9 need a non-NULL event pointer even though it is ignored... */ - op = EPOLL_CTL_DEL; + epoll_op = EPOLL_CTL_DEL; + epe_templ.events = 0; erts_atomic_dec_nob(&ps->no_of_user_fds); - } - else if (!ps->fds_status[fd].used_events) { - op = EPOLL_CTL_ADD; + break; + case ERTS_POLL_OP_ADD: + epoll_op = EPOLL_CTL_ADD; erts_atomic_inc_nob(&ps->no_of_user_fds); - } - else { - op = EPOLL_CTL_MOD; + break; + case ERTS_POLL_OP_MOD: + epoll_op = EPOLL_CTL_MOD; + break; + default: + ASSERT(0); + break; } do { @@ -1201,33 +703,32 @@ update_pollset(ErtsPollSet ps, int fd) (not declared const and not documented as const). */ epe.events = epe_templ.events; epe.data.fd = epe_templ.data.fd; - res = epoll_ctl(ps->kp_fd, op, fd, &epe); + res = epoll_ctl(ps->kp_fd, epoll_op, fd, &epe); } while (res != 0 && errno == EINTR); -#if defined(ERTS_POLL_DEBUG_PRINT) && 1 +#if ERTS_POLL_DEBUG_PRINT { int saved_errno = errno; - erts_printf("%s = epoll_ctl(%d, %s, %d, {Ox%x, %d})\n", - res == 0 ? "0" : erl_errno_id(errno), - ps->kp_fd, - (op == EPOLL_CTL_ADD - ? "EPOLL_CTL_ADD" - : (op == EPOLL_CTL_MOD - ? "EPOLL_CTL_MOD" - : (op == EPOLL_CTL_DEL - ? "EPOLL_CTL_DEL" - : "UNKNOWN"))), - fd, - epe_templ.events, - fd); + DEBUG_PRINT_FD("%s = epoll_ctl(%d, %s, %d, {0x%x, %d})", + ps, fd, + res == 0 ? "0" : erl_errno_id(errno), + ps->kp_fd, + (epoll_op == EPOLL_CTL_ADD + ? "EPOLL_CTL_ADD" + : (epoll_op == EPOLL_CTL_MOD + ? "EPOLL_CTL_MOD" + : (epoll_op == EPOLL_CTL_DEL + ? "EPOLL_CTL_DEL" + : "UNKNOWN"))), + fd, + epe_templ.events, + fd); errno = saved_errno; } #endif - if (res == 0) - ps->fds_status[fd].used_events = ps->fds_status[fd].events; - else { + if (res != 0) { switch (op) { - case EPOLL_CTL_MOD: + case ERTS_POLL_OP_MOD: epe.events = 0; do { /* We init 'epe' every time since epoll_ctl() may modify it @@ -1236,29 +737,18 @@ update_pollset(ErtsPollSet ps, int fd) epe.data.fd = fd; res = epoll_ctl(ps->kp_fd, EPOLL_CTL_DEL, fd, &epe); } while (res != 0 && errno == EINTR); - ps->fds_status[fd].used_events = 0; /* Fall through ... */ - case EPOLL_CTL_ADD: { - ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK; + case ERTS_POLL_OP_ADD: { erts_atomic_dec_nob(&ps->no_of_user_fds); -#if ERTS_POLL_USE_CONCURRENT_UPDATE - if (!*update_fallback) { - *update_fallback = 1; - return 0; - } -#endif - ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK)); - res = update_fallback_pollset(ps, fd); - ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK); + res = ERTS_POLL_EV_NVAL; break; } - case EPOLL_CTL_DEL: { + case ERTS_POLL_OP_DEL: { /* * Since we use a lazy update approach EPOLL_CTL_DEL will * frequently fail. This since epoll automatically removes * a filedescriptor that is closed from the poll set. */ - ps->fds_status[fd].used_events = 0; res = 0; break; } @@ -1267,67 +757,277 @@ update_pollset(ErtsPollSet ps, int fd) __FILE__, __LINE__); break; } + } else { + res = events; } - ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_RST; return res; } -#if ERTS_POLL_USE_CONCURRENT_UPDATE +#endif /* ERTS_POLL_USE_EPOLL */ + +#if ERTS_POLL_USE_KQUEUE + +/* Some versions of the EV_SET macro used kevp multiple times, + so we define out own version that make sure that it is safe + to do kevp++ in the argument list. */ +#define ERTS_EV_SET(kevp, a, b, c, f) do { \ + struct kevent *kevp_ = kevp; \ + EV_SET(kevp_, a, b, c, 0, 0, f); \ + } while(0) + static int -update_pollset(ErtsPollSet ps, int fd) +update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events) { - int update_fallback = 1; - return conc_update_pollset(ps, fd, &update_fallback); -} -#endif - -#endif /* ERTS_POLL_USE_EPOLL */ + int res = 0, len = 0; + struct kevent evts[2]; + struct timespec ts = {0, 0}; -#endif /* ERTS_POLL_USE_BATCH_UPDATE_POLLSET */ +#ifdef EV_DISPATCH + /* If we have EV_DISPATCH we use it. The kevent descriptions for both + read and write are added on OP_ADD and removed on OP_DEL. And then + after than only EV_ENABLE|EV_DISPATCH are used. + + It could be possible to not modify the pollset when disabling and/or + deleting events, but that may cause the poll threads to be awoken + a lot more than they should so we take the cost here instead of + in the poll thread. + + Note: We need to have EV_DISPATCH both when the event is enabled and + disabled, as otherwise the event may be triggered twice on each re-arm. + Not sure if this is intended or not (can't find anything about it in the + man page), but it seems to be the way it works... + */ + + if (op == ERTS_POLL_OP_DEL) { + erts_atomic_dec_nob(&ps->no_of_user_fds); + /* We could probably skip this delete, do we want to? */ + ERTS_EV_SET(&evts[len++], fd, EVFILT_READ, EV_DELETE, (void *) 0); + ERTS_EV_SET(&evts[len++], fd, EVFILT_WRITE, EV_DELETE, (void *) 0); + } else if (op == ERTS_POLL_OP_ADD) { + uint32_t flags; + erts_atomic_inc_nob(&ps->no_of_user_fds); + + flags = EV_ADD|EV_DISPATCH; + flags |= ((events & ERTS_POLL_EV_IN) ? 0 : EV_DISABLE); + ERTS_EV_SET(&evts[len++], fd, EVFILT_READ, flags, (void *) ERTS_POLL_EV_IN); + + flags = EV_ADD|EV_DISPATCH; + flags |= ((events & ERTS_POLL_EV_OUT) ? 0 : EV_DISABLE); + ERTS_EV_SET(&evts[len++], fd, EVFILT_WRITE, flags, (void *) ERTS_POLL_EV_OUT); + } else { + uint32_t flags; + ASSERT(op == ERTS_POLL_OP_MOD); -#if ERTS_POLL_USE_POLL || ERTS_POLL_USE_SELECT || ERTS_POLL_USE_FALLBACK + flags = EV_DISPATCH; + flags |= (events & ERTS_POLL_EV_IN) ? EV_ENABLE : EV_DISABLE; + ERTS_EV_SET(&evts[len++], fd, EVFILT_READ, flags, (void *) ERTS_POLL_EV_IN); -#if ERTS_POLL_USE_FALLBACK -static int update_fallback_pollset(ErtsPollSet ps, int fd) + flags = EV_DISPATCH; + flags |= (events & ERTS_POLL_EV_OUT) ? EV_ENABLE : EV_DISABLE; + ERTS_EV_SET(&evts[len++], fd, EVFILT_WRITE, flags, (void *) ERTS_POLL_EV_OUT); + } #else -static int update_pollset(ErtsPollSet ps, int fd) + uint32_t flags = EV_ADD|EV_ONESHOT; + + if (op == ERTS_POLL_OP_DEL) { + erts_atomic_dec_nob(&ps->no_of_user_fds); + /* We don't do anything when a delete is issued. The fds will be removed + when they are triggered, or when they are closed. */ + events = 0; + } else if (op == ERTS_POLL_OP_ADD) { + erts_atomic_inc_nob(&ps->no_of_user_fds); + } + + if (events & ERTS_POLL_EV_IN) { + ERTS_EV_SET(&evts[len++], fd, EVFILT_READ, flags, (void *) ERTS_POLL_EV_IN); + } + if (events & ERTS_POLL_EV_OUT) { + ERTS_EV_SET(&evts[len++], fd, EVFILT_WRITE, flags, (void *) ERTS_POLL_EV_OUT); + } + #endif -{ -#ifdef ERTS_POLL_DEBUG_PRINT -#if ERTS_POLL_USE_FALLBACK - erts_printf("Doing fallback update on fd=%d\n", fd); + if (len) + do { + res = kevent(ps->kp_fd, evts, len, NULL, 0, &ts); + } while (res < 0 && errno == EINTR); +#if ERTS_POLL_DEBUG_PRINT + { + int saved_errno = errno, i; + char keventb[255], *keventbp = keventb; + if (res < 0) + keventbp += sprintf(keventbp,"%s = ",erl_errno_id(saved_errno)); + else + keventbp += sprintf(keventbp,"%d = ",res); + keventbp += sprintf(keventbp, "kevent(%d, {",ps->kp_fd); + for (i = 0; i < len; i++) { + const char *flags = "UNKNOWN"; + if (evts[i].flags == EV_DELETE) flags = "EV_DELETE"; + if (evts[i].flags == (EV_ADD|EV_ONESHOT)) flags = "EV_ADD|EV_ONESHOT"; +#ifdef EV_DISPATCH + if (evts[i].flags == (EV_ADD|EV_DISPATCH)) flags = "EV_ADD|EV_DISPATCH"; + if (evts[i].flags == (EV_ADD|EV_DISABLE)) flags = "EV_ADD|EV_DISABLE"; + if (evts[i].flags == (EV_ENABLE|EV_DISPATCH)) flags = "EV_ENABLE|EV_DISPATCH"; + if (evts[i].flags == EV_DISABLE) flags = "EV_DISABLE"; + if (evts[i].flags == (EV_DISABLE|EV_DISPATCH)) flags = "EV_DISABLE|EV_DISABLE"; +#endif + + keventbp += sprintf(keventbp, "%s{%lu, %s, %s}",i > 0 ? ", " : "", + evts[i].ident, + (evts[i].filter == EVFILT_READ + ? "EVFILT_READ" + : (evts[i].filter == EVFILT_WRITE + ? "EVFILT_WRITE" + : "UNKNOWN")), flags); + } + keventbp += sprintf(keventbp, "}, %d)", len); + DEBUG_PRINT_FD("%s", ps, fd, keventb); + errno = saved_errno; + } +#endif + if (res < 0) { + if (op != ERTS_POLL_OP_DEL) { +#ifdef EV_RECEIPT + struct kevent receipt_evts[2]; + len = 0; + ERTS_EV_SET(&evts[len++], fd, EVFILT_WRITE, EV_DELETE|EV_RECEIPT, (void *) 0); + ERTS_EV_SET(&evts[len++], fd, EVFILT_READ, EV_DELETE|EV_RECEIPT, (void *) 0); + do { + res = kevent(ps->kp_fd, evts, len, receipt_evts, 2, &ts); + } while (res < 0 && errno == EINTR); #else - erts_printf("Doing update on fd=%d\n", fd); + ERTS_EV_SET(&evts[0], fd, EVFILT_WRITE, EV_DELETE, (void *) 0); + do { + res = kevent(ps->kp_fd, evts, 1, NULL, 0, &ts); + } while (res < 0 && errno == EINTR); + ERTS_EV_SET(&evts[0], fd, EVFILT_READ, EV_DELETE, (void *) 0); + do { + res = kevent(ps->kp_fd, evts, 1, NULL, 0, &ts); + } while (res < 0 && errno == EINTR); +#endif + if (op == ERTS_POLL_OP_ADD) + erts_atomic_dec_nob(&ps->no_of_user_fds); + events = ERTS_POLL_EV_NVAL; + } else + events = 0; + } + return events; +} + +#endif /* ERTS_POLL_USE_KQUEUE */ + +#if !ERTS_POLL_USE_CONCURRENT_UPDATE + +static ERTS_INLINE void +init_batch_update(ErtsPollSet *ps, int len) +{ +#if ERTS_POLL_USE_DEVPOLL + ASSERT(ps->poll_fds == NULL); + ps->poll_fds = erts_alloc(ERTS_ALC_T_TMP, sizeof(struct pollfd) * len); + ps->poll_fds_ix = 0; #endif +} + +static ERTS_INLINE void +write_batch_update(ErtsPollSet *ps) +{ +#if ERTS_POLL_USE_DEVPOLL + ssize_t wres; + char *buf = (char *) ps->poll_fds; + size_t buf_size = sizeof(struct pollfd)*ps->poll_fds_ix; + + while (1) { + wres = write(ps->kp_fd, (void *) buf, buf_size); + if (wres < 0) { + if (errno == EINTR) + continue; + fatal_error("%s:%d:write_batch_buf(): " + "Failed to write to /dev/poll: " + "%s (%d)\n", + __FILE__, __LINE__, + erl_errno_id(errno), errno); + } +#if ERTS_POLL_DEBUG_PRINT + { + int saved_errno = errno, i; + char devpollb[2048], *devpollbp = devpollb; + devpollbp += sprintf(devpollbp, "%d = devpoll(%d, {", wres, ps->kp_fd); + for (i = 0; i < wres / sizeof(struct pollfd); i++) { + if (devpollbp == devpollb) + devpollbp += sprintf(devpollbp, "%d = devpoll(%d, {", wres, ps->kp_fd); + devpollbp += sprintf(devpollbp, "%s{fd = %d, events = %s}", + i > 0 ? ", " : "", + ps->poll_fds[i].fd, + ev2str(ps->poll_fds[i].events)); + if (devpollbp - devpollb > 512) { + devpollbp += sprintf(devpollbp, "}, %d)", ps->poll_fds_ix); + DEBUG_PRINT("%s", ps, devpollb); + devpollbp = devpollb; + } + } + devpollbp += sprintf(devpollbp, "}, %d)", ps->poll_fds_ix); + DEBUG_PRINT("%s", ps, devpollb); + errno = saved_errno; + } #endif - ASSERT(fd < ps->fds_status_len); -#if ERTS_POLL_USE_FALLBACK - ASSERT(ps->fds_status[fd].used_events - ? (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK) - : (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_USEFLBCK)); + buf_size -= wres; + if (buf_size <= 0) + break; + buf += wres; + } + + if (buf_size < 0) { + fatal_error("%s:%d:write_devpoll_buf(): Internal error\n", + __FILE__, __LINE__); + } + erts_free(ERTS_ALC_T_TMP, ps->poll_fds); + ps->poll_fds = NULL; #endif +} - if (!need_update(ps, fd)) - return 0; +static ERTS_INLINE int +need_update(ErtsPollSet *ps, int fd, int *resetp) +{ + int reset; + ASSERT(fd < ps->fds_status_len); -#if ERTS_POLL_USE_FALLBACK + reset = (int) (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST); ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_RST; -#endif + + *resetp = reset; + + if (reset || ps->fds_status[fd].used_events != ps->fds_status[fd].events) + return 1; + + return 0; +} + +static int update_pollset(ErtsPollSet *ps, ErtsPollResFd pr[], int fd) +{ + int res = 0, reset = 0; + ErtsPollEvents events = ps->fds_status[fd].events; + ASSERT(fd < ps->fds_status_len); + + if (!need_update(ps, fd, &reset)) + return res; #if ERTS_POLL_USE_POLL /* --- poll -------------------------------- */ - if (!ps->fds_status[fd].events) { + if (!events) { int pix = ps->fds_status[fd].pix; int last_pix; + + if (reset) { + /* When a fd has been reset, we tell the caller of erts_poll_wait + this by setting the fd as ERTS_POLL_EV_NONE */ + ERTS_POLL_RES_SET_FD(&pr[res], fd); + ERTS_POLL_RES_SET_EVTS(&pr[res], ERTS_POLL_EV_NONE); + DEBUG_PRINT_FD("trig %s (poll)", ps, fd, ev2str(ERTS_POLL_EV_NONE)); + res++; + } + if (pix < 0) { -#if ERTS_POLL_USE_FALLBACK - ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK)); -#endif - return -1; + return res; } -#if ERTS_POLL_USE_FALLBACK - ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK); -#endif erts_atomic_dec_nob(&ps->no_of_user_fds); last_pix = --ps->no_poll_fds; if (pix != last_pix) { @@ -1344,126 +1044,151 @@ static int update_pollset(ErtsPollSet ps, int fd) /* Clear this fd status */ ps->fds_status[fd].pix = -1; ps->fds_status[fd].used_events = (ErtsPollEvents) 0; -#if ERTS_POLL_USE_FALLBACK - ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_INFLBCK; -#endif + } else { int pix = ps->fds_status[fd].pix; if (pix < 0) { -#if ERTS_POLL_USE_FALLBACK - ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK) - || fd == ps->kp_fd); -#endif erts_atomic_inc_nob(&ps->no_of_user_fds); ps->fds_status[fd].pix = pix = ps->no_poll_fds++; if (pix >= ps->poll_fds_len) grow_poll_fds(ps, pix); ps->poll_fds[pix].fd = fd; ps->fds_status[fd].pix = pix; -#if ERTS_POLL_USE_FALLBACK - ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_INFLBCK; -#endif } -#if ERTS_POLL_USE_FALLBACK - ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK); -#endif - /* Events to be used in next poll */ - ps->poll_fds[pix].events = ev2pollev(ps->fds_status[fd].events); + ps->poll_fds[pix].events = ev2pollev(events); if (ps->poll_fds[pix].revents) { /* Remove result events that we should not poll for anymore */ ps->poll_fds[pix].revents &= ev2pollev(~(~ps->fds_status[fd].used_events - & ps->fds_status[fd].events)); + & events)); } /* Save events to be used in next poll */ - ps->fds_status[fd].used_events = ps->fds_status[fd].events; + ps->fds_status[fd].used_events = events; } - return 0; + return res; #elif ERTS_POLL_USE_SELECT /* --- select ------------------------------ */ - { - ErtsPollEvents events = ps->fds_status[fd].events; + if (!events) { + + if (reset) { + /* When a fd has been reset, we tell the caller of erts_poll_wait + this by setting the fd as ERTS_POLL_EV_NONE */ + ERTS_POLL_RES_SET_FD(&pr[res], fd); + ERTS_POLL_RES_SET_EVTS(&pr[res], ERTS_POLL_EV_NONE); + DEBUG_PRINT_FD("trig %s (select)", ps, fd, ev2str(ERTS_POLL_EV_NONE)); + res++; + } + + ERTS_FD_CLR(fd, &ps->input_fds); + ERTS_FD_CLR(fd, &ps->output_fds); + + if (ps->fds_status[fd].used_events) { + erts_atomic_dec_nob(&ps->no_of_user_fds); + ps->fds_status[fd].used_events = (ErtsPollEvents) 0; + } + + if (fd == ps->max_fd) { + int max = ps->max_fd; + for (max = ps->max_fd; max >= 0; max--) + if (ps->fds_status[max].used_events) + break; + ps->max_fd = max; + } + + } else { + ensure_select_fds(fd, &ps->input_fds, &ps->output_fds); - if ((ERTS_POLL_EV_IN & events) - != (ERTS_POLL_EV_IN & ps->fds_status[fd].used_events)) { - if (ERTS_POLL_EV_IN & events) { - ERTS_FD_SET(fd, &ps->input_fds); - } - else { - ERTS_FD_CLR(fd, &ps->input_fds); - } - } - if ((ERTS_POLL_EV_OUT & events) - != (ERTS_POLL_EV_OUT & ps->fds_status[fd].used_events)) { - if (ERTS_POLL_EV_OUT & events) { - ERTS_FD_SET(fd, &ps->output_fds); - } - else { - ERTS_FD_CLR(fd, &ps->output_fds); - } - } - if (!ps->fds_status[fd].used_events) { - ASSERT(events); - erts_atomic_inc_nob(&ps->no_of_user_fds); -#if ERTS_POLL_USE_FALLBACK - ps->no_select_fds++; - ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_INFLBCK; -#endif - } - else if (!events) { - ASSERT(ps->fds_status[fd].used_events); - erts_atomic_dec_nob(&ps->no_of_user_fds); - ps->fds_status[fd].events = events; -#if ERTS_POLL_USE_FALLBACK - ps->no_select_fds--; - ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_INFLBCK; -#endif - } + if (!ps->fds_status[fd].used_events) + erts_atomic_inc_nob(&ps->no_of_user_fds); + + if (events & ERTS_POLL_EV_IN) + ERTS_FD_SET(fd, &ps->input_fds); + else + ERTS_FD_CLR(fd, &ps->input_fds); + + if (events & ERTS_POLL_EV_OUT) + ERTS_FD_SET(fd, &ps->output_fds); + else + ERTS_FD_CLR(fd, &ps->output_fds); ps->fds_status[fd].used_events = events; - if (events && fd > ps->max_fd) - ps->max_fd = fd; - else if (!events && fd == ps->max_fd) { - int max = ps->max_fd; - for (max = ps->max_fd; max >= 0; max--) - if (ps->fds_status[max].used_events) - break; - ps->max_fd = max; - } + if (fd > ps->max_fd) + ps->max_fd = fd; } - return 0; -#endif -} -#endif /* ERTS_POLL_USE_POLL || ERTS_POLL_USE_SELECT || ERTS_POLL_USE_FALLBACK */ + return res; +#elif ERTS_POLL_USE_DEVPOLL + + if (!events) { + if (reset) { + /* When a fd has been reset, we tell the caller of erts_poll_wait + this by setting the fd as ERTS_POLL_EV_NONE */ + ERTS_POLL_RES_SET_FD(&pr[res], fd); + ERTS_POLL_RES_SET_EVTS(&pr[res], ERTS_POLL_EV_NONE); + DEBUG_PRINT_FD("trig %s (devpoll)", ps, fd, ev2str(ERTS_POLL_EV_NONE)); + res++; + } -static void -handle_update_requests(ErtsPollSet ps) -{ - ErtsPollSetUpdateRequestsBlock *urqbp = &ps->update_requests; -#if ERTS_POLL_USE_BATCH_UPDATE_POLLSET - ErtsPollBatchBuf bb; - setup_batch_buf(ps, &bb); + ps->poll_fds[ps->poll_fds_ix].fd = fd; + ps->poll_fds[ps->poll_fds_ix].revents = 0; + ps->poll_fds[ps->poll_fds_ix++].events = POLLREMOVE; + + if (ps->fds_status[fd].used_events) { + erts_atomic_dec_nob(&ps->no_of_user_fds); + ps->fds_status[fd].used_events = 0; + } + + } else { + if (!ps->fds_status[fd].used_events) { + erts_atomic_inc_nob(&ps->no_of_user_fds); + } + ps->poll_fds[ps->poll_fds_ix].fd = fd; + ps->poll_fds[ps->poll_fds_ix].revents = 0; + ps->poll_fds[ps->poll_fds_ix++].events = ERTS_POLL_EV_E2N(events); + ps->fds_status[fd].used_events = ps->fds_status[fd].events; + } + + return res; #endif +} + +static int +handle_update_requests(ErtsPollSet *ps, ErtsPollResFd pr[], int no_fds) +{ + int res = 0; + ErtsPollSetUpdateRequestsBlock *urqbp = ps->curr_upd_req_block; while (urqbp) { ErtsPollSetUpdateRequestsBlock *free_urqbp = urqbp; int i; int len = urqbp->len; + + init_batch_update(ps, len); + for (i = 0; i < len; i++) { int fd = urqbp->fds[i]; ASSERT(fd < ps->fds_status_len); - ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_INURQ; -#if ERTS_POLL_USE_BATCH_UPDATE_POLLSET - batch_update_pollset(ps, fd, &bb); -#else - update_pollset(ps, fd); -#endif + ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INURQ); + + /* We have run out of PollResFd slots to put results in, + so we yield here and return later for more. */ + if (res == no_fds && pr != NULL) { + memmove(urqbp->fds, urqbp->fds+i, sizeof(int) * (len - i)); + urqbp->len -= i; + ps->curr_upd_req_block = urqbp; + write_batch_update(ps); + return res; + } + + if (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INURQ) { + ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_INURQ; + res += update_pollset(ps, pr + res, fd); + } } free_urqbp = urqbp; @@ -1471,12 +1196,9 @@ handle_update_requests(ErtsPollSet ps) free_update_requests_block(ps, free_urqbp); - } + write_batch_update(ps); -#if ERTS_POLL_USE_BATCH_UPDATE_POLLSET - if (bb.len) - write_batch_buf(ps, &bb); -#endif + } ps->curr_upd_req_block = &ps->update_requests; @@ -1485,16 +1207,19 @@ handle_update_requests(ErtsPollSet ps) #endif ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(ps); + return res; } +#endif /* !ERTS_POLL_USE_CONCURRENT_UPDATE */ static ERTS_INLINE ErtsPollEvents -poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, int *do_wake) +poll_control(ErtsPollSet *ps, int fd, ErtsPollOp op, + ErtsPollEvents events, int *do_wake) { ErtsPollEvents new_events; if (fd < ps->internal_fd_limit || fd >= max_fds) { - if (fd < 0) { + if (fd < 0 || fd >= max_fds) { new_events = ERTS_POLL_EV_ERR; goto done; } @@ -1504,115 +1229,55 @@ poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, int *do_wake goto done; } #endif +#if ERTS_POLL_USE_WAKEUP_PIPE if (fd == ps->wake_fds[0] || fd == ps->wake_fds[1]) { new_events = ERTS_POLL_EV_NVAL; goto done; } -#if ERTS_POLL_USE_TIMERFD - if (fd == ps->timer_fd) { - new_events = ERTS_POLL_EV_NVAL; - goto done; - } #endif } +#if ERTS_POLL_USE_CONCURRENT_UPDATE + + new_events = update_pollset(ps, fd, op, events); + +#else /* !ERTS_POLL_USE_CONCURRENT_UPDATE */ if (fd >= ps->fds_status_len) grow_fds_status(ps, fd); ASSERT(fd < ps->fds_status_len); - new_events = ps->fds_status[fd].events; - - if (events == 0) { - *do_wake = 0; - goto done; - } - - if (on) - new_events |= events; - else - new_events &= ~events; - - if (new_events == (ErtsPollEvents) 0) { - ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_RST; -#if ERTS_POLL_USE_FALLBACK - ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_USEFLBCK; -#endif - } - - ps->fds_status[fd].events = new_events; - - if (new_events == ps->fds_status[fd].used_events - && !(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST) - ) { - *do_wake = 0; - goto done; - } - - -#if ERTS_POLL_USE_CONCURRENT_UPDATE - if (ERTS_POLLSET_IS_POLLED(ps)) { - int update_fallback = 0; - conc_update_pollset(ps, fd, &update_fallback); - if (!update_fallback) { - *do_wake = 0; /* no need to wake kernel poller */ - goto done; - } + if (op == ERTS_POLL_OP_DEL) { + ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_RST; + ps->fds_status[fd].events = 0; + *do_wake = 1; + } else if (op == ERTS_POLL_OP_ADD) { + ASSERT(ps->fds_status[fd].events == 0); + ps->fds_status[fd].events = events; + *do_wake = 1; + } else { + ASSERT(op == ERTS_POLL_OP_MOD); + ps->fds_status[fd].events = events; + *do_wake = 1; } -#endif + new_events = ps->fds_status[fd].events; enqueue_update_request(ps, fd); - - /* - * If new events have been added, we need to wake up the - * polling thread, but if events have been removed we don't. - */ - if ((new_events && (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST)) - || (~ps->fds_status[fd].used_events & new_events)) - *do_wake = 1; +#endif /* !ERTS_POLL_USE_CONCURRENT_UPDATE */ done: -#ifdef ERTS_POLL_DEBUG_PRINT - erts_printf("0x%x = poll_control(ps, %d, 0x%x, %s) do_wake=%d\n", - (int) new_events, fd, (int) events, (on ? "on" : "off"), *do_wake); -#endif + DEBUG_PRINT_FD("%s = %s(%p, %d, %s, %s) do_wake=%d", + ps, fd, ev2str(new_events), __FUNCTION__, ps, + fd, op2str(op), ev2str(events), *do_wake); return new_events; } -void -ERTS_POLL_EXPORT(erts_poll_controlv)(ErtsPollSet ps, - ErtsPollControlEntry pcev[], - int len) -{ - int i; - int do_wake; - int final_do_wake = 0; - - ERTS_POLLSET_LOCK(ps); - - for (i = 0; i < len; i++) { - do_wake = 0; - pcev[i].events = poll_control(ps, - pcev[i].fd, - pcev[i].events, - pcev[i].on, - &do_wake); - final_do_wake |= do_wake; - } - - ERTS_POLLSET_UNLOCK(ps); - - if (final_do_wake) - wake_poller(ps, 0, 0); - -} - ErtsPollEvents -ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet ps, +ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet *ps, ErtsSysFdType fd, + ErtsPollOp op, ErtsPollEvents events, - int on, int* do_wake) /* In: Wake up polling thread */ /* Out: Poller is woken */ { @@ -1620,13 +1285,15 @@ ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet ps, ERTS_POLLSET_LOCK(ps); - res = poll_control(ps, fd, events, on, do_wake); + res = poll_control(ps, fd, op, events, do_wake); ERTS_POLLSET_UNLOCK(ps); +#if !ERTS_POLL_USE_CONCURRENT_UPDATE if (*do_wake) { - wake_poller(ps, 0, 0); + wake_poller(ps, 0); } +#endif return res; } @@ -1638,180 +1305,64 @@ ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet ps, #if ERTS_POLL_USE_KERNEL_POLL static ERTS_INLINE int -save_kp_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, int chk_fds_res) +ERTS_POLL_EXPORT(save_result)(ErtsPollSet *ps, ErtsPollResFd pr[], int max_res, int chk_fds_res, int ebadf) { - int res = 0; - int i; - int n = chk_fds_res < max_res ? chk_fds_res : max_res; +#if !ERTS_POLL_USE_CONCURRENT_UPDATE || ERTS_POLL_DEBUG_PRINT || ERTS_POLL_USE_WAKEUP_PIPE + int n = chk_fds_res < max_res ? chk_fds_res : max_res, i; + int res = n; +#if ERTS_POLL_USE_WAKEUP_PIPE int wake_fd = ps->wake_fds[0]; -#if ERTS_POLL_USE_TIMERFD - int timer_fd = ps->timer_fd; #endif for (i = 0; i < n; i++) { - -#if ERTS_POLL_USE_EPOLL /* --- epoll ------------------------------- */ - - if (ps->res_events[i].events) { - int fd = ps->res_events[i].data.fd; - int ix; - ErtsPollEvents revents; - if (fd == wake_fd) { - cleanup_wakeup_pipe(ps); - continue; - } -#if ERTS_POLL_USE_TIMERFD - if (fd == timer_fd) { - continue; - } -#endif - ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK)); - /* epoll_wait() can repeat the same fd in result array... */ - ix = (int) ps->fds_status[fd].res_ev_ix; - ASSERT(ix >= 0); - if (ix >= res || pr[ix].fd != fd) { - ix = res; - pr[ix].fd = fd; - pr[ix].events = (ErtsPollEvents) 0; - } - - revents = ERTS_POLL_EV_N2E(ps->res_events[i].events); - pr[ix].events |= revents; - if (revents) { - if (res == ix) { - ps->fds_status[fd].res_ev_ix = (unsigned short) ix; - res++; - } - } - } - -#elif ERTS_POLL_USE_KQUEUE /* --- kqueue ------------------------------ */ - - struct kevent *ev; - int fd; - int ix; - - ev = &ps->res_events[i]; - fd = (int) ev->ident; - ASSERT(fd < ps->fds_status_len); - ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK)); - ix = (int) ps->fds_status[fd].res_ev_ix; - - ASSERT(ix >= 0); - if (ix >= res || pr[ix].fd != fd) { - ix = res; - pr[ix].fd = (int) ev->ident; - pr[ix].events = (ErtsPollEvents) 0; - } - - if (ev->filter == EVFILT_READ) { - if (fd == wake_fd) { - cleanup_wakeup_pipe(ps); - continue; - } - pr[ix].events |= ERTS_POLL_EV_IN; - } - else if (ev->filter == EVFILT_WRITE) - pr[ix].events |= ERTS_POLL_EV_OUT; - if (ev->flags & (EV_ERROR|EV_EOF)) { - if ((ev->flags & EV_ERROR) && (((int) ev->data) == EBADF)) - pr[ix].events |= ERTS_POLL_EV_NVAL; - else - pr[ix].events |= ERTS_POLL_EV_ERR; - } - if (pr[ix].events) { - if (res == ix) { - ps->fds_status[fd].res_ev_ix = (unsigned short) ix; - res++; - } - } - -#elif ERTS_POLL_USE_DEVPOLL /* --- devpoll ----------------------------- */ - - if (ps->res_events[i].revents) { - int fd = ps->res_events[i].fd; - ErtsPollEvents revents; - if (fd == wake_fd) { - cleanup_wakeup_pipe(ps); - continue; - } -#if ERTS_POLL_USE_TIMERFD - if (fd == timer_fd) { - continue; - } -#endif - revents = ERTS_POLL_EV_N2E(ps->res_events[i].events); - pr[res].fd = fd; - pr[res].events = revents; - res++; - } - + int fd = ERTS_POLL_RES_GET_FD(&pr[i]); +#ifdef DEBUG_PRINT_MODE + ErtsPollEvents evts = ERTS_POLL_RES_GET_EVTS(pr+i); #endif - } - - return res; -} - -#endif /* ERTS_POLL_USE_KERNEL_POLL */ - -#if ERTS_POLL_USE_FALLBACK - -static int -get_kp_results(ErtsPollSet ps, ErtsPollResFd pr[], int max_res) -{ - int res; + DEBUG_PRINT_FD("trig %s (%s)", ps, fd, + ev2str(evts), #if ERTS_POLL_USE_KQUEUE - struct timespec ts = {0, 0}; + "kqueue" +#elif ERTS_POLL_USE_EPOLL + "epoll" +#else + "/dev/poll" #endif + ); - if (max_res > ps->res_events_len) - grow_res_events(ps, max_res); - - do { -#if ERTS_POLL_USE_EPOLL - res = epoll_wait(ps->kp_fd, ps->res_events, max_res, 0); -#elif ERTS_POLL_USE_KQUEUE - res = kevent(ps->kp_fd, NULL, 0, ps->res_events, max_res, &ts); +#if ERTS_POLL_USE_WAKEUP_PIPE + if (fd == wake_fd) { + cleanup_wakeup_pipe(ps); + ERTS_POLL_RES_SET_EVTS(&pr[i], ERTS_POLL_EV_NONE); + if (n == 1) + return 0; + } #endif - } while (res < 0 && errno == EINTR); - - if (res < 0) { - fatal_error("%s:%d: %s() failed: %s (%d)\n", - __FILE__, __LINE__, -#if ERTS_POLL_USE_EPOLL - "epoll_wait", -#elif ERTS_POLL_USE_KQUEUE - "kevent", +#if !ERTS_POLL_USE_CONCURRENT_UPDATE + else { + /* Reset the events to emulate ONESHOT semantics */ + ps->fds_status[fd].events = 0; + enqueue_update_request(ps, fd); + } #endif - erl_errno_id(errno), errno); } - return save_kp_result(ps, pr, max_res, res); + return res; +#else + ASSERT(chk_fds_res <= max_res); + return chk_fds_res; +#endif } -#endif /* ERTS_POLL_USE_FALLBACK */ - - +#else /* !ERTS_POLL_USE_KERNEL_POLL */ static ERTS_INLINE int -save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, - int chk_fds_res, int ebadf) +ERTS_POLL_EXPORT(save_result)(ErtsPollSet *ps, ErtsPollResFd pr[], int max_res, int chk_fds_res, int ebadf) { -#if ERTS_POLL_USE_DEVPOLL - return save_kp_result(ps, pr, max_res, chk_fds_res); -#elif ERTS_POLL_USE_FALLBACK - if (!ps->fallback_used) - return save_kp_result(ps, pr, max_res, chk_fds_res); - else -#endif /* ERTS_POLL_USE_FALLBACK */ - { - #if ERTS_POLL_USE_POLL /* --- poll -------------------------------- */ int res = 0; -#if !ERTS_POLL_USE_FALLBACK int wake_fd = ps->wake_fds[0]; -#endif int i, first_ix, end_ix; /* @@ -1828,23 +1379,30 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, if (ps->poll_fds[i].revents != (short) 0) { int fd = ps->poll_fds[i].fd; ErtsPollEvents revents; -#if ERTS_POLL_USE_FALLBACK - if (fd == ps->kp_fd) { - res += get_kp_results(ps, &pr[res], max_res-res); - i++; - continue; - } -#else if (fd == wake_fd) { cleanup_wakeup_pipe(ps); i++; continue; } -#endif revents = pollev2ev(ps->poll_fds[i].revents); - pr[res].fd = fd; - pr[res].events = revents; + ERTS_POLL_RES_SET_FD(&pr[res], fd); + ERTS_POLL_RES_SET_EVTS(&pr[res], revents); + + /* If an fd returns as error, we may want to check the + update_requests queue to see if it has been reset + before delivering the result?!?! This should allow + the user to do driver_dselect + close without waiting + for stop_select... */ + + DEBUG_PRINT_FD("trig %s (poll)", ps, ERTS_POLL_RES_GET_FD(&pr[res]), + ev2str(ERTS_POLL_RES_GET_EVTS(&pr[res]))); + res++; + + /* Clear the events for this fd in order to mimic + how epoll ONESHOT works */ + ps->fds_status[fd].events = 0; + enqueue_update_request(ps, fd); } i++; } @@ -1860,9 +1418,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, #elif ERTS_POLL_USE_SELECT /* --- select ------------------------------ */ int res = 0; -#if !ERTS_POLL_USE_FALLBACK int wake_fd = ps->wake_fds[0]; -#endif int fd, first_fd, end_fd; /* @@ -1875,29 +1431,23 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, if (!ebadf) { while (1) { while (fd < end_fd && res < max_res) { - - pr[res].events = (ErtsPollEvents) 0; + ErtsPollEvents events = 0; if (ERTS_FD_ISSET(fd, &ps->res_input_fds)) { -#if ERTS_POLL_USE_FALLBACK - if (fd == ps->kp_fd) { - res += get_kp_results(ps, &pr[res], max_res-res); - fd++; - continue; - } -#else if (fd == wake_fd) { cleanup_wakeup_pipe(ps); fd++; continue; } -#endif - pr[res].events |= ERTS_POLL_EV_IN; + events |= ERTS_POLL_EV_IN; } if (ERTS_FD_ISSET(fd, &ps->res_output_fds)) - pr[res].events |= ERTS_POLL_EV_OUT; - if (pr[res].events) { - pr[res].fd = fd; + events |= ERTS_POLL_EV_OUT; + if (events) { + ERTS_POLL_RES_SET_FD(&pr[res], fd); + ERTS_POLL_RES_SET_EVTS(&pr[res], events); res++; + ps->fds_status[fd].events = 0; + enqueue_update_request(ps, fd); } fd++; } @@ -1930,7 +1480,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, if (ps->fds_status[fd].events & ERTS_POLL_EV_OUT) { oset = &ps->res_output_fds; ERTS_FD_ZERO(oset); - ERTS_FD_SET(fd, oset); + ERTS_FD_SET(fd, oset); } do { /* Initiate 'tv' each time; @@ -1939,49 +1489,31 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, sres = ERTS_SELECT(ps->max_fd+1, iset, oset, NULL, &tv); } while (sres < 0 && errno == EINTR); if (sres < 0) { -#if ERTS_POLL_USE_FALLBACK - if (fd == ps->kp_fd) { - res += get_kp_results(ps, - &pr[res], - max_res-res); - fd++; - continue; - } -#else if (fd == wake_fd) { cleanup_wakeup_pipe(ps); fd++; continue; } -#endif - pr[res].fd = fd; - pr[res].events = ERTS_POLL_EV_NVAL; + ERTS_POLL_RES_SET_FD(&pr[res], fd); + ERTS_POLL_RES_SET_EVTS(&pr[res], ERTS_POLL_EV_NVAL); res++; } else if (sres > 0) { - pr[res].fd = fd; + ErtsPollEvents events = 0; + ERTS_POLL_RES_SET_FD(&pr[res], fd); if (iset && ERTS_FD_ISSET(fd, iset)) { -#if ERTS_POLL_USE_FALLBACK - if (fd == ps->kp_fd) { - res += get_kp_results(ps, - &pr[res], - max_res-res); - fd++; - continue; - } -#else if (fd == wake_fd) { cleanup_wakeup_pipe(ps); fd++; continue; } -#endif - pr[res].events |= ERTS_POLL_EV_IN; + events |= ERTS_POLL_EV_IN; } if (oset && ERTS_FD_ISSET(fd, oset)) { - pr[res].events |= ERTS_POLL_EV_OUT; + events |= ERTS_POLL_EV_OUT; } - ASSERT(pr[res].events); + ASSERT(events); + ERTS_POLL_RES_SET_EVTS(&pr[res], events); res++; } } @@ -1996,353 +1528,145 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, } ps->next_sel_fd = fd; return res; -#endif - } +#endif /* ERTS_POLL_USE_SELECT */ } -static ERTS_INLINE ErtsMonotonicTime -get_timeout(ErtsPollSet ps, - int resolution, - ErtsMonotonicTime timeout_time) -{ - ErtsMonotonicTime timeout, save_timeout_time; - - if (timeout_time == ERTS_POLL_NO_TIMEOUT) { - save_timeout_time = ERTS_MONOTONIC_TIME_MIN; - timeout = 0; - } - else { - ErtsMonotonicTime diff_time, current_time; - current_time = erts_get_monotonic_time(NULL); - diff_time = timeout_time - current_time; - if (diff_time <= 0) { - save_timeout_time = ERTS_MONOTONIC_TIME_MIN; - timeout = 0; - } - else { - save_timeout_time = current_time; - switch (resolution) { - case 1000: - /* Round up to nearest even milli second */ - timeout = ERTS_MONOTONIC_TO_MSEC(diff_time - 1) + 1; - if (timeout > (ErtsMonotonicTime) INT_MAX) - timeout = (ErtsMonotonicTime) INT_MAX; - save_timeout_time += ERTS_MSEC_TO_MONOTONIC(timeout); - timeout -= ERTS_PREMATURE_TIMEOUT(timeout, 1000); - break; - case 1000000: - /* Round up to nearest even micro second */ - timeout = ERTS_MONOTONIC_TO_USEC(diff_time - 1) + 1; - save_timeout_time += ERTS_USEC_TO_MONOTONIC(timeout); - timeout -= ERTS_PREMATURE_TIMEOUT(timeout, 1000*1000); - break; - case 1000000000: - /* Round up to nearest even nano second */ - timeout = ERTS_MONOTONIC_TO_NSEC(diff_time - 1) + 1; - save_timeout_time += ERTS_NSEC_TO_MONOTONIC(timeout); - timeout -= ERTS_PREMATURE_TIMEOUT(timeout, 1000*1000*1000); - break; - default: - ERTS_INTERNAL_ERROR("Invalid resolution"); - timeout = 0; - save_timeout_time = 0; - break; - } - } - } - set_timeout_time(ps, save_timeout_time); - return timeout; -} - -#if ERTS_POLL_USE_SELECT - -static ERTS_INLINE int -get_timeout_timeval(ErtsPollSet ps, - SysTimeval *tvp, - ErtsMonotonicTime timeout_time) -{ - ErtsMonotonicTime timeout = get_timeout(ps, - 1000*1000, - timeout_time); - - if (!timeout) { - tvp->tv_sec = 0; - tvp->tv_usec = 0; - - return 0; - } - else { - ErtsMonotonicTime sec = timeout/(1000*1000); - tvp->tv_sec = sec; - tvp->tv_usec = timeout - sec*(1000*1000); - - ASSERT(tvp->tv_sec >= 0); - ASSERT(tvp->tv_usec >= 0); - ASSERT(tvp->tv_usec < 1000*1000); - - return !0; - } - -} - -#endif - -#if ERTS_POLL_USE_KQUEUE || (ERTS_POLL_USE_POLL && defined(HAVE_PPOLL)) || ERTS_POLL_USE_TIMERFD - -static ERTS_INLINE int -get_timeout_timespec(ErtsPollSet ps, - struct timespec *tsp, - ErtsMonotonicTime timeout_time) -{ - ErtsMonotonicTime timeout = get_timeout(ps, - 1000*1000*1000, - timeout_time); - - if (!timeout) { - tsp->tv_sec = 0; - tsp->tv_nsec = 0; - return 0; - } - else { - ErtsMonotonicTime sec = timeout/(1000*1000*1000); - tsp->tv_sec = sec; - tsp->tv_nsec = timeout - sec*(1000*1000*1000); - - ASSERT(tsp->tv_sec >= 0); - ASSERT(tsp->tv_nsec >= 0); - ASSERT(tsp->tv_nsec < 1000*1000*1000); - - return !0; - } -} - -#endif - -#if ERTS_POLL_USE_TIMERFD - -static ERTS_INLINE int -get_timeout_itimerspec(ErtsPollSet ps, - struct itimerspec *itsp, - ErtsMonotonicTime timeout_time) -{ - - itsp->it_interval.tv_sec = 0; - itsp->it_interval.tv_nsec = 0; - - return get_timeout_timespec(ps, &itsp->it_value, timeout_time); -} - -#endif +#endif /* !ERTS_POLL_USE_KERNEL_POLL */ static ERTS_INLINE int -check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res) +check_fd_events(ErtsPollSet *ps, ErtsPollResFd pr[], int do_wait, int max_res) { int res; - ERTS_MSACC_PUSH_STATE_M(); - if (erts_atomic_read_nob(&ps->no_of_user_fds) == 0 - && timeout_time == ERTS_POLL_NO_TIMEOUT) { - /* Nothing to poll and zero timeout; done... */ - return 0; - } - else { - int timeout; -#if ERTS_POLL_USE_FALLBACK - if (!(ps->fallback_used = ERTS_POLL_NEED_FALLBACK(ps))) { - -#if ERTS_POLL_USE_EPOLL /* --- epoll ------------------------------- */ - if (max_res > ps->res_events_len) - grow_res_events(ps, max_res); -#if ERTS_POLL_USE_TIMERFD - { - struct itimerspec its; - timeout = get_timeout_itimerspec(ps, &its, timeout_time); - if (timeout) { - erts_thr_progress_prepare_wait(NULL); - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); - timerfd_set(ps, &its); - res = epoll_wait(ps->kp_fd, ps->res_events, max_res, -1); - res = timerfd_clear(ps, res, max_res); - } else { - res = epoll_wait(ps->kp_fd, ps->res_events, max_res, 0); - } - } -#else /* !ERTS_POLL_USE_TIMERFD */ - timeout = (int) get_timeout(ps, 1000, timeout_time); - if (timeout) { - erts_thr_progress_prepare_wait(NULL); - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); - } - res = epoll_wait(ps->kp_fd, ps->res_events, max_res, timeout); -#endif /* !ERTS_POLL_USE_TIMERFD */ -#elif ERTS_POLL_USE_KQUEUE /* --- kqueue ------------------------------ */ - struct timespec ts; - if (max_res > ps->res_events_len) - grow_res_events(ps, max_res); - timeout = get_timeout_timespec(ps, &ts, timeout_time); - if (timeout) { - erts_thr_progress_prepare_wait(NULL); - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); - } - res = kevent(ps->kp_fd, NULL, 0, ps->res_events, max_res, &ts); -#endif /* ----------------------------------------- */ - } - else /* use fallback (i.e. poll() or select()) */ -#endif /* ERTS_POLL_USE_FALLBACK */ - { + int timeout = do_wait ? -1 : 0; + DEBUG_PRINT_WAIT("Entering check_fd_events(), do_wait=%d", ps, do_wait); + { +#if ERTS_POLL_USE_EPOLL /* --- epoll ------------------------------- */ + res = epoll_wait(ps->kp_fd, pr, max_res, timeout); + +#elif ERTS_POLL_USE_KQUEUE /* --- kqueue ------------------------------ */ + struct timespec ts = {0, 0}; + struct timespec *tsp = timeout ? NULL : &ts; + res = kevent(ps->kp_fd, NULL, 0, pr, max_res, tsp); +#elif ERTS_POLL_USE_DEVPOLL /* --- devpoll ----------------------------- */ + /* + * The ioctl() will fail with EINVAL on Solaris 10 if dp_nfds + * is set too high. dp_nfds should not be set greater than + * the maximum number of file descriptors in the poll set. + */ + struct dvpoll poll_res; + int nfds = (int) erts_atomic_read_nob(&ps->no_of_user_fds) + 1 /* wakeup pipe */; + poll_res.dp_nfds = nfds < max_res ? nfds : max_res; + poll_res.dp_fds = pr; + poll_res.dp_timeout = timeout; + res = ioctl(ps->kp_fd, DP_POLL, &poll_res); -#if ERTS_POLL_USE_DEVPOLL /* --- devpoll ----------------------------- */ - /* - * The ioctl() will fail with EINVAL on Solaris 10 if dp_nfds - * is set too high. dp_nfds should not be set greater than - * the maximum number of file descriptors in the poll set. - */ - struct dvpoll poll_res; - int nfds = (int) erts_atomic_read_nob(&ps->no_of_user_fds); - nfds++; /* Wakeup pipe */ - timeout = (int) get_timeout(ps, 1000, timeout_time); - poll_res.dp_nfds = nfds < max_res ? nfds : max_res; - if (poll_res.dp_nfds > ps->res_events_len) - grow_res_events(ps, poll_res.dp_nfds); - poll_res.dp_fds = ps->res_events; - if (timeout) { - erts_thr_progress_prepare_wait(NULL); - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); - } - poll_res.dp_timeout = timeout; - res = ioctl(ps->kp_fd, DP_POLL, &poll_res); -#elif ERTS_POLL_USE_POLL && defined(HAVE_PPOLL) /* --- ppoll ---------------- */ - struct timespec ts; - timeout = get_timeout_timespec(ps, &ts, timeout_time); - if (timeout) { - erts_thr_progress_prepare_wait(NULL); - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); - } - res = ppoll(ps->poll_fds, ps->no_poll_fds, &ts, NULL); #elif ERTS_POLL_USE_POLL /* --- poll --------------------------------- */ - timeout = (int) get_timeout(ps, 1000, timeout_time); - if (timeout) { - erts_thr_progress_prepare_wait(NULL); - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); - } - res = poll(ps->poll_fds, ps->no_poll_fds, timeout); + res = poll(ps->poll_fds, ps->no_poll_fds, timeout); + #elif ERTS_POLL_USE_SELECT /* --- select ------------------------------ */ - SysTimeval to; - timeout = get_timeout_timeval(ps, &to, timeout_time); + SysTimeval tv = {0, 0}; + SysTimeval *tvp = timeout ? NULL : &tv; - ERTS_FD_COPY(&ps->input_fds, &ps->res_input_fds); - ERTS_FD_COPY(&ps->output_fds, &ps->res_output_fds); + ERTS_FD_COPY(&ps->input_fds, &ps->res_input_fds); + ERTS_FD_COPY(&ps->output_fds, &ps->res_output_fds); - if (timeout) { - erts_thr_progress_prepare_wait(NULL); - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); - } - res = ERTS_SELECT(ps->max_fd + 1, - &ps->res_input_fds, - &ps->res_output_fds, - NULL, - &to); - if (timeout) { - erts_thr_progress_finalize_wait(NULL); - ERTS_MSACC_POP_STATE_M(); - } - if (res < 0 - && errno == EBADF - && ERTS_POLLSET_HAVE_UPDATE_REQUESTS(ps)) { - /* - * This may have happened because another thread deselected - * a fd in our poll set and then closed it, i.e. the driver - * behaved correctly. We wan't to avoid looking for a bad - * fd, that may even not exist anymore. Therefore, handle - * update requests and try again. - * - * We don't know how much of the timeout is left; therfore, - * we use a zero timeout. If no error occur and no events - * have triggered, we fake an EAGAIN error and let the caller - * restart us. - */ - to.tv_sec = 0; - to.tv_usec = 0; - ERTS_POLLSET_LOCK(ps); - handle_update_requests(ps); - ERTS_POLLSET_UNLOCK(ps); - res = ERTS_SELECT(ps->max_fd + 1, - &ps->res_input_fds, - &ps->res_output_fds, - NULL, - &to); - if (res == 0) { - errno = EAGAIN; - res = -1; - } - } - return res; + res = ERTS_SELECT(ps->max_fd + 1, + &ps->res_input_fds, + &ps->res_output_fds, + NULL, + tvp); #endif /* ----------------------------------------- */ - } - if (timeout) { - erts_thr_progress_finalize_wait(NULL); - ERTS_MSACC_POP_STATE_M(); - } - return res; } + DEBUG_PRINT_WAIT("Leaving check_fd_events(), res=%d", ps, res); + return res; } int -ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, +ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet *ps, ErtsPollResFd pr[], - int *len, - ErtsMonotonicTime timeout_time) + int *len) { - ErtsMonotonicTime to; - int res, no_fds; + int res, no_fds, used_fds = 0; int ebadf = 0; + int do_wait; int ps_locked = 0; + ERTS_MSACC_DECLARE_CACHE(); no_fds = *len; -#ifdef ERTS_POLL_MAX_RES - if (no_fds >= ERTS_POLL_MAX_RES) - no_fds = ERTS_POLL_MAX_RES; -#endif - *len = 0; + ASSERT(no_fds > 0); -#ifdef ERTS_POLL_DEBUG_PRINT - erts_printf("Entering erts_poll_wait(), timeout_time=%bps\n", - timeout_time); -#endif +#if !ERTS_POLL_USE_CONCURRENT_UPDATE + if (ERTS_POLLSET_HAVE_UPDATE_REQUESTS(ps)) { + ERTS_POLLSET_LOCK(ps); + used_fds = handle_update_requests(ps, pr, no_fds); + ERTS_POLLSET_UNLOCK(ps); - if (ERTS_POLLSET_SET_POLLED_CHK(ps)) { - res = EINVAL; /* Another thread is in erts_poll_wait() - on this pollset... */ - goto done; + if (used_fds == no_fds) { + *len = used_fds; + return 0; + } } +#endif - to = (is_woken(ps) - ? ERTS_POLL_NO_TIMEOUT /* Use zero timeout */ - : timeout_time); + do_wait = !is_woken(ps) && used_fds == 0; - if (ERTS_POLLSET_HAVE_UPDATE_REQUESTS(ps)) { - ERTS_POLLSET_LOCK(ps); - handle_update_requests(ps); - ERTS_POLLSET_UNLOCK(ps); + DEBUG_PRINT_WAIT("Entering %s(), do_wait=%d", ps, __FUNCTION__, do_wait); + + if (do_wait) { + erts_thr_progress_prepare_wait(NULL); + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_SLEEP); } while (1) { - res = check_fd_events(ps, to, no_fds); + res = check_fd_events(ps, pr + used_fds, do_wait, no_fds - used_fds); + +#if !ERTS_POLL_USE_CONCURRENT_UPDATE + if (res < 0 + && errno == EBADF + && ERTS_POLLSET_HAVE_UPDATE_REQUESTS(ps)) { + /* + * This may have happened because another thread deselected + * a fd in our poll set and then closed it, i.e. the driver + * behaved correctly. We wan't to avoid looking for a bad + * fd, that may even not exist anymore. Therefore, handle + * update requests and try again. This behaviour should only + * happen when using SELECT as the polling mechanism. + */ + ERTS_POLLSET_LOCK(ps); + used_fds += handle_update_requests(ps, pr + used_fds, no_fds - used_fds); + if (used_fds == no_fds) { + *len = used_fds; + ERTS_POLLSET_UNLOCK(ps); + return 0; + } + res = check_fd_events(ps, pr + used_fds, 0, no_fds - used_fds); + /* Keep the lock over the non-blocking poll in order to not + get any nasty races happening. */ + ERTS_POLLSET_UNLOCK(ps); + if (res == 0) { + errno = EAGAIN; + res = -1; + } + } +#endif + if (res != 0) break; - if (to == ERTS_POLL_NO_TIMEOUT) - break; - if (erts_get_monotonic_time(NULL) >= timeout_time) - break; + if (!do_wait) + break; + } + + if (do_wait) { + erts_thr_progress_finalize_wait(NULL); + ERTS_MSACC_UPDATE_CACHE(); + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_CHECK_IO); } woke_up(ps); - if (res == 0) { - res = ETIMEDOUT; - } - else if (res < 0) { + if (res < 0) { #if ERTS_POLL_USE_SELECT if (errno == EBADF) { ebadf = 1; @@ -2359,26 +1683,21 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, ps_locked = 1; ERTS_POLLSET_LOCK(ps); - no_fds = save_poll_result(ps, pr, no_fds, res, ebadf); + used_fds += ERTS_POLL_EXPORT(save_result)(ps, pr + used_fds, no_fds - used_fds, res, ebadf); #ifdef HARD_DEBUG - check_poll_result(pr, no_fds); + check_poll_result(pr, used_fds); #endif - res = (no_fds == 0 ? (is_interrupted_reset(ps) ? EINTR : EAGAIN) : 0); - *len = no_fds; + res = (used_fds == 0 ? (is_interrupted_reset(ps) ? EINTR : EAGAIN) : 0); + *len = used_fds; } if (ps_locked) ERTS_POLLSET_UNLOCK(ps); - ERTS_POLLSET_UNSET_POLLED(ps); - done: - set_timeout_time(ps, ERTS_MONOTONIC_TIME_MAX); -#ifdef ERTS_POLL_DEBUG_PRINT - erts_printf("Leaving %s = erts_poll_wait()\n", - res == 0 ? "0" : erl_errno_id(res)); -#endif + DEBUG_PRINT_WAIT("Leaving %s = %s(len = %d)", ps, + res == 0 ? "0" : erl_errno_id(res), __FUNCTION__, *len); return res; } @@ -2388,40 +1707,14 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, */ void -ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet ps, int set) +ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet *ps, int set) { +#if !ERTS_POLL_USE_CONCURRENT_UPDATE if (!set) reset_wakeup_state(ps); else - wake_poller(ps, 1, 0); -} - - -/* - * erts_poll_interrupt_timed(): - * If 'set' != 0, interrupt thread blocked in erts_poll_wait() if it - * is not guaranteed that it will timeout before 'msec' milli seconds. - */ -void -ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps, - int set, - ErtsMonotonicTime timeout_time) -{ - if (!set) - reset_wakeup_state(ps); - else { - ErtsMonotonicTime max_wait_time = get_timeout_time(ps); - if (max_wait_time > timeout_time) - wake_poller(ps, 1, 0); -#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS - else { - if (ERTS_POLLSET_IS_POLLED(ps)) - erts_atomic_inc_nob(&ps->no_avoided_wakeups); - erts_atomic_inc_nob(&ps->no_avoided_interrupts); - } - erts_atomic_inc_nob(&ps->no_interrupt_timed); + wake_poller(ps, 1); #endif - } } int @@ -2434,14 +1727,19 @@ ERTS_POLL_EXPORT(erts_poll_max_fds)(void) */ void -ERTS_POLL_EXPORT(erts_poll_init)(void) +ERTS_POLL_EXPORT(erts_poll_init)(int *concurrent_updates) { - erts_mtx_init(&pollsets_lock, "pollsets_lock", NIL, - ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); - pollsets = NULL; errno = 0; + if (concurrent_updates) { +#if ERTS_POLL_USE_CONCURRENT_UPDATE + *concurrent_updates = 1; +#else + *concurrent_updates = 0; +#endif + } + #if !defined(NO_SYSCONF) max_fds = sysconf(_SC_OPEN_MAX); #elif ERTS_POLL_USE_SELECT @@ -2460,37 +1758,28 @@ ERTS_POLL_EXPORT(erts_poll_init)(void) fatal_error("erts_poll_init(): Failed to get max number of files: %s\n", erl_errno_id(errno)); -#ifdef ERTS_POLL_DEBUG_PRINT print_misc_debug_info(); -#endif } -ErtsPollSet -ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) +ErtsPollSet * +ERTS_POLL_EXPORT(erts_poll_create_pollset)(int id) { #if ERTS_POLL_USE_KERNEL_POLL int kp_fd; #endif - ErtsPollSet ps = erts_alloc(ERTS_ALC_T_POLLSET, - sizeof(struct ErtsPollSet_)); + ErtsPollSet *ps = erts_alloc(ERTS_ALC_T_POLLSET, + sizeof(struct ERTS_POLL_EXPORT(erts_pollset))); + ps->id = id; ps->internal_fd_limit = 0; - ps->fds_status = NULL; - ps->fds_status_len = 0; erts_atomic_init_nob(&ps->no_of_user_fds, 0); #if ERTS_POLL_USE_KERNEL_POLL ps->kp_fd = -1; #if ERTS_POLL_USE_EPOLL kp_fd = epoll_create(256); - ps->res_events_len = 0; - ps->res_events = NULL; #elif ERTS_POLL_USE_DEVPOLL kp_fd = open("/dev/poll", O_RDWR); - ps->res_events_len = 0; - ps->res_events = NULL; #elif ERTS_POLL_USE_KQUEUE kp_fd = kqueue(); - ps->res_events_len = 0; - ps->res_events = NULL; #endif if (kp_fd < 0) fatal_error("erts_poll_create_pollset(): Failed to " @@ -2504,10 +1793,6 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) ": %s (%d)\n", erl_errno_id(errno), errno); #endif /* ERTS_POLL_USE_KERNEL_POLL */ -#if ERTS_POLL_USE_BATCH_UPDATE_POLLSET - /* res_events is also used as write buffer */ - grow_res_events(ps, ERTS_POLL_MIN_BATCH_BUF_SIZE); -#endif #if ERTS_POLL_USE_POLL ps->next_poll_fds_ix = 0; ps->no_poll_fds = 0; @@ -2516,9 +1801,6 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) #elif ERTS_POLL_USE_SELECT ps->next_sel_fd = 0; ps->max_fd = -1; -#if ERTS_POLL_USE_FALLBACK - ps->no_select_fds = 0; -#endif #ifdef _DARWIN_UNLIMITED_SELECT ps->input_fds.sz = 0; ps->input_fds.ptr = NULL; @@ -2535,140 +1817,85 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) ERTS_FD_ZERO(&ps->res_output_fds); #endif #endif +#if !ERTS_POLL_USE_CONCURRENT_UPDATE + ps->fds_status = NULL; + ps->fds_status_len = 0; ps->update_requests.next = NULL; ps->update_requests.len = 0; ps->curr_upd_req_block = &ps->update_requests; erts_atomic32_init_nob(&ps->have_update_requests, 0); - erts_atomic32_init_nob(&ps->polled, 0); erts_mtx_init(&ps->mtx, "pollset", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO); - erts_atomic32_init_nob(&ps->wakeup_state, (erts_aint32_t) 0); - create_wakeup_pipe(ps); -#if ERTS_POLL_USE_TIMERFD - create_timerfd(ps); -#endif -#if ERTS_POLL_USE_FALLBACK - if (kp_fd >= ps->fds_status_len) - grow_fds_status(ps, kp_fd); - /* Force kernel poll fd into fallback (poll/select) set */ - ps->fds_status[kp_fd].flags - |= ERTS_POLL_FD_FLG_INFLBCK|ERTS_POLL_FD_FLG_USEFLBCK; - { - int do_wake = 0; - ERTS_POLL_EXPORT(erts_poll_control)(ps, kp_fd, ERTS_POLL_EV_IN, 1, - &do_wake); - } #endif #if ERTS_POLL_USE_KERNEL_POLL if (ps->internal_fd_limit <= kp_fd) ps->internal_fd_limit = kp_fd + 1; ps->kp_fd = kp_fd; #endif - init_timeout_time(ps); -#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS - erts_atomic_init_nob(&ps->no_avoided_wakeups, 0); - erts_atomic_init_nob(&ps->no_avoided_interrupts, 0); - erts_atomic_init_nob(&ps->no_interrupt_timed, 0); +#if !ERTS_POLL_USE_CONCURRENT_UPDATE + erts_atomic32_init_nob(&ps->wakeup_state, (erts_aint32_t) 0); + create_wakeup_pipe(ps); + handle_update_requests(ps, NULL, 0); + cleanup_wakeup_pipe(ps); #endif - handle_update_requests(ps); -#if ERTS_POLL_USE_FALLBACK - ps->fallback_used = 0; +#if ERTS_POLL_USE_KERNEL_POLL && (defined(__DARWIN__) || defined(__APPLE__) && defined(__MACH__)) + { + /* + * Using kqueue on OS X is a mess of brokenness... + * + * On OS X version older than 15.6 (i.e. OS X El Capitan released in July 2015), + * a thread waiting in kevent is not woken if an event is inserted into the kqueue + * by another thread and the event becomes ready. However if a new call to kevent + * is done by the waiting thread, the new event is found. + * + * So on effected OS X versions we could trigger the wakeup pipe so that + * the waiters will be woken and re-issue the kevent. However... + * + * On OS X version older then 16 (i.e. OS X Sierra released in September 2016), + * running the emulator driver_SUITE smp_select testcase consistently causes a + * kernel panic. I don't know why or what events that trigger it. But it seems + * like updates of the pollset while another thread is sleeping in it Creates + * some kind of race that triggers the kernel panic. + * + * So to deal with this, the erts configure check what OS X version is run + * and only enabled kernel poll on OS X 16 or newer. In addition, if someone + * attempts to compile Erlang on OS X 16 and then run it on OS X 15, we do the + * run-time check below to disallow this. + */ + int major, minor, build; + os_version(&major,&minor,&build); + if (major < 16) { + erts_fprintf(stderr,"BROKEN KQUEUE!\n" + "Erlang has been compiled with kernel-poll support,\n" + "but this OS X version is known to have kernel bugs\n" + "when using kernel-poll. You have two options:\n" + " 1) update to a newer OS X version (OS X Sierra or newer)\n" + " 2) recompile erlang without kernel-poll support\n"); + erts_exit(1, ""); + } + } #endif erts_atomic_set_nob(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */ - erts_mtx_lock(&pollsets_lock); - ps->next = pollsets; - pollsets = ps; - erts_mtx_unlock(&pollsets_lock); - return ps; } -void -ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps) -{ - - if (ps->fds_status) - erts_free(ERTS_ALC_T_FD_STATUS, (void *) ps->fds_status); - -#if ERTS_POLL_USE_EPOLL - if (ps->kp_fd >= 0) - close(ps->kp_fd); - if (ps->res_events) - erts_free(ERTS_ALC_T_POLL_RES_EVS, (void *) ps->res_events); -#elif ERTS_POLL_USE_DEVPOLL - if (ps->kp_fd >= 0) - close(ps->kp_fd); - if (ps->res_events) - erts_free(ERTS_ALC_T_POLL_RES_EVS, (void *) ps->res_events); -#elif ERTS_POLL_USE_POLL - if (ps->poll_fds) - erts_free(ERTS_ALC_T_POLL_FDS, (void *) ps->poll_fds); -#elif ERTS_POLL_USE_SELECT -#ifdef _DARWIN_UNLIMITED_SELECT - if (ps->input_fds.ptr) - erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->input_fds.ptr); - if (ps->res_input_fds.ptr) - erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->res_input_fds.ptr); - if (ps->output_fds.ptr) - erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->output_fds.ptr); - if (ps->res_output_fds.ptr) - erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->res_output_fds.ptr); -#endif -#endif - { - ErtsPollSetUpdateRequestsBlock *urqbp = ps->update_requests.next; - while (urqbp) { - ErtsPollSetUpdateRequestsBlock *free_urqbp = urqbp; - urqbp = urqbp->next; - free_update_requests_block(ps, free_urqbp); - } - } - erts_mtx_destroy(&ps->mtx); - if (ps->wake_fds[0] >= 0) - close(ps->wake_fds[0]); - if (ps->wake_fds[1] >= 0) - close(ps->wake_fds[1]); -#if ERTS_POLL_USE_TIMERFD - if (ps->timer_fd >= 0) - close(ps->timer_fd); -#endif - - erts_mtx_lock(&pollsets_lock); - if (ps == pollsets) - pollsets = pollsets->next; - else { - ErtsPollSet prev_ps; - for (prev_ps = pollsets; ps != prev_ps->next; prev_ps = prev_ps->next) - ; - ASSERT(ps == prev_ps->next); - prev_ps->next = ps->next; - } - erts_mtx_unlock(&pollsets_lock); - - erts_free(ERTS_ALC_T_POLLSET, (void *) ps); -} - /* * --- Info ------------------------------------------------------------------ */ void -ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) +ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet *ps, ErtsPollInfo *pip) { +#if !ERTS_POLL_USE_CONCURRENT_UPDATE int pending_updates; +#endif Uint size = 0; ERTS_POLLSET_LOCK(ps); - size += sizeof(struct ErtsPollSet_); + size += sizeof(struct ERTS_POLL_EXPORT(erts_pollset)); +#if !ERTS_POLL_USE_CONCURRENT_UPDATE size += ps->fds_status_len*sizeof(ErtsFdStatus); - -#if ERTS_POLL_USE_EPOLL - size += ps->res_events_len*sizeof(struct epoll_event); -#elif ERTS_POLL_USE_DEVPOLL - size += ps->res_events_len*sizeof(struct pollfd); -#elif ERTS_POLL_USE_KQUEUE - size += ps->res_events_len*sizeof(struct kevent); #endif #if ERTS_POLL_USE_POLL @@ -2680,6 +1907,7 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) #endif #endif +#if !ERTS_POLL_USE_CONCURRENT_UPDATE { ErtsPollSetUpdateRequestsBlock *urqbp = ps->update_requests.next; pending_updates = ps->update_requests.len; @@ -2689,8 +1917,9 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) urqbp = urqbp->next; } } +#endif - pip->primary = + pip->primary = #if ERTS_POLL_USE_KQUEUE "kqueue" #elif ERTS_POLL_USE_EPOLL @@ -2704,17 +1933,7 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) #endif ; - pip->fallback = -#if !ERTS_POLL_USE_FALLBACK - NULL -#elif ERTS_POLL_USE_POLL - "poll" -#elif ERTS_POLL_USE_SELECT - "select" -#endif - ; - - pip->kernel_poll = + pip->kernel_poll = #if !ERTS_POLL_USE_KERNEL_POLL NULL #elif ERTS_POLL_USE_KQUEUE @@ -2729,40 +1948,21 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) pip->memory_size = size; pip->poll_set_size = (int) erts_atomic_read_nob(&ps->no_of_user_fds); +#if !ERTS_POLL_USE_CONCURRENT_UPDATE pip->poll_set_size++; /* Wakeup pipe */ -#if ERTS_POLL_USE_TIMERFD - pip->poll_set_size++; /* timerfd */ -#endif - - pip->fallback_poll_set_size = -#if !ERTS_POLL_USE_FALLBACK - 0 -#elif ERTS_POLL_USE_POLL - ps->no_poll_fds -#elif ERTS_POLL_USE_SELECT - ps->no_select_fds -#endif - ; - -#if ERTS_POLL_USE_FALLBACK - /* If only kp_fd is in fallback poll set we don't use fallback... */ - if (pip->fallback_poll_set_size == 1) - pip->fallback_poll_set_size = 0; - else - pip->poll_set_size++; /* kp_fd */ #endif pip->lazy_updates = +#if !ERTS_POLL_USE_CONCURRENT_UPDATE 1 +#else + 0 +#endif ; pip->pending_updates = +#if !ERTS_POLL_USE_CONCURRENT_UPDATE pending_updates - ; - - pip->batch_updates = -#if ERTS_POLL_USE_BATCH_UPDATE_POLLSET - 1 #else 0 #endif @@ -2776,13 +1976,23 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) #endif ; - pip->max_fds = max_fds; + pip->is_fallback = +#if ERTS_POLL_IS_FALLBACK + 1 +#else + 0 +#endif + ; -#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS - pip->no_avoided_wakeups = erts_atomic_read_nob(&ps->no_avoided_wakeups); - pip->no_avoided_interrupts = erts_atomic_read_nob(&ps->no_avoided_interrupts); - pip->no_interrupt_timed = erts_atomic_read_nob(&ps->no_interrupt_timed); + pip->batch_updates = +#if ERTS_POLL_USE_DEVPOLL + 1 +#else + 0 #endif + ; + + pip->max_fds = max_fds; ERTS_POLLSET_UNLOCK(ps); @@ -2818,35 +2028,60 @@ fatal_error(char *format, ...) abort(); } -static void -fatal_error_async_signal_safe(char *error_str) +/* + * --- Debug ----------------------------------------------------------------- + */ + +#if ERTS_POLL_USE_EPOLL +uint32_t epoll_events(int kp_fd, int fd) { - if (ERTS_SOMEONE_IS_CRASH_DUMPING || ERTS_GOT_SIGUSR1) { - /* See comment above in fatal_error() */ - return; + /* For epoll we read the information about what is selected upon from the proc fs.*/ + char fname[30]; + FILE *f; + unsigned int pos, flags, mnt_id; + int line = 0; + sprintf(fname,"/proc/%d/fdinfo/%d",getpid(), kp_fd); + f = fopen(fname,"r"); + if (!f) { + fprintf(stderr,"failed to open file %s, errno = %d\n", fname, errno); + ASSERT(0); + return 0; } - if (error_str) { - int len = 0; - while (error_str[len]) - len++; - if (len) { - /* async signal safe */ - erts_silence_warn_unused_result(write(2, error_str, len)); - } + if (fscanf(f,"pos:\t%x\nflags:\t%x", &pos, &flags) != 2) { + fprintf(stderr,"failed to parse file %s, errno = %d\n", fname, errno); + ASSERT(0); + return 0; } - abort(); + if (fscanf(f,"\nmnt_id:\t%x\n", &mnt_id)); + line += 3; + while (!feof(f)) { + /* tfd: 10 events: 40000019 data: 180000000a */ + int ev_fd; + uint32_t events; + uint64_t data; + if (fscanf(f,"tfd:%d events:%x data:%lx\n", &ev_fd, &events, &data) != 3) { + fprintf(stderr,"failed to parse file %s on line %d, errno = %d\n", fname, + line, + errno); + return 0; + } + if (fd == ev_fd) { + fclose(f); + return events; + } + } + fclose(f); + return 0; } - -/* - * --- Debug ----------------------------------------------------------------- - */ +#endif void -ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet ps, +ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet *ps, ErtsPollEvents ev[], int len) { int fd; +#if !ERTS_POLL_USE_CONCURRENT_UPDATE ERTS_POLLSET_LOCK(ps); for (fd = 0; fd < len; fd++) { if (fd >= ps->fds_status_len) @@ -2855,9 +2090,6 @@ ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet ps, ev[fd] = ps->fds_status[fd].events; if ( fd == ps->wake_fds[0] || fd == ps->wake_fds[1] || -#if ERTS_POLL_USE_TIMERFD - fd == ps->timer_fd || -#endif #if ERTS_POLL_USE_KERNEL_POLL fd == ps->kp_fd || #endif @@ -2866,7 +2098,50 @@ ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet ps, } } ERTS_POLLSET_UNLOCK(ps); +#elif ERTS_POLL_USE_EPOLL + /* For epoll we read the information about what is selected upon from the proc fs.*/ + char fname[30]; + FILE *f; + unsigned int pos, flags, mnt_id; + int line = 0; + sprintf(fname,"/proc/%d/fdinfo/%d",getpid(), ps->kp_fd); + for (fd = 0; fd < len; fd++) + ev[fd] = ERTS_POLL_EV_NONE; + f = fopen(fname,"r"); + if (!f) { + fprintf(stderr,"failed to open file %s, errno = %d\n", fname, errno); + return; + } + if (fscanf(f,"pos:\t%x\nflags:\t%x", &pos, &flags) != 2) { + fprintf(stderr,"failed to parse file %s, errno = %d\n", fname, errno); + ASSERT(0); + return; + } + if (fscanf(f,"\nmnt_id:\t%x\n", &mnt_id)); + line += 3; + while (!feof(f)) { + /* tfd: 10 events: 40000019 data: 180000000a */ + int fd; + uint32_t events; + uint64_t data; + if (fscanf(f,"tfd:%d events:%x data:%lx\n", &fd, &events, &data) != 3) { + fprintf(stderr,"failed to parse file %s on line %d, errno = %d\n", + fname, line, errno); + ASSERT(0); + return; + } + data &= 0xFFFFFFFF; + ASSERT(fd == data); + /* Events are the events that are being monitored, which of course include + error and hup events, but we are only interested in IN/OUT events */ + ev[fd] = (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT) & ERTS_POLL_EV_N2E(events); + line++; + } +#else + for (fd = 0; fd < len; fd++) + ev[fd] = ERTS_POLL_EV_NONE; +#endif } #ifdef HARD_DEBUG @@ -2886,10 +2161,10 @@ check_poll_result(ErtsPollResFd pr[], int len) } -#if ERTS_POLL_USE_DEVPOLL +#if ERTS_POLL_USE_DEVPOLL && defined(DEBUG) static void -check_poll_status(ErtsPollSet ps) +check_poll_status(ErtsPollSet *ps) { int i; for (i = 0; i < ps->fds_status_len; i++) { @@ -2921,30 +2196,24 @@ check_poll_status(ErtsPollSet ps) #endif /* ERTS_POLL_USE_DEVPOLL */ #endif /* HARD_DEBUG */ -#ifdef ERTS_POLL_DEBUG_PRINT static void print_misc_debug_info(void) { - erts_printf("erts_poll using: %s lazy_updates:%s batch_updates:%s\n", +#if ERTS_POLL_DEBUG_PRINT + erts_printf("erts_poll using: %s lazy_updates:%s\n", #if ERTS_POLL_USE_KQUEUE "kqueue" #elif ERTS_POLL_USE_EPOLL "epoll" #elif ERTS_POLL_USE_DEVPOLL "/dev/poll" -#endif -#if ERTS_POLL_USE_FALLBACK - "-" -#endif -#if ERTS_POLL_USE_POLL +#elif ERTS_POLL_USE_POLL "poll" #elif ERTS_POLL_USE_SELECT "select" #endif , - "true" - , -#if ERTS_POLL_USE_BATCH_UPDATE_POLLSET +#if !ERTS_POLL_USE_CONCURRENT_UPDATE "true" #else "false" @@ -2963,29 +2232,20 @@ print_misc_debug_info(void) #ifdef FD_SETSIZE erts_printf("FD_SETSIZE=%d\n", FD_SETSIZE); #endif -} - #endif +} #ifdef ERTS_ENABLE_LOCK_COUNT -static void erts_lcnt_enable_pollset_lock_count(ErtsPollSet pollset, int enable) { +void ERTS_POLL_EXPORT(erts_lcnt_enable_pollset_lock_count)(ErtsPollSet *pollset, int enable) +{ +#if !ERTS_POLL_USE_CONCURRENT_UPDATE if(enable) { erts_lcnt_install_new_lock_info(&pollset->mtx.lcnt, "pollset_rm", NIL, ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO); } else { erts_lcnt_uninstall(&pollset->mtx.lcnt); } -} - -void ERTS_POLL_EXPORT(erts_lcnt_update_pollset_locks)(int enable) { - ErtsPollSet iterator; - - erts_mtx_lock(&pollsets_lock); - - for(iterator = pollsets; iterator != NULL; iterator = iterator->next) { - erts_lcnt_enable_pollset_lock_count(iterator, enable); - } - - erts_mtx_unlock(&pollsets_lock); +#endif + return; } #endif diff --git a/erts/emulator/sys/common/erl_poll.h b/erts/emulator/sys/common/erl_poll.h index a57dc51e5b..e9a667cac1 100644 --- a/erts/emulator/sys/common/erl_poll.h +++ b/erts/emulator/sys/common/erl_poll.h @@ -18,11 +18,31 @@ * %CopyrightEnd% */ -/* - * Description: Poll interface suitable for ERTS with or without - * SMP support. +/** + * @description: Poll interface suitable for ERTS with SMP support. + * + * @author: Rickard Green + * @author: Lukas Larsson + * + * This header file exports macros and functions that are used to + * react to I/O polling events from file descriptors or wait-able + * objects. The API exported is the following: * - * Author: Rickard Green + * defines: + * ERTS_POLL_EV_NONE - No events have been set. This is not the same as 0. + * ERTS_POLL_EV_IN - Represent an IN event + * ERTS_POLL_EV_OUT - Represent an OUT event + * ERTS_POLL_EV_ERR - Represent an error event + * ERTS_POLL_EV_NVAL - Represent an invalid event + * + * macro functions: + * ErtsSysFdType ERTS_POLL_RES_GET_FD(ErtsPollResFd *evt); + * void ERTS_POLL_RES_SET_FD(ErtsPollResFd *evt, ErtsSysFdType fd); + * ErtsPollEvents ERTS_POLL_RES_GET_EVTS(ErtsPollResFd *evt) + * void ERTS_POLL_RES_SET_EVTS(ErtsPollResFd *evt, ErtsPollEvents fd); + * + * functions: + * See erl_poll_api.h */ #ifndef ERL_POLL_H__ @@ -32,33 +52,27 @@ #define ERTS_POLL_NO_TIMEOUT ERTS_MONOTONIC_TIME_MIN -#if 0 -#define ERTS_POLL_COUNT_AVOIDED_WAKEUPS -#endif - #ifdef ERTS_ENABLE_KERNEL_POLL -# if defined(ERTS_KERNEL_POLL_VERSION) -# define ERTS_POLL_EXPORT(FUNC) FUNC ## _kp +# undef ERTS_ENABLE_KERNEL_POLL +# define ERTS_ENABLE_KERNEL_POLL 1 +# if defined(ERTS_NO_KERNEL_POLL_VERSION) +# define ERTS_POLL_EXPORT(FUNC) FUNC ## _flbk +# undef ERTS_NO_KERNEL_POLL_VERSION +# define ERTS_NO_KERNEL_POLL_VERSION 1 +# define ERTS_KERNEL_POLL_VERSION 0 # else -# define ERTS_POLL_EXPORT(FUNC) FUNC ## _nkp -# undef ERTS_POLL_DISABLE_KERNEL_POLL -# define ERTS_POLL_DISABLE_KERNEL_POLL +# undef ERTS_KERNEL_POLL_VERSION +# define ERTS_KERNEL_POLL_VERSION 1 +# define ERTS_NO_KERNEL_POLL_VERSION 0 +# define ERTS_POLL_EXPORT(FUNC) FUNC # endif #else # define ERTS_POLL_EXPORT(FUNC) FUNC -# undef ERTS_POLL_DISABLE_KERNEL_POLL -# define ERTS_POLL_DISABLE_KERNEL_POLL -#endif - -#ifdef ERTS_POLL_DISABLE_KERNEL_POLL -# undef HAVE_SYS_EPOLL_H -# undef HAVE_SYS_EVENT_H -# undef HAVE_SYS_DEVPOLL_H +# define ERTS_ENABLE_KERNEL_POLL 0 +# define ERTS_NO_KERNEL_POLL_VERSION 1 +# define ERTS_KERNEL_POLL_VERSION 0 #endif -#undef ERTS_POLL_USE_KERNEL_POLL -#define ERTS_POLL_USE_KERNEL_POLL 0 - #undef ERTS_POLL_USE_KQUEUE #define ERTS_POLL_USE_KQUEUE 0 #undef ERTS_POLL_USE_EPOLL @@ -70,68 +84,96 @@ #undef ERTS_POLL_USE_SELECT #define ERTS_POLL_USE_SELECT 0 -#if defined(HAVE_SYS_EVENT_H) -# undef ERTS_POLL_USE_KQUEUE -# define ERTS_POLL_USE_KQUEUE 1 -# undef ERTS_POLL_USE_KERNEL_POLL -# define ERTS_POLL_USE_KERNEL_POLL 1 -#elif defined(HAVE_SYS_EPOLL_H) -# undef ERTS_POLL_USE_EPOLL -# define ERTS_POLL_USE_EPOLL 1 -# undef ERTS_POLL_USE_KERNEL_POLL -# define ERTS_POLL_USE_KERNEL_POLL 1 -#elif defined(HAVE_SYS_DEVPOLL_H) -# undef ERTS_POLL_USE_DEVPOLL -# define ERTS_POLL_USE_DEVPOLL 1 -# undef ERTS_POLL_USE_KERNEL_POLL -# define ERTS_POLL_USE_KERNEL_POLL 1 +/* Defines which structure that erts_poll_wait should use to wait with + and how events should be represented */ +#define ERTS_POLL_USE_EPOLL_EVS 0 +#define ERTS_POLL_USE_KQUEUE_EVS 0 +#define ERTS_POLL_USE_DEVPOLL_EVS 0 +#define ERTS_POLL_USE_POLL_EVS 0 +#define ERTS_POLL_USE_SELECT_EVS 0 + +#define ERTS_POLL_USE_KERNEL_POLL ERTS_KERNEL_POLL_VERSION + +#if ERTS_ENABLE_KERNEL_POLL +# if defined(HAVE_SYS_EVENT_H) +# undef ERTS_POLL_USE_KQUEUE_EVS +# define ERTS_POLL_USE_KQUEUE_EVS 1 +# undef ERTS_POLL_USE_KQUEUE +# define ERTS_POLL_USE_KQUEUE ERTS_KERNEL_POLL_VERSION +# elif defined(HAVE_SYS_EPOLL_H) +# undef ERTS_POLL_USE_EPOLL_EVS +# define ERTS_POLL_USE_EPOLL_EVS 1 +# undef ERTS_POLL_USE_EPOLL +# define ERTS_POLL_USE_EPOLL ERTS_KERNEL_POLL_VERSION +# elif defined(HAVE_SYS_DEVPOLL_H) +# undef ERTS_POLL_USE_DEVPOLL_EVS +# define ERTS_POLL_USE_DEVPOLL_EVS 1 +# undef ERTS_POLL_USE_DEVPOLL +# define ERTS_POLL_USE_DEVPOLL ERTS_KERNEL_POLL_VERSION +# else +# error "Missing kernel poll implementation of erts_poll()" +# endif #endif -#define ERTS_POLL_USE_FALLBACK (ERTS_POLL_USE_KQUEUE || ERTS_POLL_USE_EPOLL) - -#if !ERTS_POLL_USE_KERNEL_POLL || ERTS_POLL_USE_FALLBACK +#if ERTS_NO_KERNEL_POLL_VERSION # if defined(ERTS_USE_POLL) +# undef ERTS_POLL_USE_POLL_EVS +# define ERTS_POLL_USE_POLL_EVS 1 # undef ERTS_POLL_USE_POLL # define ERTS_POLL_USE_POLL 1 # elif !defined(__WIN32__) +# undef ERTS_POLL_USE_SELECT_EVS +# define ERTS_POLL_USE_SELECT_EVS 1 # undef ERTS_POLL_USE_SELECT # define ERTS_POLL_USE_SELECT 1 # endif #endif -#define ERTS_POLL_USE_TIMERFD 0 +#define ERTS_POLL_USE_FALLBACK (ERTS_POLL_USE_KQUEUE || ERTS_POLL_USE_EPOLL) typedef Uint32 ErtsPollEvents; -#undef ERTS_POLL_EV_E2N + +typedef enum { + ERTS_POLL_OP_ADD = 0, /* Add the FD to the pollset */ + ERTS_POLL_OP_MOD = 1, /* Modify the FD in the pollset */ + ERTS_POLL_OP_DEL = 2 /* Delete the FD from the pollset */ +} ErtsPollOp; + +#define op2str(op) (op == ERTS_POLL_OP_ADD ? "add" : \ + (op == ERTS_POLL_OP_MOD ? "mod" : "del")) #if defined(__WIN32__) /* --- win32 --------------------------------------- */ -#define ERTS_POLL_EV_IN 1 -#define ERTS_POLL_EV_OUT 2 -#define ERTS_POLL_EV_ERR 4 -#define ERTS_POLL_EV_NVAL 8 +#define ERTS_POLL_EV_IN 1 +#define ERTS_POLL_EV_OUT 2 +#define ERTS_POLL_EV_ERR 4 +#define ERTS_POLL_EV_NVAL 8 -#elif ERTS_POLL_USE_EPOLL /* --- epoll ------------------------------- */ +#define ERTS_POLL_EV_E2N(EV) (EV) +#define ERTS_POLL_EV_N2E(EV) (EV) -#include <sys/epoll.h> +#elif ERTS_POLL_USE_EPOLL_EVS /* --- epoll ------------------------------- */ -#ifdef HAVE_SYS_TIMERFD_H -#include <sys/timerfd.h> -#undef ERTS_POLL_USE_TIMERFD -#define ERTS_POLL_USE_TIMERFD 1 -#endif +#include <sys/epoll.h> #define ERTS_POLL_EV_E2N(EV) \ ((uint32_t) (EV)) #define ERTS_POLL_EV_N2E(EV) \ - ((ErtsPollEvents) (EV)) + ((ErtsPollEvents) (EV) & ~EPOLLONESHOT) #define ERTS_POLL_EV_IN ERTS_POLL_EV_N2E(EPOLLIN) #define ERTS_POLL_EV_OUT ERTS_POLL_EV_N2E(EPOLLOUT) #define ERTS_POLL_EV_NVAL ERTS_POLL_EV_N2E(EPOLLET) #define ERTS_POLL_EV_ERR ERTS_POLL_EV_N2E(EPOLLERR|EPOLLHUP) -#elif ERTS_POLL_USE_DEVPOLL /* --- devpoll ----------------------------- */ +typedef struct epoll_event ErtsPollResFd; + +#define ERTS_POLL_RES_GET_FD(evt) ((ErtsSysFdType)((evt)->data.fd)) +#define ERTS_POLL_RES_SET_FD(evt, ident) (evt)->data.fd = ident +#define ERTS_POLL_RES_GET_EVTS(evt) ERTS_POLL_EV_N2E((evt)->events) +#define ERTS_POLL_RES_SET_EVTS(evt, evts) (evt)->events = ERTS_POLL_EV_E2N(evts) + +#elif ERTS_POLL_USE_DEVPOLL_EVS /* --- devpoll ----------------------------- */ #include <sys/devpoll.h> @@ -145,12 +187,37 @@ typedef Uint32 ErtsPollEvents; #define ERTS_POLL_EV_NVAL ERTS_POLL_EV_N2E(POLLNVAL) #define ERTS_POLL_EV_ERR ERTS_POLL_EV_N2E(POLLERR|POLLHUP) -#elif ERTS_POLL_USE_KQUEUE /* --- kqueue ------------------------------ */ +typedef struct pollfd ErtsPollResFd; + +#define ERTS_POLL_RES_GET_FD(evt) ((ErtsSysFdType)((evt)->fd)) +#define ERTS_POLL_RES_SET_FD(evt, ident) (evt)->fd = ident +#define ERTS_POLL_RES_GET_EVTS(evt) ERTS_POLL_EV_N2E((evt)->revents) +#define ERTS_POLL_RES_SET_EVTS(evt, evts) (evt)->revents = ERTS_POLL_EV_E2N(evts) + +#elif ERTS_POLL_USE_KQUEUE_EVS /* --- kqueue ------------------------------ */ /* Kqueue use fallback defines (poll() or select()) */ + +#include <sys/event.h> + +#ifdef ERTS_USE_POLL +# undef ERTS_POLL_USE_POLL_EVS +# define ERTS_POLL_USE_POLL_EVS 1 +#elif !defined(__WIN32__) +# undef ERTS_POLL_USE_SELECT_EVS +# define ERTS_POLL_USE_SELECT_EVS 1 #endif -#if ERTS_POLL_USE_POLL /* --- poll -------------------------------- */ +typedef struct kevent ErtsPollResFd; + +#define ERTS_POLL_RES_GET_FD(evt) ((ErtsSysFdType)((evt)->ident)) +#define ERTS_POLL_RES_SET_FD(evt, fd) (evt)->ident = fd +#define ERTS_POLL_RES_GET_EVTS(evt) ERTS_POLL_EV_N2E((ErtsPollEvents)(evt)->udata) +#define ERTS_POLL_RES_SET_EVTS(evt, evts) (evt)->udata = (void*)(UWord)(ERTS_POLL_EV_E2N(evts)) + +#endif +#if ERTS_POLL_USE_POLL_EVS + /* --- poll -------------------------------- */ #include <poll.h> #define ERTS_POLL_EV_NKP_E2N(EV) \ @@ -169,7 +236,7 @@ typedef Uint32 ErtsPollEvents; #define ERTS_POLL_EV_NKP_NVAL ERTS_POLL_EV_N2E(POLLNVAL) #define ERTS_POLL_EV_NKP_ERR ERTS_POLL_EV_N2E(POLLERR|POLLHUP) -#elif ERTS_POLL_USE_SELECT /* --- select ------------------------------ */ +#elif ERTS_POLL_USE_SELECT_EVS /* --- select ------------------------------ */ #define ERTS_POLL_EV_NKP_E2N(EV) (EV) #define ERTS_POLL_EV_NKP_N2E(EV) (EV) @@ -195,70 +262,65 @@ typedef Uint32 ErtsPollEvents; #endif -typedef struct ErtsPollSet_ *ErtsPollSet; +#if !ERTS_ENABLE_KERNEL_POLL -typedef struct { - ErtsSysFdType fd; - ErtsPollEvents events; - int on; -} ErtsPollControlEntry; - -typedef struct { +typedef struct _ErtsPollResFd { ErtsSysFdType fd; ErtsPollEvents events; } ErtsPollResFd; +#define ERTS_POLL_RES_GET_FD(evt) (evt)->fd +#define ERTS_POLL_RES_SET_FD(evt, ident) (evt)->fd = (ident) +#define ERTS_POLL_RES_GET_EVTS(evt) ERTS_POLL_EV_N2E((evt)->events) +#define ERTS_POLL_RES_SET_EVTS(evt, evts) (evt)->events = ERTS_POLL_EV_E2N(evts) + +#endif + +#define ERTS_POLL_EV_NONE (UINT_MAX & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT|ERTS_POLL_EV_NVAL|ERTS_POLL_EV_ERR)) + +#define ev2str(ev) \ + (((ev) == 0 || (ev) == ERTS_POLL_EV_NONE) ? "NONE" : \ + ((ev) == ERTS_POLL_EV_IN ? "IN" : \ + ((ev) == ERTS_POLL_EV_OUT ? "OUT" : \ + ((ev) == (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT) ? "IN|OUT" : \ + ((ev) & ERTS_POLL_EV_ERR ? "ERR" : \ + ((ev) & ERTS_POLL_EV_NVAL ? "NVAL" : "OTHER")))))) + + +typedef struct ERTS_POLL_EXPORT(erts_pollset) ErtsPollSet; + typedef struct { char *primary; - char *fallback; char *kernel_poll; Uint memory_size; - int poll_set_size; - int fallback_poll_set_size; + Uint poll_set_size; int lazy_updates; - int pending_updates; + Uint pending_updates; int batch_updates; int concurrent_updates; - int max_fds; -#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS - long no_avoided_wakeups; - long no_avoided_interrupts; - long no_interrupt_timed; -#endif + int is_fallback; + Uint max_fds; + Uint active_fds; + Uint poll_threads; } ErtsPollInfo; -void ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet, - int); -void ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet, - int, - ErtsMonotonicTime); -ErtsPollEvents ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet, - ErtsSysFdType, - ErtsPollEvents, - int on, - int* wake_poller - ); -void ERTS_POLL_EXPORT(erts_poll_controlv)(ErtsPollSet, - ErtsPollControlEntry [], - int on); -int ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet, - ErtsPollResFd [], - int *, - ErtsMonotonicTime); -int ERTS_POLL_EXPORT(erts_poll_max_fds)(void); -void ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet, - ErtsPollInfo *); -ErtsPollSet ERTS_POLL_EXPORT(erts_poll_create_pollset)(void); -void ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet); -void ERTS_POLL_EXPORT(erts_poll_init)(void); -void ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet, - ErtsPollEvents [], - int); +#if defined(ERTS_POLL_USE_FALLBACK) && ERTS_KERNEL_POLL_VERSION +# undef ERTS_POLL_EXPORT +# define ERTS_POLL_EXPORT(FUNC) FUNC ## _flbk +# include "erl_poll_api.h" +# undef ERTS_POLL_EXPORT +# define ERTS_POLL_EXPORT(FUNC) FUNC +#elif !defined(ERTS_POLL_USE_FALLBACK) +# define ERTS_POLL_USE_FALLBACK 0 +#endif -int erts_poll_new_table_len(int old_len, int need_len); +#include "erl_poll_api.h" -#ifdef ERTS_ENABLE_LOCK_COUNT -void ERTS_POLL_EXPORT(erts_lcnt_update_pollset_locks)(int enable); -#endif +/** + * Get the next size of the array that holds the file descriptors. + * This function is used in order for the check io array and the + * pollset array to be of the same size. + */ +int erts_poll_new_table_len(int old_len, int need_len); #endif /* #ifndef ERL_POLL_H__ */ diff --git a/erts/emulator/sys/common/erl_poll_api.h b/erts/emulator/sys/common/erl_poll_api.h new file mode 100644 index 0000000000..04beb37d1c --- /dev/null +++ b/erts/emulator/sys/common/erl_poll_api.h @@ -0,0 +1,122 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2006-2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/** + * @description: Poll interface functions + * @author Lukas Larsson + * + * The functions in the header are used to interact with the poll + * implementation. Iff the kernel-poll implementation needs a fallback + * pollset, then all functions are exported twice. Once with a _flbk + * suffix and once without any suffix. If no fallback is needed, then + * only the non-suffix version is exported. + */ + +/** + * Initialize the poll implementation. Has to be called before any other function. + * @param[out] concurrent_waiters if not NULL, set to 1 if more then one thread + * is allowed to wait in the pollsets at the same time. + */ +void ERTS_POLL_EXPORT(erts_poll_init)(int *concurrent_waiters); +/** + * @brief Create a new pollset. + * @param id The unique debug id of this pollset. + */ +ErtsPollSet *ERTS_POLL_EXPORT(erts_poll_create_pollset)(int id); + +/** + * Modify the contents of a pollset. This function can be called while one + * (or possibly more) thread is waiting in the pollset. + * + * @param ps the pollset to modify + * @param fd the file descriptor to modify + * @param op the type of operation to do. Normal usage is ADD,MOD...MOD,DEL. + * @param evts the events that we are changing interest to. Ignored if op is DEL. + * @param[in] wake_poller if set to 1 any thread waiting in the pollset will be woken. + * This parameter is ignored if the pollset supports concurrent waiters. + * @param[out] wake_poller set to 1 if the waiting thread was woken. + * @return The events set, or ERTS_POLL_EV_NVAL if it was not possible to add the + * fd to the pollset. + */ +ErtsPollEvents ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet *ps, + ErtsSysFdType fd, + ErtsPollOp op, + ErtsPollEvents evts, + int *wake_poller); + +/** + * Wait for events to be ready in the pollset. If the erts_poll_init call + * set concurrent_waiters to 1, then multiple threads are allowed to call + * this function at the same time. + * + * When an event has been triggered on a fd, that event is disabled. To + * re-enable it the implementation has to call erts_poll_control again. + * + * @param ps the pollset to wait for events in + * @param res an array of fd results that the ready fds are put in. + * @param[in] length the length of the res array + * @param[out] length the number of ready events returned in res + * @return 0 on success, else the ERRNO of the error that happened. + */ +int ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet *ps, + ErtsPollResFd res[], + int *length); +/** + * Interrupt the thread waiting in the pollset. This function should be called + * with set = 0 before any thread calls erts_poll_wait in order to clear any + * interrupts that have happened while the thread was awake. + * + * This function has no effect on pollsets that support concurrent waiters. + * + * @param ps the pollset to wake + * @param set if 1, interrupt the pollset, if 0 clear the interrupt flag. + */ +void ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet *ps, int set); + +/* Debug functions */ + +/** + * Get the maximum number of fds supported by the pollset + */ +int ERTS_POLL_EXPORT(erts_poll_max_fds)(void); +/** + * Get information about the given pollset + */ +void ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet *ps, + ErtsPollInfo *info); +/** + * Get information about which events are currently selected. + * + * The unix fd is used to index into the array, so naturally this function does + * not work on windows. If the pollset cannot figure out what the selected + * events for a given fd is, it is set to ERTS_POLL_EV_NONE. + * + * @param ps the pollset to get events from + * @param evts an array of which events are selected on. + */ +void ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet *ps, + ErtsPollEvents evts[], + int length); + +#ifdef ERTS_ENABLE_LOCK_COUNT +/** + * Enable lock counting of any locks within the pollset. + */ +void ERTS_POLL_EXPORT(erts_lcnt_enable_pollset_lock_count)(ErtsPollSet *, int enable); +#endif diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c index 09237c81ce..420138ff0a 100644 --- a/erts/emulator/sys/common/erl_sys_common_misc.c +++ b/erts/emulator/sys/common/erl_sys_common_misc.c @@ -45,14 +45,6 @@ #endif #endif -/* - * erts_check_io_time is used by the erl_check_io implementation. The - * global erts_check_io_time variable is declared here since there - * (often) exist two versions of erl_check_io (kernel-poll and - * non-kernel-poll), and we dont want two versions of this variable. - */ -erts_atomic_t erts_check_io_time; - /* Written once and only once */ static int filename_encoding = ERL_FILENAME_UNKNOWN; diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h index cbbd0e6f40..b6f5b319ee 100644 --- a/erts/emulator/sys/unix/erl_unix_sys.h +++ b/erts/emulator/sys/unix/erl_unix_sys.h @@ -86,6 +86,10 @@ #include <sys/times.h> +#ifdef HAVE_SYS_RESOURCE_H +# include <sys/resource.h> +#endif + #ifdef HAVE_IEEEFP_H #include <ieeefp.h> #endif diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index 237614b0fb..6315135151 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -131,121 +131,6 @@ static int replace_intr = 0; /* assume yes initially, ttsl_init will clear it */ int using_oldshell = 1; -#ifdef ERTS_ENABLE_KERNEL_POLL - -int erts_use_kernel_poll = 0; - -struct { - int (*select)(ErlDrvPort, ErlDrvEvent, int, int); - int (*enif_select)(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm); - int (*event)(ErlDrvPort, ErlDrvEvent, ErlDrvEventData); - void (*check_io_as_interrupt)(void); - void (*check_io_interrupt)(int); - void (*check_io_interrupt_tmd)(int, ErtsMonotonicTime); - void (*check_io)(int); - Uint (*size)(void); - Eterm (*info)(void *); - int (*check_io_debug)(ErtsCheckIoDebugInfo *); -} io_func = {0}; - - -int -driver_select(ErlDrvPort port, ErlDrvEvent event, int mode, int on) -{ - return (*io_func.select)(port, event, mode, on); -} - -int -driver_event(ErlDrvPort port, ErlDrvEvent event, ErlDrvEventData event_data) -{ - return (*io_func.event)(port, event, event_data); -} - -int enif_select(ErlNifEnv* env, ErlNifEvent event, - enum ErlNifSelectFlags flags, void* obj, const ErlNifPid* pid, Eterm ref) -{ - return (*io_func.enif_select)(env, event, flags, obj, pid, ref); -} - - -Eterm erts_check_io_info(void *p) -{ - return (*io_func.info)(p); -} - -int -erts_check_io_debug(ErtsCheckIoDebugInfo *ip) -{ - return (*io_func.check_io_debug)(ip); -} - - -static void -init_check_io(void) -{ - if (erts_use_kernel_poll) { - io_func.select = driver_select_kp; - io_func.enif_select = enif_select_kp; - io_func.event = driver_event_kp; - io_func.check_io_interrupt = erts_check_io_interrupt_kp; - io_func.check_io_interrupt_tmd = erts_check_io_interrupt_timed_kp; - io_func.check_io = erts_check_io_kp; - io_func.size = erts_check_io_size_kp; - io_func.info = erts_check_io_info_kp; - io_func.check_io_debug = erts_check_io_debug_kp; - erts_init_check_io_kp(); - max_files = erts_check_io_max_files_kp(); - } - else { - io_func.select = driver_select_nkp; - io_func.enif_select = enif_select_nkp; - io_func.event = driver_event_nkp; - io_func.check_io_interrupt = erts_check_io_interrupt_nkp; - io_func.check_io_interrupt_tmd = erts_check_io_interrupt_timed_nkp; - io_func.check_io = erts_check_io_nkp; - io_func.size = erts_check_io_size_nkp; - io_func.info = erts_check_io_info_nkp; - io_func.check_io_debug = erts_check_io_debug_nkp; - erts_init_check_io_nkp(); - max_files = erts_check_io_max_files_nkp(); - } -} - -#define ERTS_CHK_IO_AS_INTR() (*io_func.check_io_interrupt)(1) -#define ERTS_CHK_IO_INTR (*io_func.check_io_interrupt) -#define ERTS_CHK_IO_INTR_TMD (*io_func.check_io_interrupt_tmd) -#define ERTS_CHK_IO (*io_func.check_io) -#define ERTS_CHK_IO_SZ (*io_func.size) - -#else /* !ERTS_ENABLE_KERNEL_POLL */ - -static void -init_check_io(void) -{ - erts_init_check_io(); - max_files = erts_check_io_max_files(); -} - -#define ERTS_CHK_IO_AS_INTR() erts_check_io_interrupt(1) -#define ERTS_CHK_IO_INTR erts_check_io_interrupt -#define ERTS_CHK_IO_INTR_TMD erts_check_io_interrupt_timed -#define ERTS_CHK_IO erts_check_io -#define ERTS_CHK_IO_SZ erts_check_io_size - -#endif - -void -erts_sys_schedule_interrupt(int set) -{ - ERTS_CHK_IO_INTR(set); -} - -void -erts_sys_schedule_interrupt_timed(int set, ErtsMonotonicTime timeout_time) -{ - ERTS_CHK_IO_INTR_TMD(set, timeout_time); -} - UWord erts_sys_get_page_size(void) { @@ -261,7 +146,7 @@ erts_sys_get_page_size(void) Uint erts_sys_misc_mem_sz(void) { - Uint res = ERTS_CHK_IO_SZ(); + Uint res = erts_check_io_size(); res += erts_atomic_read_mb(&sys_misc_mem_sz); return res; } @@ -618,14 +503,12 @@ break_requested(void) * just set a flag - checked for and handled by * scheduler threads erts_check_io() (not signal handler). */ -#ifdef DEBUG - fprintf(stderr,"break!\n"); -#endif if (ERTS_BREAK_REQUESTED) erts_exit(ERTS_INTR_EXIT, ""); ERTS_SET_BREAK_REQUESTED; - ERTS_CHK_IO_AS_INTR(); /* Make sure we don't sleep in poll */ + /* Wake aux thread to get handle break */ + erts_aux_thread_poke(); } static RETSIGTYPE request_break(int signum) @@ -739,7 +622,6 @@ signum_to_signalterm(int signum) } } - static RETSIGTYPE generic_signal_handler(int signum) { smp_sig_notify(signum); @@ -814,7 +696,7 @@ erts_sys_unix_later_init(void) int sys_max_files(void) { - return(max_files); + return max_files; } /************************** OS info *******************************/ @@ -1191,20 +1073,6 @@ erl_debug(char* fmt, ...) #endif /* DEBUG */ -/* - * Called from schedule() when it runs out of runnable processes, - * or when Erlang code has performed INPUT_REDUCTIONS reduction - * steps. runnable == 0 iff there are no runnable Erlang processes. - */ -void -erl_sys_schedule(int runnable) -{ - ERTS_CHK_IO(!runnable); - ERTS_LC_ASSERT(!erts_thr_progress_is_blocking()); -} - - - static erts_tid_t sig_dispatcher_tid; static void @@ -1393,99 +1261,18 @@ erts_sys_main_thread(void) } -#ifdef ERTS_ENABLE_KERNEL_POLL /* get_value() is currently only used when - kernel-poll is enabled */ - -/* Get arg marks argument as handled by - putting NULL in argv */ -static char * -get_value(char* rest, char** argv, int* ip) -{ - char *param = argv[*ip]+1; - argv[*ip] = NULL; - if (*rest == '\0') { - char *next = argv[*ip + 1]; - if (next[0] == '-' - && next[1] == '-' - && next[2] == '\0') { - erts_fprintf(stderr, "bad \"%s\" value: \n", param); - erts_usage(); - } - (*ip)++; - argv[*ip] = NULL; - return next; - } - return rest; -} - -#endif /* ERTS_ENABLE_KERNEL_POLL */ - void erl_sys_args(int* argc, char** argv) { - int i, j; erts_rwmtx_init(&environ_rwmtx, "environ", NIL, ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); - i = 1; - ASSERT(argc && argv); - while (i < *argc) { - if(argv[i][0] == '-') { - switch (argv[i][1]) { -#ifdef ERTS_ENABLE_KERNEL_POLL - case 'K': { - char *arg = get_value(argv[i] + 2, argv, &i); - if (strcmp("true", arg) == 0) { - erts_use_kernel_poll = 1; - } - else if (strcmp("false", arg) == 0) { - erts_use_kernel_poll = 0; - } - else { - erts_fprintf(stderr, "bad \"K\" value: %s\n", arg); - erts_usage(); - } - break; - } -#endif - case '-': - goto done_parsing; - default: - break; - } - } - i++; - } - - done_parsing: - -#ifdef ERTS_ENABLE_KERNEL_POLL - if (erts_use_kernel_poll) { - char no_kp[10]; - size_t no_kp_sz = sizeof(no_kp); - int res = erts_sys_getenv_raw("ERL_NO_KERNEL_POLL", no_kp, &no_kp_sz); - if (res > 0 - || (res == 0 - && sys_strcmp("false", no_kp) != 0 - && sys_strcmp("FALSE", no_kp) != 0)) { - erts_use_kernel_poll = 0; - } - } -#endif - - init_check_io(); + max_files = erts_check_io_max_files(); init_smp_sig_notify(); init_smp_sig_suspend(); - /* Handled arguments have been marked with NULL. Slide arguments - not handled towards the beginning of argv. */ - for (i = 0, j = 0; i < *argc; i++) { - if (argv[i]) - argv[j++] = argv[i]; - } - *argc = j; } diff --git a/erts/emulator/sys/unix/sys_drivers.c b/erts/emulator/sys/unix/sys_drivers.c index 7c9a532fed..0228e1af54 100644 --- a/erts/emulator/sys/unix/sys_drivers.c +++ b/erts/emulator/sys/unix/sys_drivers.c @@ -1723,8 +1723,6 @@ static ErlDrvData forker_start(ErlDrvPort port_num, char* name, SET_NONBLOCKING(forker_fd); - driver_select(port_num, forker_fd, ERL_DRV_READ|ERL_DRV_USE, 1); - return (ErlDrvData)port_num; } @@ -1821,10 +1819,19 @@ static void forker_ready_output(ErlDrvData e, ErlDrvEvent fd) static ErlDrvSSizeT forker_control(ErlDrvData e, unsigned int cmd, char *buf, ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen) { + static int first_call = 1; ErtsSysForkerProto *proto = (ErtsSysForkerProto *)buf; ErlDrvPort port_num = (ErlDrvPort)e; int res; + if (first_call) { + /* + * Do driver_select here when schedulers and their pollsets have started. + */ + driver_select(port_num, forker_fd, ERL_DRV_READ|ERL_DRV_USE, 1); + first_call = 0; + } + driver_enq(port_num, buf, len); if (driver_sizeq(port_num) > sizeof(*proto)) { return 0; diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c index 0bd43bb4fb..fd4c745c3b 100644 --- a/erts/emulator/sys/win32/erl_poll.c +++ b/erts/emulator/sys/win32/erl_poll.c @@ -34,6 +34,7 @@ */ /*#define HARDDEBUG */ +/*#define HARDTRACE */ #ifdef HARDDEBUG #ifdef HARDTRACE #define HARDTRACEF(X) my_debug_printf##X @@ -50,7 +51,7 @@ static void my_debug_printf(char *fmt, ...) va_start(args, fmt); erts_vsnprintf(buffer,1024,fmt,args); va_end(args); - erts_fprintf(stderr,"%s\r\n",buffer); + erts_printf("%s\r\n",buffer); } #else #define HARDTRACEF(X) @@ -274,20 +275,17 @@ typedef struct _Waiter { /* * The structure for a pollset. There can currently be only one... */ -struct ErtsPollSet_ { +struct erts_pollset { Waiter** waiter; int allocated_waiters; /* Size ow waiter array */ int num_waiters; /* Number of waiter threads. */ - int restore_events; /* Tells us to restore waiters events - next time around */ HANDLE event_io_ready; /* To be used when waiting for io */ /* These are used to wait for workers to enter standby */ volatile int standby_wait_counter; /* Number of threads to wait for */ CRITICAL_SECTION standby_crit; /* CS to guard the counter */ - HANDLE standby_wait_event; /* Event signalled when counte == 0 */ + HANDLE standby_wait_event; /* Event signalled when counter == 0 */ erts_atomic32_t wakeup_state; erts_mtx_t mtx; - erts_atomic64_t timeout_time; }; @@ -352,39 +350,19 @@ do { \ wait_standby(PS); \ } while(0) -static ERTS_INLINE void -init_timeout_time(ErtsPollSet ps) -{ - erts_atomic64_init_nob(&ps->timeout_time, - (erts_aint64_t) ERTS_MONOTONIC_TIME_MAX); -} - -static ERTS_INLINE void -set_timeout_time(ErtsPollSet ps, ErtsMonotonicTime time) -{ - erts_atomic64_set_relb(&ps->timeout_time, - (erts_aint64_t) time); -} - -static ERTS_INLINE ErtsMonotonicTime -get_timeout_time(ErtsPollSet ps) -{ - return (ErtsMonotonicTime) erts_atomic64_read_acqb(&ps->timeout_time); -} - #define ERTS_POLL_NOT_WOKEN ((erts_aint32_t) 0) #define ERTS_POLL_WOKEN_IO_READY ((erts_aint32_t) 1) #define ERTS_POLL_WOKEN_INTR ((erts_aint32_t) 2) #define ERTS_POLL_WOKEN_TIMEDOUT ((erts_aint32_t) 3) static ERTS_INLINE int -is_io_ready(ErtsPollSet ps) +is_io_ready(ErtsPollSet *ps) { return erts_atomic32_read_nob(&ps->wakeup_state) == ERTS_POLL_WOKEN_IO_READY; } static ERTS_INLINE void -woke_up(ErtsPollSet ps) +woke_up(ErtsPollSet *ps) { if (erts_atomic32_read_nob(&ps->wakeup_state) == ERTS_POLL_NOT_WOKEN) erts_atomic32_cmpxchg_nob(&ps->wakeup_state, @@ -407,7 +385,7 @@ woke_up(ErtsPollSet ps) } static ERTS_INLINE int -wakeup_cause(ErtsPollSet ps) +wakeup_cause(ErtsPollSet *ps) { int res; erts_aint32_t wakeup_state = erts_atomic32_read_acqb(&ps->wakeup_state); @@ -430,46 +408,8 @@ wakeup_cause(ErtsPollSet ps) return res; } -static ERTS_INLINE DWORD -poll_wait_timeout(ErtsPollSet ps, ErtsMonotonicTime timeout_time) -{ - ErtsMonotonicTime current_time, diff_time, timeout; - - if (timeout_time == ERTS_POLL_NO_TIMEOUT) { - no_timeout: - set_timeout_time(ps, ERTS_MONOTONIC_TIME_MIN); - woke_up(ps); - return (DWORD) 0; - } - - current_time = erts_get_monotonic_time(NULL); - diff_time = timeout_time - current_time; - if (diff_time <= 0) - goto no_timeout; - - /* Round up to nearest milli second */ - timeout = (ERTS_MONOTONIC_TO_MSEC(diff_time - 1) + 1); - if (timeout > INT_MAX) - timeout = INT_MAX; /* Also prevents DWORD overflow */ - - set_timeout_time(ps, current_time + ERTS_MSEC_TO_MONOTONIC(timeout)); - - ResetEvent(ps->event_io_ready); - /* - * Since we don't know the internals of ResetEvent() we issue - * a memory barrier as a safety precaution ensuring that - * the load of wakeup_state wont be reordered with stores made - * by ResetEvent(). - */ - ERTS_THR_MEMORY_BARRIER; - if (erts_atomic32_read_nob(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN) - return (DWORD) 0; - - return (DWORD) timeout; -} - static ERTS_INLINE void -wake_poller(ErtsPollSet ps, int io_ready) +wake_poller(ErtsPollSet *ps, int io_ready) { erts_aint32_t wakeup_state; if (io_ready) { @@ -504,13 +444,13 @@ wake_poller(ErtsPollSet ps, int io_ready) } static ERTS_INLINE void -reset_io_ready(ErtsPollSet ps) +reset_io_ready(ErtsPollSet *ps) { erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); } static ERTS_INLINE void -restore_io_ready(ErtsPollSet ps) +restore_io_ready(ErtsPollSet *ps) { erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY); } @@ -520,13 +460,13 @@ restore_io_ready(ErtsPollSet ps) * notifying a poller thread about I/O ready. */ static ERTS_INLINE void -notify_io_ready(ErtsPollSet ps) +notify_io_ready(ErtsPollSet *ps) { wake_poller(ps, 1); } static ERTS_INLINE void -reset_interrupt(ErtsPollSet ps) +reset_interrupt(ErtsPollSet *ps) { /* We need to keep io-ready if set */ erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); @@ -543,12 +483,12 @@ reset_interrupt(ErtsPollSet ps) } static ERTS_INLINE void -set_interrupt(ErtsPollSet ps) +set_interrupt(ErtsPollSet *ps) { wake_poller(ps, 0); } -static void setup_standby_wait(ErtsPollSet ps, int num_threads) +static void setup_standby_wait(ErtsPollSet *ps, int num_threads) { EnterCriticalSection(&(ps->standby_crit)); ps->standby_wait_counter = num_threads; @@ -556,7 +496,7 @@ static void setup_standby_wait(ErtsPollSet ps, int num_threads) LeaveCriticalSection(&(ps->standby_crit)); } -static void signal_standby(ErtsPollSet ps) +static void signal_standby(ErtsPollSet *ps) { EnterCriticalSection(&(ps->standby_crit)); --(ps->standby_wait_counter); @@ -570,7 +510,7 @@ static void signal_standby(ErtsPollSet ps) LeaveCriticalSection(&(ps->standby_crit)); } -static void wait_standby(ErtsPollSet ps) +static void wait_standby(ErtsPollSet *ps) { WaitForSingleObject(ps->standby_wait_event,INFINITE); } @@ -638,7 +578,7 @@ static void consistency_check(Waiter* w) #endif -static void new_waiter(ErtsPollSet ps) +static void new_waiter(ErtsPollSet *ps) { register Waiter* w; DWORD tid; /* Id for thread. */ @@ -732,7 +672,7 @@ static void *break_waiter(void *param) static void *threaded_waiter(void *param) { register Waiter* w = (Waiter *) param; - ErtsPollSet ps = (ErtsPollSet) w->xdata; + ErtsPollSet *ps = (ErtsPollSet*) w->xdata; #ifdef HARD_POLL_DEBUG2 HANDLE oold_fired[64]; int num_oold_fired; @@ -835,9 +775,9 @@ event_happened: ASSERT(i >= WAIT_OBJECT_0+1); i -= WAIT_OBJECT_0; ASSERT(i >= 1); - w->active_events--; HARDDEBUGF(("i = %d, a,h,t = %d,%d,%d",i, w->active_events, w->highwater, w->total_events)); + w->active_events--; #ifdef HARD_POLL_DEBUG2 fired[num_fired++] = w->events[i]; #endif @@ -867,7 +807,7 @@ event_happened: * The actual adding and removing from pollset utilities */ -static int set_driver_select(ErtsPollSet ps, HANDLE event, ErtsPollEvents mode) +static int set_driver_select(ErtsPollSet *ps, HANDLE event, ErtsPollEvents mode) { int i; int best_waiter = -1; /* The waiter with lowest number of events. */ @@ -957,13 +897,13 @@ static int set_driver_select(ErtsPollSet ps, HANDLE event, ErtsPollEvents mode) #endif erts_mtx_unlock(&w->mtx); START_WAITER(ps,w); - HARDDEBUGF(("add select %d %d %d %d",best_waiter, + HARDDEBUGF(("%d: add select %d %d %d %d", event, best_waiter, w->active_events,w->highwater,w->total_events)); return mode; } -static int cancel_driver_select(ErtsPollSet ps, HANDLE event) +static int cancel_driver_select(ErtsPollSet *ps, HANDLE event) { int i; @@ -1018,7 +958,7 @@ static int cancel_driver_select(ErtsPollSet ps, HANDLE event) * Interface functions */ -void erts_poll_interrupt(ErtsPollSet ps, int set /* bool */) +void erts_poll_interrupt(ErtsPollSet *ps, int set /* bool */) { HARDTRACEF(("In erts_poll_interrupt(%d)",set)); if (!set) @@ -1028,35 +968,23 @@ void erts_poll_interrupt(ErtsPollSet ps, int set /* bool */) HARDTRACEF(("Out erts_poll_interrupt(%d)",set)); } -void erts_poll_interrupt_timed(ErtsPollSet ps, - int set /* bool */, - ErtsMonotonicTime timeout_time) -{ - HARDTRACEF(("In erts_poll_interrupt_timed(%d,%ld)",set,timeout_time)); - if (!set) - reset_interrupt(ps); - else if (get_timeout_time(ps) > timeout_time) - set_interrupt(ps); - HARDTRACEF(("Out erts_poll_interrupt_timed")); -} - /* * Windows is special, there is actually only one event type, and * the only difference between ERTS_POLL_EV_IN and ERTS_POLL_EV_OUT * is which driver callback will eventually be called. */ -static ErtsPollEvents do_poll_control(ErtsPollSet ps, - ErtsSysFdType fd, - ErtsPollEvents pe, - int on /* bool */) +static ErtsPollEvents do_poll_control(ErtsPollSet *ps, + ErtsSysFdType fd, + ErtsPollOp op, + ErtsPollEvents pe) { HANDLE event = (HANDLE) fd; ErtsPollEvents mode; ErtsPollEvents result; ASSERT(event != INVALID_HANDLE_VALUE); - if (on) { + if (op != ERTS_POLL_OP_DEL) { if (pe & ERTS_POLL_EV_IN || !(pe & ERTS_POLL_EV_OUT )) { mode = ERTS_POLL_EV_IN; } else { @@ -1069,51 +997,30 @@ static ErtsPollEvents do_poll_control(ErtsPollSet ps, return result; } -ErtsPollEvents erts_poll_control(ErtsPollSet ps, +ErtsPollEvents erts_poll_control(ErtsPollSet *ps, ErtsSysFdType fd, + ErtsPollOp op, ErtsPollEvents pe, - int on, int* do_wake) /* In: Wake up polling thread */ /* Out: Poller is woken */ { ErtsPollEvents result; - HARDTRACEF(("In erts_poll_control(0x%08X, %u, %d)",(unsigned long) fd, (unsigned) pe, on)); + HARDTRACEF(("In erts_poll_control(0x%08X, %s, %s)", + (unsigned long) fd, op2str(op), ev2str(pe))); ERTS_POLLSET_LOCK(ps); - result=do_poll_control(ps,fd,pe,on); + result=do_poll_control(ps, fd, op, pe); ERTS_POLLSET_UNLOCK(ps); *do_wake = 0; /* Never any need to wake polling threads on windows */ HARDTRACEF(("Out erts_poll_control -> %u",(unsigned) result)); return result; } -void erts_poll_controlv(ErtsPollSet ps, - ErtsPollControlEntry pcev[], - int len) -{ - int i; - int hshur = 0; - int do_wake = 0; - - HARDTRACEF(("In erts_poll_controlv(%d)",len)); - ERTS_POLLSET_LOCK(ps); - - for (i = 0; i < len; i++) { - pcev[i].events = do_poll_control(ps, - pcev[i].fd, - pcev[i].events, - pcev[i].on); - } - ERTS_POLLSET_UNLOCK(ps); - HARDTRACEF(("Out erts_poll_controlv")); -} - -int erts_poll_wait(ErtsPollSet ps, +int erts_poll_wait(ErtsPollSet *ps, ErtsPollResFd pr[], - int *len, - ErtsMonotonicTime timeout_time) + int *len) { int no_fds; - DWORD timeout; + DWORD timeout = INFINITE; EventData* ev; int res = 0; int num = 0; @@ -1124,42 +1031,6 @@ int erts_poll_wait(ErtsPollSet ps, HARDTRACEF(("In erts_poll_wait")); ERTS_POLLSET_LOCK(ps); - if (!is_io_ready(ps) && ps->restore_events) { - HARDDEBUGF(("Restore events: %d",ps->num_waiters)); - ps->restore_events = 0; - for (i = 0; i < ps->num_waiters; ++i) { - Waiter* w = ps->waiter[i]; - erts_mtx_lock(&w->mtx); - HARDDEBUGF(("Maybe reset %d %d %d %d",i, - w->active_events,w->highwater,w->total_events)); - if (w->active_events < w->total_events) { - erts_mtx_unlock(&w->mtx); - STOP_WAITER(ps,w); - HARDDEBUGF(("Need reset %d %d %d %d",i, - w->active_events,w->highwater,w->total_events)); - erts_mtx_lock(&w->mtx); - /* Need reset, just check that it doesn't have got more to tell */ - if (w->highwater != w->active_events) { - HARDDEBUGF(("Oups!")); - /* Oups, got signalled before we took the lock, can't reset */ - if(!is_io_ready(ps)) { - erts_exit(ERTS_ERROR_EXIT,"Internal error: " - "Inconsistent io structures in erl_poll.\n"); - } - START_WAITER(ps,w); - erts_mtx_unlock(&w->mtx); - ps->restore_events = 1; - continue; - } - w->active_events = w->highwater = w->total_events; - START_WAITER(ps,w); - erts_mtx_unlock(&w->mtx); - } else { - erts_mtx_unlock(&w->mtx); - } - } - } - no_fds = *len; #ifdef ERTS_POLL_MAX_RES @@ -1167,22 +1038,29 @@ int erts_poll_wait(ErtsPollSet ps, no_fds = ERTS_POLL_MAX_RES; #endif - timeout = poll_wait_timeout(ps, timeout_time); - - /*HARDDEBUGF(("timeout = %ld",(long) timeout));*/ + ResetEvent(ps->event_io_ready); + /* + * Since we don't know the internals of ResetEvent() we issue + * a memory barrier as a safety precaution ensuring that + * the load of wakeup_state wont be reordered with stores made + * by ResetEvent(). + */ + ERTS_THR_MEMORY_BARRIER; + if (erts_atomic32_read_nob(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN) + timeout = (DWORD) 0; - if (timeout > 0 && !erts_atomic32_read_nob(&break_waiter_state)) { + if (!erts_atomic32_read_nob(&break_waiter_state)) { HANDLE harr[2] = {ps->event_io_ready, break_happened_event}; int num_h = 2; - ERTS_MSACC_PUSH_STATE_M(); + ERTS_MSACC_PUSH_STATE(); HARDDEBUGF(("Start waiting %d [%d]",num_h, (int) timeout)); ERTS_POLLSET_UNLOCK(ps); erts_thr_progress_prepare_wait(NULL); - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_SLEEP); WaitForMultipleObjects(num_h, harr, FALSE, timeout); erts_thr_progress_finalize_wait(NULL); - ERTS_MSACC_POP_STATE_M(); + ERTS_MSACC_POP_STATE(); ERTS_POLLSET_LOCK(ps); HARDDEBUGF(("Stop waiting %d [%d]",num_h, (int) timeout)); woke_up(ps); @@ -1215,7 +1093,7 @@ int erts_poll_wait(ErtsPollSet ps, reset_io_ready(ps); - n = ps->num_waiters; + n = ps->num_waiters; for (i = 0; i < n; i++) { Waiter* w = ps->waiter[i]; @@ -1241,11 +1119,10 @@ int erts_poll_wait(ErtsPollSet ps, HARDDEBUGF(("To many FD's to report!")); goto done; } - HARDDEBUGF(("SET! Restore events")); - ps->restore_events = 1; HARDDEBUGF(("Report %d,%d",i,j)); - pr[num].fd = (ErtsSysFdType) w->events[j]; - pr[num].events = w->evdata[j]->mode; + ERTS_POLL_RES_SET_FD(&pr[num], w->events[j]); + ERTS_POLL_RES_SET_EVTS(&pr[num], w->evdata[j]->mode); + remove_event_from_set(w, j); #ifdef HARD_POLL_DEBUG poll_debug_reported(w->events[j],w->highwater | (j << 16)); poll_debug_reported(w->events[j],first | (last << 16)); @@ -1253,13 +1130,14 @@ int erts_poll_wait(ErtsPollSet ps, ++num; } + w->total_events = w->highwater = w->active_events; + #ifdef DEBUG consistency_check(w); #endif erts_mtx_unlock(&w->mtx); } done: - set_timeout_time(ps, ERTS_MONOTONIC_TIME_MAX); *len = num; ERTS_POLLSET_UNLOCK(ps); HARDTRACEF(("Out erts_poll_wait")); @@ -1274,7 +1152,7 @@ int erts_poll_max_fds(void) return res; } -void erts_poll_info(ErtsPollSet ps, +void erts_poll_info(ErtsPollSet *ps, ErtsPollInfo *pip) { Uint size = 0; @@ -1284,7 +1162,7 @@ void erts_poll_info(ErtsPollSet ps, HARDTRACEF(("In erts_poll_info")); ERTS_POLLSET_LOCK(ps); - size += sizeof(struct ErtsPollSet_); + size += sizeof(struct erts_pollset); size += sizeof(Waiter *) * ps->allocated_waiters; for (i = 0; i < ps->num_waiters; ++i) { Waiter *w = ps->waiter[i]; @@ -1299,16 +1177,12 @@ void erts_poll_info(ErtsPollSet ps, pip->primary = "WaitForMultipleObjects"; - pip->fallback = NULL; - pip->kernel_poll = NULL; pip->memory_size = size; pip->poll_set_size = num_events; - pip->fallback_poll_set_size = 0; - pip->lazy_updates = 0; pip->pending_updates = 0; @@ -1316,6 +1190,8 @@ void erts_poll_info(ErtsPollSet ps, pip->batch_updates = 0; pip->concurrent_updates = 0; + + pip->is_fallback = 0; ERTS_POLLSET_UNLOCK(ps); pip->max_fds = erts_poll_max_fds(); @@ -1323,10 +1199,10 @@ void erts_poll_info(ErtsPollSet ps, } -ErtsPollSet erts_poll_create_pollset(void) +ErtsPollSet *erts_poll_create_pollset(int no) { - ErtsPollSet ps = SEL_ALLOC(ERTS_ALC_T_POLLSET, - sizeof(struct ErtsPollSet_)); + ErtsPollSet *ps = SEL_ALLOC(ERTS_ALC_T_POLLSET, + sizeof(struct erts_pollset)); HARDTRACEF(("In erts_poll_create_pollset")); ps->num_waiters = 0; @@ -1337,17 +1213,15 @@ ErtsPollSet erts_poll_create_pollset(void) ps->standby_wait_counter = 0; ps->event_io_ready = CreateManualEvent(FALSE); ps->standby_wait_event = CreateManualEvent(FALSE); - ps->restore_events = 0; erts_atomic32_init_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); erts_mtx_init(&ps->mtx, "pollset", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO); - init_timeout_time(ps); HARDTRACEF(("Out erts_poll_create_pollset")); return ps; } -void erts_poll_destroy_pollset(ErtsPollSet ps) +void erts_poll_destroy_pollset(ErtsPollSet *ps) { int i; HARDTRACEF(("In erts_poll_destroy_pollset")); @@ -1378,14 +1252,16 @@ void erts_poll_destroy_pollset(ErtsPollSet ps) /* * Actually mostly initializes the friend module sys_interrupt... */ -void erts_poll_init(void) +void erts_poll_init(int *concurrent_updates) { - erts_tid_t thread; #ifdef HARD_POLL_DEBUG poll_debug_init(); #endif + if (concurrent_updates) + *concurrent_updates = 0; + HARDTRACEF(("In erts_poll_init")); erts_sys_break_event = CreateManualEvent(FALSE); @@ -1394,21 +1270,26 @@ void erts_poll_init(void) break_happened_event = CreateManualEvent(FALSE); erts_atomic32_init_nob(&break_waiter_state, 0); + HARDTRACEF(("Out erts_poll_init")); +} + +void erts_poll_late_init(void) +{ + erts_tid_t thread; erts_thr_create(&thread, &break_waiter, NULL, NULL); ERTS_UNSET_BREAK_REQUESTED; - HARDTRACEF(("Out erts_poll_init")); } /* * Non windows friendly interface, not used when fd's are not continous */ -void erts_poll_get_selected_events(ErtsPollSet ps, +void erts_poll_get_selected_events(ErtsPollSet *ps, ErtsPollEvents ev[], int len) { int i; HARDTRACEF(("In erts_poll_get_selected_events")); for (i = 0; i < len; ++i) - ev[i] = 0; + ev[i] = ERTS_POLL_EV_NONE; HARDTRACEF(("Out erts_poll_get_selected_events")); } diff --git a/erts/emulator/sys/win32/erl_win_dyn_driver.h b/erts/emulator/sys/win32/erl_win_dyn_driver.h index 6f28d513c2..0d1a6d4c87 100644 --- a/erts/emulator/sys/win32/erl_win_dyn_driver.h +++ b/erts/emulator/sys/win32/erl_win_dyn_driver.h @@ -40,7 +40,6 @@ WDD_TYPEDEF(int, driver_exit, (ErlDrvPort, int)); WDD_TYPEDEF(int, driver_failure_eof, (ErlDrvPort)); WDD_TYPEDEF(void, erl_drv_busy_msgq_limits, (ErlDrvPort, ErlDrvSizeT *, ErlDrvSizeT *)); WDD_TYPEDEF(int, driver_select, (ErlDrvPort, ErlDrvEvent, int, int)); -WDD_TYPEDEF(int, driver_event, (ErlDrvPort, ErlDrvEvent,ErlDrvEventData)); WDD_TYPEDEF(int, driver_output, (ErlDrvPort, char *, ErlDrvSizeT)); WDD_TYPEDEF(int, driver_output2, (ErlDrvPort, char *, ErlDrvSizeT ,char *, ErlDrvSizeT)); WDD_TYPEDEF(int, driver_output_binary, (ErlDrvPort, char *, ErlDrvSizeT, ErlDrvBinary*, ErlDrvSizeT, ErlDrvSizeT)); @@ -162,7 +161,7 @@ typedef struct { WDD_FTYPE(driver_failure_eof) *driver_failure_eof; WDD_FTYPE(erl_drv_busy_msgq_limits) *erl_drv_busy_msgq_limits; WDD_FTYPE(driver_select) *driver_select; - WDD_FTYPE(driver_event) *driver_event; + void *REMOVED_driver_event; WDD_FTYPE(driver_output) *driver_output; WDD_FTYPE(driver_output2) *driver_output2; WDD_FTYPE(driver_output_binary) *driver_output_binary; @@ -276,7 +275,6 @@ extern TWinDynDriverCallbacks WinDynDriverCallbacks; #define driver_failure_eof (WinDynDriverCallbacks.driver_failure_eof) #define erl_drv_busy_msgq_limits (WinDynDriverCallbacks.erl_drv_busy_msgq_limits) #define driver_select (WinDynDriverCallbacks.driver_select) -#define driver_event (WinDynDriverCallbacks.driver_event) #define driver_output (WinDynDriverCallbacks.driver_output) #define driver_output2 (WinDynDriverCallbacks.driver_output2) #define driver_output_binary (WinDynDriverCallbacks.driver_output_binary) @@ -414,7 +412,7 @@ do { \ ((W).driver_failure_eof) = driver_failure_eof; \ ((W).erl_drv_busy_msgq_limits) = erl_drv_busy_msgq_limits;\ ((W).driver_select) = driver_select; \ -((W).driver_event) = driver_event; \ +((W).REMOVED_driver_event) = NULL; \ ((W).driver_output) = driver_output; \ ((W).driver_output2) = driver_output2; \ ((W).driver_output_binary) = driver_output_binary; \ diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c index b23dbecbac..0598a12351 100644 --- a/erts/emulator/sys/win32/sys.c +++ b/erts/emulator/sys/win32/sys.c @@ -3235,40 +3235,16 @@ void erl_sys_init(void) SetStdHandle(STD_ERROR_HANDLE, GetStdHandle(STD_OUTPUT_HANDLE)); } erts_sys_init_float(); - erts_init_check_io(); /* Suppress windows error message popups */ SetErrorMode(SetErrorMode(0) | SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX); } +void erts_poll_late_init(void); void erl_sys_late_init(void) { /* do nothing */ + erts_poll_late_init(); } - -void -erts_sys_schedule_interrupt(int set) -{ - erts_check_io_interrupt(set); -} - -void -erts_sys_schedule_interrupt_timed(int set, ErtsMonotonicTime timeout_time) -{ - erts_check_io_interrupt_timed(set, timeout_time); -} - -/* - * Called from schedule() when it runs out of runnable processes, - * or when Erlang code has performed INPUT_REDUCTIONS reduction - * steps. runnable == 0 iff there are no runnable Erlang processes. - */ -void -erl_sys_schedule(int runnable) -{ - erts_check_io(!runnable); - ERTS_LC_ASSERT(!erts_thr_progress_is_blocking()); -} - diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile index 5898a8ee8f..bf00de2204 100644 --- a/erts/emulator/test/Makefile +++ b/erts/emulator/test/Makefile @@ -72,6 +72,7 @@ MODULES= \ hash_SUITE \ hibernate_SUITE \ hipe_SUITE \ + iovec_SUITE \ list_bif_SUITE \ lttng_SUITE \ lcnt_SUITE \ diff --git a/erts/emulator/test/beam_SUITE.erl b/erts/emulator/test/beam_SUITE.erl index 6a54fa87e0..bdf8f6c34e 100644 --- a/erts/emulator/test/beam_SUITE.erl +++ b/erts/emulator/test/beam_SUITE.erl @@ -113,20 +113,41 @@ packed_registers(Config) when is_list(Config) -> VarName = list_to_atom("M"++integer_to_list(V)), merl:var(VarName) end || V <- Seq], + MoreNewVars = [begin + VarName = list_to_atom("MM"++integer_to_list(V)), + merl:var(VarName) + end || V <- Seq], + TupleEls = [?Q("id(_@Value@)") || {_,Value} <- S0], S = [?Q("_@Var = id(_@Value@)") || {Var,Value} <- S0], Code = ?Q(["-module('@Mod@').\n" "-export([f/0]).\n" "f() ->\n" + "Tuple = id({_@TupleEls}),\n" + "{_@MoreNewVars} = Tuple,\n" "_@S,\n" "_ = id(0),\n" "L = [_@Vars],\n" "_ = id(1),\n" "[_@NewVars] = L,\n" %Test get_list/3. "_ = id(2),\n" - "id([_@Vars,_@NewVars]).\n" + "id([_@Vars,_@NewVars,_@MoreNewVars]).\n" "id(I) -> I.\n"]), merl:compile_and_load(Code), - CombinedSeq = Seq ++ Seq, + + %% Optionally print the generated code. + PrintCode = false, %Change to true to print code. + + case PrintCode of + false -> + ok; + true -> + merl:print(Code), + erts_debug:df(Mod), + {ok,Dis} = file:read_file(atom_to_list(Mod)++".dis"), + io:put_chars(Dis) + end, + + CombinedSeq = Seq ++ Seq ++ Seq, CombinedSeq = Mod:f(), %% Clean up. diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl index 339c827602..04b7f2de15 100644 --- a/erts/emulator/test/bif_SUITE.erl +++ b/erts/emulator/test/bif_SUITE.erl @@ -24,7 +24,7 @@ -include_lib("kernel/include/file.hrl"). -export([all/0, suite/0, - display/1, display_huge/0, + display/1, display_huge/0, display_string/1, erl_bif_types/1,guard_bifs_in_erl_bif_types/1, shadow_comments/1,list_to_utf8_atom/1, specs/1,improper_bif_stubs/1,auto_imports/1, @@ -43,7 +43,7 @@ all() -> [erl_bif_types, guard_bifs_in_erl_bif_types, shadow_comments, specs, improper_bif_stubs, auto_imports, t_list_to_existing_atom, os_env, otp_7526, - display, list_to_utf8_atom, + display, display_string, list_to_utf8_atom, atom_to_binary, binary_to_atom, binary_to_existing_atom, erl_crash_dump_bytes, min_max, erlang_halt, is_builtin, error_stacktrace, error_stacktrace_during_call_trace]. @@ -68,6 +68,28 @@ deeep(N,Acc) -> deeep(N) -> deeep(N,[hello]). +display_string(Config) when is_list(Config) -> + true = erlang:display_string("hej"), + true = erlang:display_string(""), + true = erlang:display_string("hopp"), + true = erlang:display_string("\n"), + true = erlang:display_string(lists:seq(1100,1200)), + {error,badarg} = try + erlang:display_string(atom), + ok + catch + T0:E0 -> + {T0, E0} + end, + {error,badarg} = try + erlang:display_string(make_ref()), + ok + catch + T1:E1 -> + {T1, E1} + end, + ok. + erl_bif_types(Config) when is_list(Config) -> ensure_erl_bif_types_compiled(), @@ -503,6 +525,8 @@ binary_to_atom(Config) when is_list(Config) -> ?BADARG(binary_to_atom(id(<<255>>), utf8)), ?BADARG(binary_to_atom(id(<<255,0>>), utf8)), ?BADARG(binary_to_atom(id(<<16#C0,16#80>>), utf8)), %Overlong 0. + <<B:1/binary, _/binary>> = id(<<194, 163>>), %Truncated character ERL-474 + ?BADARG(binary_to_atom(B, utf8)), %% system_limit failures. ?SYS_LIMIT(binary_to_atom(id(<<0:512/unit:8,255>>), utf8)), @@ -691,6 +715,9 @@ erlang_halt(Config) when is_list(Config) -> {badrpc,nodedown} = rpc:call(N3, erlang, halt, [0,[]]), {ok,N4} = slave:start(H, halt_node4), {badrpc,nodedown} = rpc:call(N4, erlang, halt, [lists:duplicate(300,$x)]), + %% Test unicode slogan + {ok,N4} = slave:start(H, halt_node4), + {badrpc,nodedown} = rpc:call(N4, erlang, halt, [[339,338,254,230,198,295,167,223,32,12507,12531,12480]]), % This test triggers a segfault when dumping a crash dump % to make sure that we can handle it properly. diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl index 4d17276e5c..61536bacd7 100644 --- a/erts/emulator/test/binary_SUITE.erl +++ b/erts/emulator/test/binary_SUITE.erl @@ -599,6 +599,9 @@ bad_binary_to_term(Config) when is_list(Config) -> %% Bad float. bad_bin_to_term(<<131,70,-1:64>>), + + %% Truncated UTF8 character (ERL-474) + bad_bin_to_term(<<131,119,1,194,163>>), ok. bad_bin_to_term(BadBin) -> diff --git a/erts/emulator/test/bs_construct_SUITE.erl b/erts/emulator/test/bs_construct_SUITE.erl index b79f4b995d..ce50bcdd86 100644 --- a/erts/emulator/test/bs_construct_SUITE.erl +++ b/erts/emulator/test/bs_construct_SUITE.erl @@ -905,14 +905,28 @@ bs_add_overflow(_Config) -> _ when Memsize < (2 bsl 30) -> {skip, "Less then 2 GB of memory"}; 4 -> - Large = <<0:((1 bsl 30)-1)>>, - {'EXIT',{system_limit,_}} = - (catch <<Large/bits, Large/bits, Large/bits, Large/bits, - Large/bits, Large/bits, Large/bits, Large/bits, - Large/bits>>), + {'EXIT', {system_limit, _}} = (catch bs_add_overflow_signed()), + {'EXIT', {system_limit, _}} = (catch bs_add_overflow_unsigned()), ok end. +bs_add_overflow_signed() -> + %% Produce a large result of bs_add that, if cast to signed int, would + %% overflow into a negative number that fits a smallnum. + Large = <<0:((1 bsl 30)-1)>>, + <<Large/bits, Large/bits, Large/bits, Large/bits, + Large/bits, Large/bits, Large/bits, Large/bits, + Large/bits>>. + +bs_add_overflow_unsigned() -> + %% Produce a large result of bs_add that goes beyond the limit of an + %% unsigned word. This used to succeed but produced an incorrect result + %% where B =:= C! + A = <<0:((1 bsl 32)-8)>>, + B = <<2, 3>>, + C = <<A/binary,1,B/binary>>, + true = byte_size(B) < byte_size(C). + id(I) -> I. memsize() -> diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl index b4ec99f902..17cd1d1a3b 100644 --- a/erts/emulator/test/distribution_SUITE.erl +++ b/erts/emulator/test/distribution_SUITE.erl @@ -56,6 +56,7 @@ bad_dist_ext_process_info/1, bad_dist_ext_control/1, bad_dist_ext_connection_id/1, + bad_dist_ext_size/1, start_epmd_false/1, epmd_module/1]). %% Internal exports. @@ -92,6 +93,7 @@ groups() -> [dist_auto_connect_never, dist_auto_connect_once]}, {bad_dist_ext, [], [bad_dist_ext_receive, bad_dist_ext_process_info, + bad_dist_ext_size, bad_dist_ext_control, bad_dist_ext_connection_id]}]. %% Tests pinging a node in different ways. @@ -418,18 +420,20 @@ make_busy(Node, Time) when is_integer(Time) -> Own = 500, freeze_node(Node, Time+Own), Data = make_busy_data(), + DCtrl = dctrl(Node), %% first make port busy Pid = spawn_link(fun () -> forever(fun () -> - dport_reg_send(Node, - '__noone__', - Data) + dctrl_dop_reg_send(Node, + '__noone__', + Data) end) end), receive after Own -> ok end, until(fun () -> - case process_info(Pid, status) of - {status, suspended} -> true; + case {DCtrl, process_info(Pid, status)} of + {DPrt, {status, suspended}} when is_port(DPrt) -> true; + {DPid, {status, waiting}} when is_pid(DPid) -> true; _ -> false end end), @@ -1672,6 +1676,57 @@ bad_dist_ext_connection_id(Config) when is_list(Config) -> stop_node(Offender), stop_node(Victim). +%% OTP-14661: Bad message is discovered by erts_msg_attached_data_size +bad_dist_ext_size(Config) when is_list(Config) -> + {ok, Offender} = start_node(bad_dist_ext_process_info_offender), + %%Prog = "Prog=/home/uabseri/src/otp_new3/bin/cerl -rr -debug", + Prog = [], + {ok, Victim} = start_node(bad_dist_ext_process_info_victim, [], Prog), + start_node_monitors([Offender,Victim]), + + Parent = self(), + P = spawn_link(Victim, + fun () -> + Parent ! {self(), started}, + receive check_msgs -> ok end, %% DID CRASH HERE + bad_dist_ext_check_msgs([one]), + Parent ! {self(), messages_checked} + end), + + receive {P, started} -> ok end, + P ! one, + + Suspended = make_ref(), + S = spawn(Victim, + fun () -> + erlang:suspend_process(P), + Parent ! Suspended, + receive after infinity -> ok end + end), + + receive Suspended -> ok end, + pong = rpc:call(Victim, net_adm, ping, [Offender]), + verify_up(Offender, Victim), + send_bad_msgs(Offender, P, 1, dmsg_bad_tag()), + + %% Make sure bad msgs has reached Victim + rpc:call(Offender, rpc, call, [Victim, erlang, node, []]), + + verify_still_up(Offender, Victim), + + rpc:call(Victim, erlang, process_info, [P, total_heap_size]), + + verify_down(Offender, connection_closed, Victim, killed), + + P ! check_msgs, + exit(S, bang), % resume Victim + receive {P, messages_checked} -> ok end, + + unlink(P), + verify_no_down(Offender, Victim), + stop_node(Offender), + stop_node(Victim). + bad_dist_struct_check_msgs([]) -> receive @@ -1703,37 +1758,38 @@ bad_dist_ext_check_msgs([M|Ms]) -> bad_dist_ext_check_msgs(Ms) end. +ensure_dctrl(Node) -> + case dctrl(Node) of + undefined -> + pong = net_adm:ping(Node), + dctrl(Node); + DCtrl -> + DCtrl + end. -dport_reg_send(Node, Name, Msg) -> - DPrt = case dport(Node) of - undefined -> - pong = net_adm:ping(Node), - dport(Node); - Prt -> - Prt - end, - port_command(DPrt, [dmsg_hdr(), - dmsg_ext({?DOP_REG_SEND, - self(), - ?COOKIE, - Name}), - dmsg_ext(Msg)]). - - -dport_send(To, Msg) -> +dctrl_send(DPrt, Data) when is_port(DPrt) -> + port_command(DPrt, Data); +dctrl_send(DPid, Data) when is_pid(DPid) -> + Ref = make_ref(), + DPid ! {send, self(), Ref, Data}, + receive {Ref, Res} -> Res end. + +dctrl_dop_reg_send(Node, Name, Msg) -> + dctrl_send(ensure_dctrl(Node), + [dmsg_hdr(), + dmsg_ext({?DOP_REG_SEND, + self(), + ?COOKIE, + Name}), + dmsg_ext(Msg)]). + +dctrl_dop_send(To, Msg) -> Node = node(To), - DPrt = case dport(Node) of - undefined -> - pong = net_adm:ping(Node), - dport(Node); - Prt -> - Prt - end, - port_command(DPrt, [dmsg_hdr(), - dmsg_ext({?DOP_SEND, - ?COOKIE, - To}), - dmsg_ext(Msg)]). + dctrl_send(ensure_dctrl(Node), + [dmsg_hdr(), + dmsg_ext({?DOP_SEND, ?COOKIE, To}), + dmsg_ext(Msg)]). + send_bad_structure(Offender,Victim,Bad,WhereToPutSelf) -> send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,[]). send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,PayLoad) -> @@ -1743,7 +1799,7 @@ send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,PayLoad) -> fun () -> Node = node(Victim), pong = net_adm:ping(Node), - DPrt = dport(Node), + DCtrl = dctrl(Node), Bad1 = case WhereToPutSelf of 0 -> Bad; @@ -1756,7 +1812,7 @@ send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,PayLoad) -> [] -> []; _Other -> [dmsg_ext(PayLoad)] end, - port_command(DPrt, DData), + dctrl_send(DCtrl, DData), Parent ! {DData,Done} end), receive @@ -1775,20 +1831,23 @@ send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,PayLoad) -> send_bad_msg(BadNode, To) -> send_bad_msgs(BadNode, To, 1). -send_bad_msgs(BadNode, To, Repeat) when is_atom(BadNode), - is_pid(To), - is_integer(Repeat) -> +send_bad_msgs(BadNode, To, Repeat) -> + send_bad_msgs(BadNode, To, Repeat, dmsg_bad_atom_cache_ref()). + +send_bad_msgs(BadNode, To, Repeat, BadTerm) when is_atom(BadNode), + is_pid(To), + is_integer(Repeat) -> Parent = self(), Done = make_ref(), spawn_link(BadNode, fun () -> Node = node(To), pong = net_adm:ping(Node), - DPrt = dport(Node), + DCtrl = dctrl(Node), DData = [dmsg_hdr(), dmsg_ext({?DOP_SEND, ?COOKIE, To}), - dmsg_bad_atom_cache_ref()], - repeat(fun () -> port_command(DPrt, DData) end, Repeat), + BadTerm], + repeat(fun () -> dctrl_send(DCtrl, DData) end, Repeat), Parent ! Done end), receive Done -> ok end. @@ -1810,11 +1869,12 @@ send_bad_ctl(BadNode, ToNode) when is_atom(BadNode), is_atom(ToNode) -> replace}), CtlBeginSize = size(Ctl) - size(Replace), <<CtlBegin:CtlBeginSize/binary, Replace/binary>> = Ctl, - port_command(dport(ToNode), - [dmsg_fake_hdr2(), - CtlBegin, - dmsg_bad_atom_cache_ref(), - dmsg_ext({a, message})]), + DCtrl = dctrl(ToNode), + Data = [dmsg_fake_hdr2(), + CtlBegin, + dmsg_bad_atom_cache_ref(), + dmsg_ext({a, message})], + dctrl_send(DCtrl, Data), Parent ! Done end), receive Done -> ok end. @@ -1827,17 +1887,17 @@ send_bad_dhdr(BadNode, ToNode) when is_atom(BadNode), is_atom(ToNode) -> spawn_link(BadNode, fun () -> pong = net_adm:ping(ToNode), - port_command(dport(ToNode), dmsg_bad_hdr()), + dctrl_send(dctrl(ToNode), dmsg_bad_hdr()), Parent ! Done end), receive Done -> ok end. -dport(Node) when is_atom(Node) -> +dctrl(Node) when is_atom(Node) -> case catch erts_debug:get_internal_state(available_internal_state) of true -> true; _ -> erts_debug:set_internal_state(available_internal_state, true) end, - erts_debug:get_internal_state({dist_port, Node}). + erts_debug:get_internal_state({dist_ctrl, Node}). dmsg_hdr() -> [131, % Version Magic @@ -1874,6 +1934,9 @@ dmsg_ext(Term) -> dmsg_bad_atom_cache_ref() -> [$R, 137]. +dmsg_bad_tag() -> %% Will fail early at heap size calculation + [$?, 66]. + start_epmd_false(Config) when is_list(Config) -> %% Start a node with the option -start_epmd false. {ok, OtherNode} = start_node(start_epmd_false, "-start_epmd false"), @@ -1979,7 +2042,7 @@ freeze_node(Node, MS) -> fun () -> erts_debug:set_internal_state(available_internal_state, true), - dport_send(Freezer, DoingIt), + dctrl_dop_send(Freezer, DoingIt), receive after Own -> ok end, erts_debug:set_internal_state(block, MS+Own) end), diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl index 5fca191679..33d0b708cf 100644 --- a/erts/emulator/test/driver_SUITE.erl +++ b/erts/emulator/test/driver_SUITE.erl @@ -43,9 +43,9 @@ outputv_errors/1, driver_unloaded/1, io_ready_exit/1, + use_fallback_pollset/0, use_fallback_pollset/1, bad_fd_in_pollset/1, - driver_event/1, fd_change/1, steal_control/1, otp_6602/1, @@ -58,11 +58,9 @@ ioq_exit_ready_output/1, ioq_exit_timeout/1, ioq_exit_ready_async/1, - ioq_exit_event/1, ioq_exit_ready_input_async/1, ioq_exit_ready_output_async/1, ioq_exit_timeout_async/1, - ioq_exit_event_async/1, zero_extended_marker_garb_drv/1, invalid_extended_marker_drv/1, larger_major_vsn_drv/1, @@ -86,6 +84,8 @@ -export([bin_prefix/2]). +-export([get_check_io_total/1]). % for z_SUITE.erl + -include_lib("common_test/include/ct.hrl"). @@ -119,17 +119,26 @@ -define(heap_binary_size, 64). init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> - case catch erts_debug:get_internal_state(available_internal_state) of - true -> ok; - _ -> erts_debug:set_internal_state(available_internal_state, true) - end, + CIOD = rpc(Config, + fun() -> + case catch erts_debug:get_internal_state(available_internal_state) of + true -> ok; + _ -> erts_debug:set_internal_state(available_internal_state, true) + end, + erts_debug:get_internal_state(check_io_debug) + end), erlang:display({init_per_testcase, Case}), - 0 = element(1, erts_debug:get_internal_state(check_io_debug)), + 0 = element(1, CIOD), [{testcase, Case}|Config]. -end_per_testcase(Case, _Config) -> +end_per_testcase(Case, Config) -> erlang:display({end_per_testcase, Case}), - 0 = element(1, erts_debug:get_internal_state(check_io_debug)), + CIOD = rpc(Config, + fun() -> + get_stable_check_io_info(), + erts_debug:get_internal_state(check_io_debug) + end), + 0 = element(1, CIOD), ok. suite() -> @@ -137,10 +146,13 @@ suite() -> {timetrap, {minutes, 1}}]. all() -> %% Keep a_test first and z_test last... - [a_test, outputv_errors, outputv_echo, queue_echo, {group, timer}, - driver_unloaded, io_ready_exit, use_fallback_pollset, - bad_fd_in_pollset, driver_event, fd_change, - steal_control, otp_6602, driver_system_info_base_ver, + [a_test, outputv_errors, outputv_echo, queue_echo, + {group, timer}, + driver_unloaded, io_ready_exit, otp_6602, + {group, polling}, + {group, poll_thread}, + {group, poll_set}, + driver_system_info_base_ver, driver_system_info_prev_ver, driver_system_info_current_ver, driver_monitor, {group, ioq_exit}, zero_extended_marker_garb_drv, @@ -148,7 +160,6 @@ all() -> %% Keep a_test first and z_test last... larger_minor_vsn_drv, smaller_major_vsn_drv, smaller_minor_vsn_drv, peek_non_existing_queue, otp_6879, caller, many_events, missing_callbacks, - smp_select, driver_select_use, thread_mseg_alloc_cache_clean, otp_9302, thr_free_drv, @@ -161,11 +172,18 @@ groups() -> [{timer, [], [timer_measure, timer_cancel, timer_delay, timer_change]}, + {poll_thread, [], [{group, polling}]}, + {poll_set, [], [{group, polling}]}, + {polling, [], + [a_test, use_fallback_pollset, + bad_fd_in_pollset, fd_change, + steal_control, smp_select, + driver_select_use, z_test]}, {ioq_exit, [], [ioq_exit_ready_input, ioq_exit_ready_output, - ioq_exit_timeout, ioq_exit_ready_async, ioq_exit_event, + ioq_exit_timeout, ioq_exit_ready_async, ioq_exit_ready_input_async, ioq_exit_ready_output_async, - ioq_exit_timeout_async, ioq_exit_event_async]}]. + ioq_exit_timeout_async]}]. init_per_suite(Config) -> Config. @@ -173,10 +191,28 @@ init_per_suite(Config) -> end_per_suite(_Config) -> catch erts_debug:set_internal_state(available_internal_state, false). +init_per_group(poll_thread, Config) -> + [{node_args, "+IOt 2"} | Config]; +init_per_group(poll_set, Config) -> + [{node_args, "+IOt 2 +IOp 2"} | Config]; +init_per_group(polling, Config) -> + case proplists:get_value(node_args, Config) of + undefined -> + Config; + Args -> + {ok, Node} = start_node(polling, Args), + [{node, Node} | Config] + end; init_per_group(_GroupName, Config) -> Config. end_per_group(_GroupName, Config) -> + case proplists:get_value(node, Config) of + undefined -> + ok; + Node -> + stop_node(Node) + end, Config. %% Test sending bad types to port with an outputv-capable driver. @@ -778,21 +814,23 @@ io_ready_exit(Config) when is_list(Config) -> -define(CHKIO_STOP, 0). -define(CHKIO_USE_FALLBACK_POLLSET, 1). -define(CHKIO_BAD_FD_IN_POLLSET, 2). --define(CHKIO_DRIVER_EVENT, 3). -define(CHKIO_FD_CHANGE, 4). -define(CHKIO_STEAL, 5). -define(CHKIO_STEAL_AUX, 6). -define(CHKIO_SMP_SELECT, 7). -define(CHKIO_DRV_USE, 8). +use_fallback_pollset() -> + [{timetrap, {minutes, 2}}]. + use_fallback_pollset(Config) when is_list(Config) -> + rpc(Config, fun() -> use_fallback_pollset_t(Config) end). + +use_fallback_pollset_t(Config) when is_list(Config) -> FlbkFun = fun () -> - ChkIoDuring = erlang:system_info(check_io), - case lists:keysearch(fallback_poll_set_size, - 1, - ChkIoDuring) of - {value, - {fallback_poll_set_size, N}} when N > 0 -> + {Flbk, _} = get_fallback(erlang:system_info(check_io)), + case lists:keysearch(total_poll_set_size, 1, Flbk) of + {value, {total_poll_set_size, N}} when N > 0 -> ok; Error -> ct:fail({failed_to_use_fallback, Error}) @@ -814,6 +852,7 @@ use_fallback_pollset(Config) when is_list(Config) -> Skip -> {fun () -> ok end, Skip, ok} end, + io:format("Node = ~p~n",[node()]), case chkio_test_fini(chkio_test(Handel, ?CHKIO_USE_FALLBACK_POLLSET, fun () -> @@ -825,27 +864,31 @@ use_fallback_pollset(Config) when is_list(Config) -> end. bad_fd_in_pollset(Config) when is_list(Config) -> - chkio_test_fini(chkio_test(chkio_test_init(Config), - ?CHKIO_BAD_FD_IN_POLLSET, - fun () -> sleep(1000) end)). - -driver_event(Config) when is_list(Config) -> - chkio_test_fini(chkio_test(chkio_test_init(Config), - ?CHKIO_DRIVER_EVENT, - fun () -> sleep(1000) end)). + rpc(Config, + fun() -> + chkio_test_fini(chkio_test(chkio_test_init(Config), + ?CHKIO_BAD_FD_IN_POLLSET, + fun () -> sleep(1000) end)) + end). fd_change(Config) when is_list(Config) -> - chkio_test_fini(chkio_test(chkio_test_init(Config), - ?CHKIO_FD_CHANGE, - fun () -> sleep(1000) end)). + rpc(Config, + fun() -> + chkio_test_fini(chkio_test(chkio_test_init(Config), + ?CHKIO_FD_CHANGE, + fun () -> sleep(1000) end)) + end). steal_control(Config) when is_list(Config) -> - chkio_test_fini(case chkio_test_init(Config) of - {erts_poll_info, _} = Hndl -> - steal_control_test(Hndl); - Skip -> - Skip - end). + rpc(Config, + fun() -> + chkio_test_fini(case chkio_test_init(Config) of + {erts_poll_info, _} = Hndl -> + steal_control_test(Hndl); + Skip -> + Skip + end) + end). steal_control_test(Hndl = {erts_poll_info, Before}) -> Port = open_chkio_port(), @@ -887,7 +930,7 @@ chkio_test_init(Config) when is_list(Config) -> ChkIo = get_stable_check_io_info(), case catch lists:keysearch(name, 1, ChkIo) of {value, {name, erts_poll}} -> - io:format("Before test: ~p~n", [ChkIo]), + ct:log("Before test: ~p~n", [ChkIo]), Path = proplists:get_value(data_dir, Config), erl_ddll:start(), ok = load_driver(Path, 'chkio_drv'), @@ -948,8 +991,9 @@ chkio_test({erts_poll_info, Before}, "ok" -> chk_chkio_port(Port), Fun(), - During = erlang:system_info(check_io), + During = get_check_io_total(erlang:system_info(check_io)), erlang:display(During), + 0 = element(1, erts_debug:get_internal_state(check_io_debug)), io:format("During test: ~p~n", [During]), chk_chkio_port(Port), @@ -1001,22 +1045,83 @@ verify_chkio_state(Before, After) -> ok. get_stable_check_io_info() -> - ChkIo = erlang:system_info(check_io), - PendUpdNo = case lists:keysearch(pending_updates, 1, ChkIo) of - {value, {pending_updates, PendNo}} -> - PendNo; - false -> - 0 - end, - {value, {active_fds, ActFds}} = lists:keysearch(active_fds, 1, ChkIo), + get_stable_check_io_info(10). +get_stable_check_io_info(0) -> + get_check_io_total(erlang:system_info(check_io)); +get_stable_check_io_info(N) -> + ChkIo = get_check_io_total(erlang:system_info(check_io)), + PendUpdNo = proplists:get_value(pending_updates, ChkIo, 0), + ActFds = proplists:get_value(active_fds, ChkIo), case {PendUpdNo, ActFds} of {0, 0} -> ChkIo; _ -> - receive after 10 -> ok end, - get_stable_check_io_info() + receive after 100 -> ok end, + get_stable_check_io_info(N-1) end. +%% Merge return from erlang:system_info(check_io) +%% as if it was one big pollset. +get_check_io_total(ChkIo) -> + ct:log("ChkIo = ~p~n",[ChkIo]), + {Fallback, Rest} = get_fallback(ChkIo), + add_fallback_infos(Fallback, + lists:foldl(fun(Pollset, Acc) -> + lists:zipwith(fun(A, B) -> + add_pollset_infos(A,B) + end, + Pollset, Acc) + end, + hd(Rest), tl(Rest))). + +add_pollset_infos({Tag, A}=TA , {Tag, B}=TB) -> + case tag_type(Tag) of + sum -> + {Tag, A + B}; + const -> + case A of + B -> TA; + _ -> + ct:fail("Unexpected diff in pollsets ~p != ~p", + [TA,TB]) + end + end. + +get_fallback([MaybeFallback | ChkIo] = AllChkIo) -> + case proplists:get_value(fallback, MaybeFallback) of + true -> + {MaybeFallback, ChkIo}; + false -> + {undefined, AllChkIo} + end. + +add_fallback_infos(undefined, Acc) -> + Acc; +add_fallback_infos(Flbk, Acc) -> + lists:zipwith(fun({Tag, A}=TA, {Tag, B}=TB) -> + case tag_type(Tag) of + sum -> {Tag, A + B}; + const when Tag =:= fallback -> TA; + const -> TB + end + end, + Flbk, Acc). + +tag_type(name) -> const; +tag_type(primary) -> const; +tag_type(fallback) -> const; +tag_type(kernel_poll) -> const; +tag_type(memory_size) -> sum; +tag_type(total_poll_set_size) -> sum; +tag_type(lazy_updates) -> const; +tag_type(pending_updates) -> sum; +tag_type(batch_updates) -> const; +tag_type(concurrent_updates) -> const; +tag_type(max_fds) -> const; +tag_type(active_fds) -> sum; +tag_type(poll_threads) -> sum. + + %% Missed port lock when stealing control of fd from a %% driver that didn't use the same lock. The lock checker %% used to trigger on this and dump core. @@ -1336,11 +1441,9 @@ driver_monitor(Config) when is_list(Config) -> -define(IOQ_EXIT_READY_OUTPUT, 2). -define(IOQ_EXIT_TIMEOUT, 3). -define(IOQ_EXIT_READY_ASYNC, 4). --define(IOQ_EXIT_EVENT, 5). -define(IOQ_EXIT_READY_INPUT_ASYNC, 6). -define(IOQ_EXIT_READY_OUTPUT_ASYNC, 7). -define(IOQ_EXIT_TIMEOUT_ASYNC, 8). --define(IOQ_EXIT_EVENT_ASYNC, 9). ioq_exit_test(Config, TestNo) -> Drv = ioq_exit_drv, @@ -1393,9 +1496,6 @@ ioq_exit_timeout(Config) when is_list(Config) -> ioq_exit_ready_async(Config) when is_list(Config) -> ioq_exit_test(Config, ?IOQ_EXIT_READY_ASYNC). -ioq_exit_event(Config) when is_list(Config) -> - ioq_exit_test(Config, ?IOQ_EXIT_EVENT). - ioq_exit_ready_input_async(Config) when is_list(Config) -> ioq_exit_test(Config, ?IOQ_EXIT_READY_INPUT_ASYNC). @@ -1405,9 +1505,6 @@ ioq_exit_ready_output_async(Config) when is_list(Config) -> ioq_exit_timeout_async(Config) when is_list(Config) -> ioq_exit_test(Config, ?IOQ_EXIT_TIMEOUT_ASYNC). -ioq_exit_event_async(Config) when is_list(Config) -> - ioq_exit_test(Config, ?IOQ_EXIT_EVENT_ASYNC). - vsn_mismatch_test(Config, LoadResult) -> Path = proplists:get_value(data_dir, Config), @@ -1641,7 +1738,7 @@ missing_callbacks(Config) when is_list(Config) -> smp_select(Config) when is_list(Config) -> case os:type() of {win32,_} -> {skipped, "Test not implemented for this OS"}; - _ -> smp_select0(Config) + _ -> rpc(Config, fun() -> smp_select0(Config) end) end. smp_select0(Config) -> @@ -1697,7 +1794,7 @@ smp_select_wait(Pids, TimeoutMsg) -> driver_select_use(Config) when is_list(Config) -> case os:type() of {win32,_} -> {skipped, "Test not implemented for this OS"}; - _ -> driver_select_use0(Config) + _ -> rpc(Config, fun() -> driver_select_use0(Config) end) end. driver_select_use0(Config) -> @@ -2264,10 +2361,10 @@ count_proc_sched(Ps, PNs) -> end. a_test(Config) when is_list(Config) -> - check_io_debug(). + rpc(Config, fun check_io_debug/0). z_test(Config) when is_list(Config) -> - check_io_debug(). + rpc(Config, fun check_io_debug/0). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Utilities @@ -2275,8 +2372,8 @@ z_test(Config) when is_list(Config) -> check_io_debug() -> get_stable_check_io_info(), - {NoErrorFds, NoUsedFds, NoDrvSelStructs, NoDrvEvStructs} = CheckIoDebug - = erts_debug:get_internal_state(check_io_debug), + {NoErrorFds, NoUsedFds, NoDrvSelStructs, NoEnifSelStructs} + = CheckIoDebug = erts_debug:get_internal_state(check_io_debug), HasGetHost = has_gethost(), ct:log("check_io_debug: ~p~n" "HasGetHost: ~p",[CheckIoDebug, HasGetHost]), @@ -2289,7 +2386,7 @@ check_io_debug() -> %% one extra used fd that is not selected on ok end, - 0 = NoDrvEvStructs, + 0 = NoEnifSelStructs, ok. has_gethost() -> @@ -2459,15 +2556,19 @@ sleep(Ms) when is_integer(Ms), Ms >= 0 -> start_node(Config) when is_list(Config) -> + start_node(proplists:get_value(testcase, Config)); +start_node(Name) -> + start_node(Name, ""). +start_node(NodeName, Args) -> Pa = filename:dirname(code:which(?MODULE)), Name = list_to_atom(atom_to_list(?MODULE) ++ "-" - ++ atom_to_list(proplists:get_value(testcase, Config)) + ++ atom_to_list(NodeName) ++ "-" ++ integer_to_list(erlang:system_time(second)) ++ "-" ++ integer_to_list(erlang:unique_integer([positive]))), - test_server:start_node(Name, slave, [{args, "-pa "++Pa}]). + test_server:start_node(Name, slave, [{args, Args ++ " -pa "++Pa}]). stop_node(Node) -> test_server:stop_node(Node). @@ -2500,3 +2601,35 @@ driver_alloc_size() -> Sz0+Sz end, 0, CS) end. + +rpc(Config, Fun) -> + case proplists:get_value(node, Config) of + undefined -> + Fun(); + Node -> + Self = self(), + Ref = make_ref(), + Pid = spawn(Node, + fun() -> + Result + = try Fun() of + Res -> Res + catch E:R -> + {'EXIT',E,R,erlang:get_stacktrace()} + end, + Self ! {Ref, Result} + end), + MRef = monitor(process, Pid), + receive + {'DOWN', MRef, _Type, _Object, Info} -> + erlang:error({died, Pid, Info}); + {Ref, {'EXIT',E,R,ST}} -> + erlang:demonitor(MRef, [flush]), + erlang:raise(E,R,ST); + {Ref, Ret} -> + erlang:demonitor(MRef, [flush]), + Ret; + Other -> + ct:fail(Other) + end + end. diff --git a/erts/emulator/test/driver_SUITE_data/chkio_drv.c b/erts/emulator/test/driver_SUITE_data/chkio_drv.c index 8e5e81665c..d548c4b1dc 100644 --- a/erts/emulator/test/driver_SUITE_data/chkio_drv.c +++ b/erts/emulator/test/driver_SUITE_data/chkio_drv.c @@ -42,7 +42,6 @@ #define CHKIO_STOP 0 #define CHKIO_USE_FALLBACK_POLLSET 1 #define CHKIO_BAD_FD_IN_POLLSET 2 -#define CHKIO_DRIVER_EVENT 3 #define CHKIO_FD_CHANGE 4 #define CHKIO_STEAL 5 #define CHKIO_STEAL_AUX 6 @@ -67,15 +66,6 @@ typedef struct { } ChkioFallbackData; typedef struct { - int in_fd; - struct erl_drv_event_data in_data; - int in_ok; - int out_fd; - struct erl_drv_event_data out_data; - int out_ok; -} ChkioDriverEvent; - -typedef struct { int fds[2]; int same_fd; } ChkioFdChange; @@ -86,14 +76,10 @@ typedef struct { typedef struct { int driver_select_fds[2]; - int driver_event_fds[2]; - struct erl_drv_event_data event_data[2]; } ChkioSteal; typedef struct { int driver_select_fds[2]; - int driver_event_fds[2]; - struct erl_drv_event_data event_data[2]; } ChkioStealAux; @@ -141,7 +127,6 @@ static ErlDrvData chkio_drv_start(ErlDrvPort, char *); static void chkio_drv_stop(ErlDrvData); static void chkio_drv_ready_input(ErlDrvData, ErlDrvEvent); static void chkio_drv_ready_output(ErlDrvData, ErlDrvEvent); -static void chkio_drv_ready_event(ErlDrvData, ErlDrvEvent, ErlDrvEventData); static ErlDrvSSizeT chkio_drv_control(ErlDrvData, unsigned int, char *, ErlDrvSizeT, char **, ErlDrvSizeT); static void chkio_drv_timeout(ErlDrvData); @@ -164,7 +149,7 @@ static ErlDrvEntry chkio_drv_entry = { NULL, /* ready_async */ NULL, /* flush */ NULL, /* call */ - chkio_drv_ready_event, + NULL, /* unused_event_callback */ ERL_DRV_EXTENDED_MARKER, ERL_DRV_EXTENDED_MAJOR_VERSION, @@ -243,25 +228,6 @@ stop_use_fallback_pollset(ChkioDrvData *cddp) } static void -stop_driver_event(ChkioDrvData *cddp) -{ - if (cddp->test_data) { - ChkioDriverEvent *cdep = cddp->test_data; - cddp->test_data = NULL; - - if (cdep->in_fd >= 0) { - driver_event(cddp->port, (ErlDrvEvent) (ErlDrvSInt) cdep->in_fd, NULL); - close(cdep->in_fd); - } - if (cdep->out_fd >= 0) { - driver_event(cddp->port, (ErlDrvEvent) (ErlDrvSInt) cdep->out_fd, NULL); - close(cdep->out_fd); - } - driver_free(cdep); - } -} - -static void stop_fd_change(ChkioDrvData *cddp) { if (cddp->test_data) { @@ -305,14 +271,6 @@ stop_steal(ChkioDrvData *cddp) (ErlDrvEvent) (ErlDrvSInt) csp->driver_select_fds[1], DO_WRITE, 0); - if (csp->driver_event_fds[0] >= 0) - driver_event(cddp->port, - (ErlDrvEvent) (ErlDrvSInt) csp->driver_event_fds[0], - NULL); - if (csp->driver_event_fds[1] >= 0) - driver_event(cddp->port, - (ErlDrvEvent) (ErlDrvSInt) csp->driver_event_fds[1], - NULL); driver_free(csp); } } @@ -327,10 +285,6 @@ stop_steal_aux(ChkioDrvData *cddp) close(csap->driver_select_fds[0]); if (csap->driver_select_fds[1] >= 0) close(csap->driver_select_fds[1]); - if (csap->driver_event_fds[0] >= 0) - close(csap->driver_event_fds[0]); - if (csap->driver_event_fds[1] >= 0) - close(csap->driver_event_fds[1]); driver_free(csap); } } @@ -354,10 +308,13 @@ static void free_smp_select(ChkioSmpSelect* pip, ErlDrvPort port) abort(); } case Selected: - driver_select(port, (ErlDrvEvent)(ErlDrvSInt)pip->read_fd, DO_READ, 0); - /*fall through*/ case Opened: - close(pip->read_fd); + TRACEF(("%T: Close pipe [%d->%d]\n", driver_mk_port(port), pip->write_fd, + pip->read_fd)); + if (pip->wasSelected) + driver_select(port, (ErlDrvEvent)(ErlDrvSInt)pip->read_fd, DO_READ|ERL_DRV_USE, 0); + else + close(pip->read_fd); close(pip->write_fd); pip->state = Closed; break; @@ -445,9 +402,6 @@ chkio_drv_stop(ErlDrvData drv_data) { case CHKIO_BAD_FD_IN_POLLSET: stop_bad_fd_in_pollset(cddp); break; - case CHKIO_DRIVER_EVENT: - stop_driver_event(cddp); - break; case CHKIO_FD_CHANGE: stop_fd_change(cddp); break; @@ -557,6 +511,9 @@ chkio_drv_ready_input(ErlDrvData drv_data, ErlDrvEvent event) driver_failure_atom(cddp->port, "input_fd_not_found"); break; } + case CHKIO_FD_CHANGE: + /* This may be triggered when an fd is closed while being selected on. */ + break; case CHKIO_STEAL: break; case CHKIO_STEAL_AUX: @@ -621,55 +578,6 @@ chkio_drv_ready_input(ErlDrvData drv_data, ErlDrvEvent event) } static void -chkio_drv_ready_event(ErlDrvData drv_data, - ErlDrvEvent event, - ErlDrvEventData event_data) -{ -#ifdef UNIX - ChkioDrvData *cddp = (ChkioDrvData *) drv_data; - switch (cddp->test) { - case CHKIO_DRIVER_EVENT: { -#ifdef HAVE_POLL_H - ChkioDriverEvent *cdep = cddp->test_data; - int fd = (int) (ErlDrvSInt) event; - if (fd == cdep->in_fd) { - if (event_data->events == POLLIN - && event_data->revents == POLLIN) { - cdep->in_ok++; - } - else { - driver_failure_atom(cddp->port, "invalid_input_fd_events"); - } - break; - } - if (fd == cdep->out_fd) { - if (event_data->events == POLLOUT - && event_data->revents == POLLOUT) { - cdep->out_ok++; - } - else { - driver_failure_atom(cddp->port, "invalid_output_fd_events"); - } - break; - } -#endif - } - case CHKIO_STEAL: -#ifdef HAVE_POLL_H - break; -#endif - case CHKIO_STEAL_AUX: -#ifdef HAVE_POLL_H - break; -#endif - default: - driver_failure_atom(cddp->port, "unexpected_ready_event"); - break; - } -#endif /* UNIX */ -} - -static void chkio_drv_timeout(ErlDrvData drv_data) { #ifdef UNIX @@ -779,25 +687,6 @@ chkio_drv_control(ErlDrvData drv_data, res_len = -1; stop_bad_fd_in_pollset(cddp); break; - case CHKIO_DRIVER_EVENT: { - ChkioDriverEvent *cdep = cddp->test_data; - if (!cdep->in_ok || !cdep->out_ok) { - if (!cdep->in_ok) - driver_failure_atom(cddp->port, "got_no_input_events"); - if (!cdep->out_ok) - driver_failure_atom(cddp->port, "got_no_output_events"); - } - else { - char *c = driver_alloc(sizeof(char)*2*30); - if (!c) - driver_failure_posix(cddp->port, ENOMEM); - *rbuf = c; - res_len = sprintf(c, "in=%d\nout=%d\n", - cdep->in_ok, cdep->out_ok); - } - stop_driver_event(cddp); - break; - } case CHKIO_FD_CHANGE: { ChkioFdChange *cfcp = cddp->test_data; if (!cfcp->same_fd) @@ -937,69 +826,6 @@ chkio_drv_control(ErlDrvData drv_data, res_len = -1; break; } - case CHKIO_DRIVER_EVENT: { -#ifndef HAVE_POLL_H - res_str = "skip: Need the poll.h header for this test, but it doesn't exist"; - res_len = -1; -#else /* HAVE_POLL_H */ - int in_fd = open("/dev/zero", O_RDONLY); - int out_fd = open("/dev/null", O_WRONLY); - - if (in_fd < 0 || out_fd < 0) { - if (in_fd >= 0) - close(in_fd); - if (out_fd >= 0) - close(out_fd); - driver_failure_posix(cddp->port, errno); - } - else { - ChkioDriverEvent *cdep = driver_alloc(sizeof(ChkioDriverEvent)); - if (!cdep) - driver_failure_posix(cddp->port, ENOMEM); - else { - int res; - cddp->test_data = cdep; - - cdep->in_fd = in_fd; - cdep->in_data.events = POLLIN; - cdep->in_data.revents = 0; - cdep->in_ok = 0; - - res = driver_event(cddp->port, - (ErlDrvEvent) (ErlDrvSInt) in_fd, - &cdep->in_data); - if (res < 0) { - res_str = "skip: driver_event() not supported"; - res_len = -1; - close(in_fd); - close(out_fd); - cdep->in_fd = -1; - cdep->out_fd = -1; - } - else { - res_str = "ok"; - res_len = -1; - - cdep->out_fd = out_fd; - cdep->out_data.events = POLLOUT; - cdep->out_data.revents = 0; - cdep->out_ok = 0; - - res = driver_event(cddp->port, - (ErlDrvEvent) (ErlDrvSInt) out_fd, - &cdep->out_data); - if (res < 0) { - close(out_fd); - cdep->out_fd = -1; - driver_failure_atom(cddp->port, "driver_event_failed"); - } - } - - } - } -#endif /* HAVE_POLL_H */ - break; - } case CHKIO_FD_CHANGE: { ChkioFdChange *cfcp = driver_alloc(sizeof(ChkioFdChange)); if (!cfcp) @@ -1028,58 +854,19 @@ chkio_drv_control(ErlDrvData drv_data, res_len = -1; } else { - int driver_event_fds[2]; int driver_select_fds[2]; cddp->test_data = csp; memcpy(c, buf, len); c[len] = '\0'; if (sscanf(c, - "fds:%d:%d:%d:%d", + "fds:%d:%d", &driver_select_fds[0], - &driver_select_fds[1], - &driver_event_fds[0], - &driver_event_fds[1]) != 4) - driver_failure_atom(cddp->port, "bad_input"); + &driver_select_fds[1]) != 2) + driver_failure_atom(cddp->port, "bad_input"); else { int res = 0; - if (driver_event_fds[0] < 0) { /* Have no working driver_event() ... */ - csp->driver_select_fds[0] = driver_select_fds[0]; /* In */ - csp->driver_select_fds[1] = driver_select_fds[1]; /* Out */ - csp->driver_event_fds[0] = -1; - csp->driver_event_fds[1] = -1; - } - else { /* Have working driver_event() ... */ -#ifndef HAVE_POLL_H - driver_failure_atom(cddp->port, "unexpected_result"); - res = -1; -#else - csp->driver_select_fds[0] = driver_select_fds[0]; /* In */ - csp->driver_event_fds[1] = driver_select_fds[1]; /* Out */ - csp->driver_event_fds[0] = driver_event_fds[0]; /* In */ - csp->driver_select_fds[1] = driver_event_fds[1]; /* Out */ - - /* Steal with driver_event() */ - - csp->event_data[0].events = POLLIN; - csp->event_data[0].revents = 0; - res = driver_event(cddp->port, - (ErlDrvEvent) (ErlDrvSInt) csp->driver_event_fds[0], - &csp->event_data[0]); - if (res < 0) - driver_failure_atom(cddp->port, - "driver_event_failed_to_steal"); - if (res >= 0) { - csp->event_data[1].events = POLLOUT; - csp->event_data[1].revents = 0; - res = driver_event(cddp->port, - (ErlDrvEvent) (ErlDrvSInt) csp->driver_event_fds[1], - &csp->event_data[1]); - if (res < 0) - driver_failure_atom(cddp->port, - "driver_event_failed_to_steal"); - } -#endif - } + csp->driver_select_fds[0] = driver_select_fds[0]; /* In */ + csp->driver_select_fds[1] = driver_select_fds[1]; /* Out */ /* Steal with driver_select() */ if (res >= 0) { @@ -1109,37 +896,17 @@ chkio_drv_control(ErlDrvData drv_data, break; } case CHKIO_STEAL_AUX: { - int read_fds[2]; - int write_fds[2]; + int read_fd; + int write_fd; - read_fds[0] = open("/dev/zero", O_RDONLY); - write_fds[0] = open("/dev/null", O_WRONLY); - -#ifdef HAVE_POLL_H - read_fds[1] = open("/dev/zero", O_RDONLY); - write_fds[1] = open("/dev/null", O_WRONLY); -#else - read_fds[1] = -1; - write_fds[1] = -1; -#endif + read_fd = open("/dev/zero", O_RDONLY); + write_fd = open("/dev/null", O_WRONLY); - if (read_fds[0] < 0 - || write_fds[0] < 0 -#ifdef HAVE_POLL_H - || read_fds[1] < 0 - || write_fds[1] < 0 -#endif - ) { - if (read_fds[0] < 0) - close(read_fds[0]); - if (write_fds[0] < 0) - close(write_fds[0]); -#ifdef HAVE_POLL_H - if (read_fds[1] < 0) - close(read_fds[1]); - if (write_fds[1] < 0) - close(write_fds[1]); -#endif + if (read_fd < 0 || write_fd < 0) { + if (read_fd < 0) + close(read_fd); + if (write_fd < 0) + close(write_fd); driver_failure_posix(cddp->port, errno); } else { @@ -1153,11 +920,8 @@ chkio_drv_control(ErlDrvData drv_data, int res; cddp->test_data = csap; - csap->driver_select_fds[0] = read_fds[0]; - csap->driver_select_fds[1] = write_fds[0]; - - csap->driver_event_fds[0] = read_fds[1]; - csap->driver_event_fds[1] = write_fds[1]; + csap->driver_select_fds[0] = read_fd; + csap->driver_select_fds[1] = write_fd; res = driver_select(cddp->port, (ErlDrvEvent) (ErlDrvSInt) csap->driver_select_fds[0], @@ -1173,32 +937,6 @@ chkio_drv_control(ErlDrvData drv_data, if (res < 0) driver_failure_atom(cddp->port, "driver_select_failed"); } -#ifdef HAVE_POLL_H - if (res >= 0) { - csap->event_data[0].events = POLLIN; - csap->event_data[0].revents = 0; - res = driver_event(cddp->port, - (ErlDrvEvent) (ErlDrvSInt) csap->driver_event_fds[0], - &csap->event_data[0]); - if (res < 0) { - close(csap->driver_event_fds[0]); - csap->driver_event_fds[0] = -1; - close(csap->driver_event_fds[1]); - csap->driver_event_fds[1] = -1; - res = 0; - } - else { - csap->event_data[1].events = POLLOUT; - csap->event_data[1].revents = 0; - res = driver_event(cddp->port, - (ErlDrvEvent) (ErlDrvSInt) csap->driver_event_fds[1], - &csap->event_data[1]); - if (res < 0) - driver_failure_atom(cddp->port, - "driver_event_failed"); - } - } -#endif if (res < 0) { res_str = "error"; res_len = -1; @@ -1213,11 +951,9 @@ chkio_drv_control(ErlDrvData drv_data, else { *rbuf = c; res_len = sprintf(c, - "fds:%d:%d:%d:%d", + "fds:%d:%d", csap->driver_select_fds[0], - csap->driver_select_fds[1], - csap->driver_event_fds[0], - csap->driver_event_fds[1]); + csap->driver_select_fds[1]); } } } @@ -1257,7 +993,7 @@ chkio_drv_control(ErlDrvData drv_data, } TRACEF(("%T: Created pipe [%d->%d]\n", cddp->id, fds[1], fds[0])); pip->read_fd = fds[0]; - pip->write_fd = fds[1]; + pip->write_fd = fds[1]; pip->state = Opened; pip->wasSelected = 0; pip->next_write = pip->next_read = rand_r(&pip->rand_state) % 1024; @@ -1267,7 +1003,8 @@ chkio_drv_control(ErlDrvData drv_data, }/*fall through*/ case Opened: { if (op & 1) { - TRACEF(("%T: Write %d to opened pipe [%d->%d]\n", cddp->id, pip->next_write, pip->write_fd, pip->read_fd)); + TRACEF(("%T: Write %d to opened pipe [%d->%d]\n", cddp->id, + pip->next_write, pip->write_fd, pip->read_fd)); if (write(pip->write_fd, &pip->next_write, sizeof(int)) != sizeof(int)) { fprintf(stderr, "Failed to write to pipe fd=%d, errno=%d\n", pip->write_fd, errno); abort(); @@ -1276,8 +1013,12 @@ chkio_drv_control(ErlDrvData drv_data, } op >>= 1; if (pip->wasSelected && (op & 1)) { - TRACEF(("%T: Close pipe [%d->%d]\n", cddp->id, pip->write_fd, pip->read_fd)); - if (close(pip->read_fd) || close(pip->write_fd)) { + TRACEF(("%T: Close pipe [%d->%d]\n", cddp->id, pip->write_fd, + pip->read_fd)); + drv_use_singleton.fd_stop_select = -2; /* disable stop_select asserts */ + if (driver_select(cddp->port, (ErlDrvEvent)(ErlDrvSInt)pip->read_fd, + DO_READ|ERL_DRV_USE, 0) + || close(pip->write_fd)) { fprintf(stderr, "Failed to close pipe, errno=%d\n", errno); abort(); } @@ -1285,8 +1026,10 @@ chkio_drv_control(ErlDrvData drv_data, break; } else { - TRACEF(("%T: Select on pipe [%d->%d]\n", cddp->id, pip->write_fd, pip->read_fd)); - if (driver_select(cddp->port, (ErlDrvEvent)(ErlDrvSInt)pip->read_fd, DO_READ, 1)) { + TRACEF(("%T: Select on pipe [%d->%d]\n", cddp->id, + pip->write_fd, pip->read_fd)); + if (driver_select(cddp->port, (ErlDrvEvent)(ErlDrvSInt)pip->read_fd, + DO_READ|ERL_DRV_USE, 1)) { fprintf(stderr, "driver_select failed for fd=%d\n", pip->read_fd); abort(); } @@ -1294,13 +1037,13 @@ chkio_drv_control(ErlDrvData drv_data, pip->wasSelected = 1; op >>= 1; if (pip->next_write != pip->next_read) { /* pipe not empty */ - if (op & 1) { + if (op & 1) { pip->state = Waiting; /* Wait for reader */ break; } op >>= 1; } - } + } }/*fall through*/ case Selected: if (op & 1) { @@ -1329,7 +1072,7 @@ chkio_drv_control(ErlDrvData drv_data, fprintf(stderr, "Failed to write to pipe fd=%d, errno=%d\n", pip->write_fd, errno); abort(); } - pip->next_write++; + pip->next_write++; } break; case Waiting: @@ -1583,7 +1326,12 @@ static void chkio_drv_stop_select(ErlDrvEvent e, void* null) if (!(drv_use_singleton.fd_stop_select < 0)) { assert_print("fd_stop_select<0", __LINE__); abort(); } - drv_use_singleton.fd_stop_select = (int)(long)e; + /* fd_stop_select counting is disabled if this is set to -2 */ + if (drv_use_singleton.fd_stop_select == -2) { + TRACEF(("closing %d\n", (int)(long)e)); + close((int)(long)e); + } else + drv_use_singleton.fd_stop_select = (int)(long)e; /* Can't call chkio_drv_use directly here. That could even be recursive. * Next timeout will detect it instead. */ diff --git a/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c b/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c index d87c2bec93..fa58e9d5ec 100644 --- a/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c +++ b/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c @@ -25,8 +25,7 @@ * - ready_input(), * - ready_output(), * - timeout(), - * - driver_async() -> read_async(), and - * - event() + * - driver_async() -> read_async() */ #ifndef UNIX @@ -65,11 +64,9 @@ typedef enum { IOQ_EXIT_READY_OUTPUT = 2, IOQ_EXIT_TIMEOUT = 3, IOQ_EXIT_READY_ASYNC = 4, - IOQ_EXIT_EVENT = 5, IOQ_EXIT_READY_INPUT_ASYNC = 6, IOQ_EXIT_READY_OUTPUT_ASYNC = 7, IOQ_EXIT_TIMEOUT_ASYNC = 8, - IOQ_EXIT_EVENT_ASYNC = 9 } IOQExitTest; typedef struct { @@ -80,9 +77,6 @@ typedef struct { int outstanding_async_task; long async_task; ErlDrvPDL pdl; -#ifdef HAVE_POLL_H - struct erl_drv_event_data event_data; -#endif } IOQExitDrvData; #define EV2FD(EV) ((int) ((long) (EV))) @@ -97,8 +91,6 @@ static ErlDrvSSizeT control(ErlDrvData, unsigned int, static void timeout(ErlDrvData drv_data); static void ready_async(ErlDrvData drv_data, ErlDrvThreadData thread_data); static void flush(ErlDrvData drv_data); -static void event(ErlDrvData drv_data, ErlDrvEvent event, - ErlDrvEventData event_data); static void async_invoke(void*); static void do_driver_async(IOQExitDrvData *); @@ -118,7 +110,7 @@ static ErlDrvEntry ioq_exit_drv_entry = { ready_async, flush, NULL /* call */, - event, + NULL /* unused_event_callback*/, ERL_DRV_EXTENDED_MARKER, ERL_DRV_EXTENDED_MAJOR_VERSION, ERL_DRV_EXTENDED_MINOR_VERSION, @@ -149,10 +141,6 @@ start(ErlDrvPort port, char *command) ddp->outstanding_async_task = 0; ddp->async_task = -1; ddp->pdl = driver_pdl_create(port); -#ifdef HAVE_POLL_H - ddp->event_data.events = (short) 0; - ddp->event_data.revents = (short) 0; -#endif return (ErlDrvData) ddp; } @@ -192,27 +180,6 @@ static ErlDrvSSizeT control(ErlDrvData drv_data, #else goto done; #endif - case IOQ_EXIT_EVENT: - case IOQ_EXIT_EVENT_ASYNC: -#ifdef UNIX -#ifdef HAVE_POLL_H - ddp->ofd = open("/dev/null", O_WRONLY); - if (ddp->ofd < 0) { - driver_failure_posix(ddp->port, errno); - return 0; - } - else if (driver_event(ddp->port, FD2EV(ddp->ofd), NULL) != 0) { - res_str = "skip: driver_event() not supported"; - goto done; - } -#else - res_str = "skip: No poll.h found which is needed for this test"; - goto done; -#endif - break; -#else /* UNIX */ - goto done; -#endif case IOQ_EXIT_TIMEOUT: case IOQ_EXIT_TIMEOUT_ASYNC: break; @@ -266,13 +233,6 @@ static void stop(ErlDrvData drv_data) close(ddp->ofd); } break; - case IOQ_EXIT_EVENT: - case IOQ_EXIT_EVENT_ASYNC: - if (ddp->ofd >= 0) { - driver_event(ddp->port, FD2EV(ddp->ofd), NULL); - close(ddp->ofd); - } - break; #endif case IOQ_EXIT_TIMEOUT: case IOQ_EXIT_TIMEOUT_ASYNC: @@ -302,13 +262,6 @@ static void flush(ErlDrvData drv_data) case IOQ_EXIT_READY_OUTPUT_ASYNC: driver_select(ddp->port, FD2EV(ddp->ofd), DO_WRITE, 1); break; - case IOQ_EXIT_EVENT: - case IOQ_EXIT_EVENT_ASYNC: -#ifdef HAVE_POLL_H - ddp->event_data.events |= POLLOUT; - driver_event(ddp->port, FD2EV(ddp->ofd), &ddp->event_data); -#endif - break; #endif case IOQ_EXIT_TIMEOUT: case IOQ_EXIT_TIMEOUT_ASYNC: @@ -395,30 +348,6 @@ static void ready_async(ErlDrvData drv_data, ErlDrvThreadData thread_data) } } -static void event(ErlDrvData drv_data, - ErlDrvEvent event, - ErlDrvEventData event_data) -{ - IOQExitDrvData *ddp = (IOQExitDrvData *) drv_data; - - PRINTF(("event(%p, %d, %p) called\r\n", drv_data, EV2FD(event), event_data)); - -#if defined(UNIX) && defined(HAVE_POLL_H) - if (ddp->ofd == EV2FD(event)) { - driver_event(ddp->port, FD2EV(ddp->ofd), NULL); - close(ddp->ofd); - ddp->ofd = -1; - if (ddp->test == IOQ_EXIT_EVENT_ASYNC) - do_driver_async(ddp); - else { - driver_pdl_lock(ddp->pdl); - driver_deq(ddp->port, 1); - driver_pdl_unlock(ddp->pdl); - } - } -#endif -} - static void async_invoke(void *arg) { PRINTF(("async_invoke(%p) called\r\n", arg)); diff --git a/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c b/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c index e7480d2e00..14838f0377 100644 --- a/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c +++ b/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c @@ -41,10 +41,6 @@ typedef struct { int ofd; int ifd; - int efd; -#ifdef HAVE_POLL_H - struct erl_drv_event_data edata; -#endif } mcd_data_t; static ErlDrvData start(ErlDrvPort port, char *command); @@ -90,7 +86,6 @@ start(ErlDrvPort port, char *command) mcd->ofd = -1; mcd->ifd = -1; - mcd->efd = -1; #ifdef UNIX @@ -105,15 +100,6 @@ start(ErlDrvPort port, char *command) goto error; if (driver_select(port, (ErlDrvEvent) (long) mcd->ifd, DO_READ, 1) != 0) goto error; - -#ifdef HAVE_POLL_H - mcd->efd = open("/dev/null", O_WRONLY); - if (mcd->efd < 0) - goto error; - mcd->edata.events = POLLOUT; - mcd->edata.revents = 0; - driver_event(port, (ErlDrvEvent) (long) mcd->efd, &mcd->edata); -#endif #endif driver_set_timer(port, 0); @@ -135,10 +121,6 @@ stop(ErlDrvData data) close(mcd->ofd); if (mcd->ifd >= 0) close(mcd->ifd); -#ifdef HAVE_POLL_H - if (mcd->efd >= 0) - close(mcd->efd); -#endif #endif driver_free(mcd); } diff --git a/erts/emulator/test/efile_SUITE.erl b/erts/emulator/test/efile_SUITE.erl index f0e1bcf04b..08d5597d78 100644 --- a/erts/emulator/test/efile_SUITE.erl +++ b/erts/emulator/test/efile_SUITE.erl @@ -19,16 +19,20 @@ -module(efile_SUITE). -export([all/0, suite/0]). --export([iter_max_files/1, async_dist/1]). +-export([async_dist/1, + iter_max_files/1, + proc_zero_sized_files/1 + ]). -export([do_iter_max_files/2, do_async_dist/1]). -include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [iter_max_files, async_dist]. + [iter_max_files, async_dist, proc_zero_sized_files]. do_async_dist(Dir) -> X = 100, @@ -162,3 +166,44 @@ open_files(Name) -> % io:format("Error reason: ~p", [_Reason]), [] end. + +%% @doc If /proc filesystem exists (no way to know if it is real proc or just +%% a /proc directory), let's read some zero sized files 500 times each, while +%% ensuring that response isn't empty << >> +proc_zero_sized_files(Config) when is_list(Config) -> + {Type, Flavor} = os:type(), + %% Some files which exist on Linux but might be missing on other systems + Inputs = ["/proc/cpuinfo", + "/proc/meminfo", + "/proc/partitions", + "/proc/swaps", + "/proc/version", + "/proc/uptime", + %% curproc is present on freebsd + "/proc/curproc/cmdline"], + case filelib:is_dir("/proc") of + false -> {skip, "/proc not found"}; % skip the test if no /proc + _ when Type =:= unix andalso Flavor =:= sunos -> + %% SunOS has a /proc, but no zero sized special files + {skip, "sunos does not have any zero sized special files"}; + true -> + %% Take away files which do not exist in proc + Inputs1 = lists:filter(fun filelib:is_file/1, Inputs), + + %% Fail if none of mentioned files exist in /proc, did we just get + %% a normal /proc directory without any special files? + ?assertNotEqual([], Inputs1), + + %% For 6 inputs and 500 attempts each this do run anywhere + %% between 500 and 3000 function calls. + lists:foreach( + fun(Filename) -> do_proc_zero_sized(Filename, 500) end, + Inputs1) + end. + +%% @doc Test one file N times to also trigger possible leaking fds and memory +do_proc_zero_sized(_Filename, 0) -> ok; +do_proc_zero_sized(Filename, N) -> + Data = file:read_file(Filename), + ?assertNotEqual(<<>>, Data), + do_proc_zero_sized(Filename, N-1). diff --git a/erts/emulator/test/emulator_smoke.spec b/erts/emulator/test/emulator_smoke.spec index b2d0de8835..fc98ba6823 100644 --- a/erts/emulator/test/emulator_smoke.spec +++ b/erts/emulator/test/emulator_smoke.spec @@ -7,3 +7,4 @@ [consistency],"Not reliable in October and March"}. {cases,'Dir',crypto_SUITE,[t_md5]}. {cases,'Dir',float_SUITE,[fpe,cmp_integer]}. +{cases,'Dir',erts_debug_SUITE,[df]}. diff --git a/erts/emulator/test/erl_link_SUITE.erl b/erts/emulator/test/erl_link_SUITE.erl index 5622cce980..d8c5b663e3 100644 --- a/erts/emulator/test/erl_link_SUITE.erl +++ b/erts/emulator/test/erl_link_SUITE.erl @@ -533,7 +533,7 @@ freeze_node(Node, MS) -> fun () -> erts_debug:set_internal_state(available_internal_state, true), - dport_send(Freezer, DoingIt), + dctrl_dop_send(Freezer, DoingIt), receive after Own -> ok end, erts_debug:set_internal_state(block, MS+Own) end), @@ -544,20 +544,22 @@ make_busy(Node, Time) when is_integer(Time) -> Own = 500, freeze_node(Node, Time+Own), Data = busy_data(), + DCtrl = dctrl(Node), %% first make port busy Pid = spawn_link(fun () -> forever(fun () -> - dport_reg_send(Node, - '__noone__', - Data) + dctrl_dop_reg_send(Node, + '__noone__', + Data) end) end), receive after Own -> ok end, wait_until(fun () -> - case process_info(Pid, status) of - {status, suspended} -> true; - _ -> false - end + case {DCtrl, process_info(Pid, status)} of + {DPrt, {status, suspended}} when is_port(DPrt) -> true; + {DPid, {status, waiting}} when is_pid(DPid) -> true; + _ -> false + end end), %% then dist entry make_busy(Node, [nosuspend], Data), @@ -1048,42 +1050,45 @@ stop_node(Node) -> -define(DOP_DEMONITOR_P, 20). -define(DOP_MONITOR_P_EXIT, 21). -dport_send(To, Msg) -> - Node = node(To), - DPrt = case dport(Node) of - undefined -> - pong = net_adm:ping(Node), - dport(Node); - Prt -> - Prt - end, - port_command(DPrt, [dmsg_hdr(), - dmsg_ext({?DOP_SEND, - ?COOKIE, - To}), - dmsg_ext(Msg)]). - -dport_reg_send(Node, Name, Msg) -> - DPrt = case dport(Node) of - undefined -> - pong = net_adm:ping(Node), - dport(Node); - Prt -> - Prt - end, - port_command(DPrt, [dmsg_hdr(), - dmsg_ext({?DOP_REG_SEND, - self(), - ?COOKIE, - Name}), - dmsg_ext(Msg)]). - -dport(Node) when is_atom(Node) -> +ensure_dctrl(Node) -> + case dctrl(Node) of + undefined -> + pong = net_adm:ping(Node), + dctrl(Node); + DCtrl -> + DCtrl + end. + +dctrl_send(DPrt, Data) when is_port(DPrt) -> + port_command(DPrt, Data); +dctrl_send(DPid, Data) when is_pid(DPid) -> + Ref = make_ref(), + DPid ! {send, self(), Ref, Data}, + receive {Ref, Res} -> Res end. + +dctrl_dop_send(To, Msg) -> + dctrl_send(ensure_dctrl(node(To)), + [dmsg_hdr(), + dmsg_ext({?DOP_SEND, + ?COOKIE, + To}), + dmsg_ext(Msg)]). + +dctrl_dop_reg_send(Node, Name, Msg) -> + dctrl_send(ensure_dctrl(Node), + [dmsg_hdr(), + dmsg_ext({?DOP_REG_SEND, + self(), + ?COOKIE, + Name}), + dmsg_ext(Msg)]). + +dctrl(Node) when is_atom(Node) -> case catch erts_debug:get_internal_state(available_internal_state) of true -> true; _ -> erts_debug:set_internal_state(available_internal_state, true) end, - erts_debug:get_internal_state({dist_port, Node}). + erts_debug:get_internal_state({dist_ctrl, Node}). dmsg_hdr() -> [131, % Version Magic diff --git a/erts/emulator/test/estone_SUITE.erl b/erts/emulator/test/estone_SUITE.erl index 8b336b366d..c9c1867049 100644 --- a/erts/emulator/test/estone_SUITE.erl +++ b/erts/emulator/test/estone_SUITE.erl @@ -20,7 +20,7 @@ -module(estone_SUITE). %% Test functions -export([all/0, suite/0, groups/0, - estone/1, estone_bench/1]). + estone/1, estone_bench/1, pgo/0]). %% Internal exports for EStone tests -export([lists/1, @@ -44,9 +44,9 @@ links/1,lproc/1, run_micro/3,p1/1,ppp/3,macro/2,micros/0]). - --include_lib("common_test/include/ct.hrl"). +-ifndef(PGO). -include_lib("common_test/include/ct_event.hrl"). +-endif. %% EStone defines -define(TOTAL, (3000 * 1000 * 100)). %% 300 secs @@ -85,13 +85,28 @@ estone(Config) when is_list(Config) -> estone_bench(Config) -> DataDir = proplists:get_value(data_dir,Config), L = ?MODULE:macro(?MODULE:micros(),DataDir), - [ct_event:notify( - #event{name = benchmark_data, - data = [{name,proplists:get_value(title,Mark)}, - {value,proplists:get_value(estones,Mark)}]}) - || Mark <- L], + {Total, Stones} = sum_micros(L, 0, 0), + notify([[{title,"ESTONES"}, {estones, Stones}] | L]), L. +-ifndef(PGO). +notify(Marks) -> + [ct_event:notify( + #event{name = benchmark_data, + data = [{name,proplists:get_value(title, Mark)}, + {value,proplists:get_value(estones, Mark)}]}) + || Mark <- Marks]. +-else. +notify(_) -> + ok. +-endif. + +%% The benchmarks to run in order to guide PGO (profile guided optimisation) +pgo() -> + %% We run all benchmarks except the port_io as we don't want to + %% have to build a custom port. + Micros = ?MODULE:micros() -- [micro(port_io)], + ?MODULE:macro(Micros,[]). %% %% Calculate CPU speed @@ -364,7 +379,7 @@ monotonic_time() -> try erlang:monotonic_time() catch error:undef -> erlang:now() end. subtr(Before, After) when is_integer(Before), is_integer(After) -> - erlang:convert_time_unit(After-Before, native, microsecond); + erlang:convert_time_unit(After-Before, native, 1000000); subtr({_,_,_}=Before, {_,_,_}=After) -> timer:now_diff(After, Before). diff --git a/erts/emulator/test/exception_SUITE.erl b/erts/emulator/test/exception_SUITE.erl index e473a10be7..0f27251fcb 100644 --- a/erts/emulator/test/exception_SUITE.erl +++ b/erts/emulator/test/exception_SUITE.erl @@ -662,6 +662,15 @@ line_numbers(Config) when is_list(Config) -> {?MODULE,line_numbers,1,_}|_]}} = (catch applied_bif_2()), + {'EXIT',{badarith, + [{?MODULE,increment1,1,[{file,"increment.erl"},{line,45}]}, + {?MODULE,line_numbers,1,_}|_]}} = + (catch increment1(x)), + {'EXIT',{badarith, + [{?MODULE,increment2,1,[{file,"increment.erl"},{line,48}]}, + {?MODULE,line_numbers,1,_}|_]}} = + (catch increment2(x)), + ok. id(I) -> I. @@ -762,3 +771,15 @@ applied_bif_2() -> %Line 8 R = process_info(self(), current_location), %Line 9 fail = R, %Line 10 ok. %Line 11 + +%% The increment instruction used to decrement the instruction +%% pointer, which would cause the line number in a stack trace to +%% be the previous line number. + +-file("increment.erl", 42). +increment1(Arg) -> %Line 43 + Res = id(Arg), %Line 44 + Res + 1. %Line 45 +increment2(Arg) -> %Line 46 + _ = id(Arg), %Line 47 + Arg + 1. %Line 48 diff --git a/erts/emulator/test/iovec_SUITE.erl b/erts/emulator/test/iovec_SUITE.erl new file mode 100644 index 0000000000..49dc64b0d2 --- /dev/null +++ b/erts/emulator/test/iovec_SUITE.erl @@ -0,0 +1,176 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(iovec_SUITE). + +-export([all/0, suite/0, init_per_suite/1, end_per_suite/1]). + +-export([integer_lists/1, binary_lists/1, empty_lists/1, empty_binary_lists/1, + mixed_lists/1, improper_lists/1, illegal_lists/1, cons_bomb/1, + sub_binary_lists/1, iolist_to_iovec_idempotence/1, + iolist_to_iovec_correctness/1]). + +-include_lib("common_test/include/ct.hrl"). + +suite() -> + [{ct_hooks,[ts_install_cth]}, + {timetrap, {minutes, 2}}]. + +all() -> + [integer_lists, binary_lists, empty_lists, empty_binary_lists, mixed_lists, + sub_binary_lists, illegal_lists, improper_lists, cons_bomb, + iolist_to_iovec_idempotence, iolist_to_iovec_correctness]. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + application:stop(os_mon), + Config. + +integer_lists(Config) when is_list(Config) -> + Variations = gen_variations([I || I <- lists:seq(1, 255)]), + equivalence_test(fun erlang:iolist_to_iovec/1, Variations). + +sub_binary_lists(Config) when is_list(Config) -> + Parent = <<0:256/unit:8, "gazurka">>, + <<0:196/unit:8, Child/binary>> = Parent, + equivalence_test(fun erlang:iolist_to_iovec/1, gen_variations(Child)). + +binary_lists(Config) when is_list(Config) -> + Variations = gen_variations([<<I:8>> || I <- lists:seq(1, 255)]), + equivalence_test(fun erlang:iolist_to_iovec/1, Variations). + +empty_lists(Config) when is_list(Config) -> + Variations = gen_variations([[] || _ <- lists:seq(1, 256)]), + equivalence_test(fun erlang:iolist_to_iovec/1, Variations), + [] = erlang:iolist_to_iovec([]), + ok. + +empty_binary_lists(Config) when is_list(Config) -> + Variations = gen_variations([<<>> || _ <- lists:seq(1, 8192)]), + equivalence_test(fun erlang:iolist_to_iovec/1, Variations), + [] = erlang:iolist_to_iovec(Variations), + ok. + +mixed_lists(Config) when is_list(Config) -> + Variations = gen_variations([<<>>, lists:seq(1, 40), <<12, 45, 78>>]), + equivalence_test(fun erlang:iolist_to_iovec/1, Variations). + +illegal_lists(Config) when is_list(Config) -> + BitStrs = gen_variations(["gurka", <<1:1>>, "gaffel"]), + BadInts = gen_variations(["gurka", 890, "gaffel"]), + Atoms = gen_variations([gurka, "gaffel"]), + BadTails = [["test" | 0], ["gurka", gaffel]], + + Variations = + BitStrs ++ BadInts ++ Atoms ++ BadTails, + + illegality_test(fun erlang:iolist_to_iovec/1, Variations). + +improper_lists(Config) when is_list(Config) -> + Variations = [ + [[[[1 | <<2>>] | <<3>>] | <<4>>] | <<5>>], + [[<<1>>, 2] | <<3, 4, 5>>], + [1, 2, 3 | <<4, 5>>] + ], + equivalence_test(fun erlang:iolist_to_iovec/1, Variations). + +cons_bomb(Config) when is_list(Config) -> + IntBase = gen_variations([I || I <- lists:seq(1, 255)]), + BinBase = gen_variations([<<I:8>> || I <- lists:seq(1, 255)]), + MixBase = gen_variations([<<12, 45, 78>>, lists:seq(1, 255)]), + + Rounds = + case system_mem_size() of + Mem when Mem >= (16 bsl 30) -> 32; + Mem when Mem >= (3 bsl 30) -> 28; + _ -> 20 + end, + + Variations = gen_variations([IntBase, BinBase, MixBase], Rounds), + equivalence_test(fun erlang:iolist_to_iovec/1, Variations). + +iolist_to_iovec_idempotence(Config) when is_list(Config) -> + IntVariations = gen_variations([I || I <- lists:seq(1, 255)]), + BinVariations = gen_variations([<<I:8>> || I <- lists:seq(1, 255)]), + MixVariations = gen_variations([<<12, 45, 78>>, lists:seq(1, 255)]), + + Variations = [IntVariations, BinVariations, MixVariations], + Optimized = erlang:iolist_to_iovec(Variations), + + true = Optimized =:= erlang:iolist_to_iovec(Optimized), + ok. + +iolist_to_iovec_correctness(Config) when is_list(Config) -> + IntVariations = gen_variations([I || I <- lists:seq(1, 255)]), + BinVariations = gen_variations([<<I:8>> || I <- lists:seq(1, 255)]), + MixVariations = gen_variations([<<12, 45, 78>>, lists:seq(1, 255)]), + + Variations = [IntVariations, BinVariations, MixVariations], + Optimized = erlang:iolist_to_iovec(Variations), + + true = is_iolist_equal(Optimized, Variations), + ok. + +illegality_test(Fun, Variations) -> + [{'EXIT',{badarg, _}} = (catch Fun(Variation)) || Variation <- Variations], + ok. + +equivalence_test(Fun, [Head | _] = Variations) -> + %% Check that each variation is equal to the others, and that the sum of + %% them is equal to the input. + Comparand = Fun(Head), + [true = is_iolist_equal(Comparand, Fun(V)) || V <- Variations], + true = is_iolist_equal(Variations, Fun(Variations)), + ok. + +is_iolist_equal(A, B) -> + iolist_to_binary(A) =:= iolist_to_binary(B). + +%% Generates a bunch of lists whose contents will be equal to Base repeated a +%% few times. The lists only differ by their structure, so their reduction to +%% a simpler format should yield the same result. +gen_variations(Base) -> + gen_variations(Base, 16). +gen_variations(Base, N) -> + [gen_flat_list(Base, N), + gen_nested_list(Base, N), + gen_nasty_list(Base, N)]. + +gen_flat_list(Base, N) -> + lists:flatten(gen_nested_list(Base, N)). + +gen_nested_list(Base, N) -> + [Base || _ <- lists:seq(1, N)]. + +gen_nasty_list(Base, N) -> + gen_nasty_list_1(gen_nested_list(Base, N), []). +gen_nasty_list_1([], Result) -> + Result; +gen_nasty_list_1([Head | Base], Result) when is_list(Head) -> + gen_nasty_list_1(Base, [[Result], [gen_nasty_list_1(Head, [])]]); +gen_nasty_list_1([Head | Base], Result) -> + gen_nasty_list_1(Base, [[Result], [Head]]). + +system_mem_size() -> + application:ensure_all_started(os_mon), + {Tot,_Used,_} = memsup:get_memory_data(), + Tot. diff --git a/erts/emulator/test/lttng_SUITE.erl b/erts/emulator/test/lttng_SUITE.erl index a012fa1da2..19c3844c40 100644 --- a/erts/emulator/test/lttng_SUITE.erl +++ b/erts/emulator/test/lttng_SUITE.erl @@ -81,7 +81,6 @@ end_per_testcase(Case, _Config) -> %% Not tested yet %% org_erlang_otp:driver_process_exit -%% org_erlang_otp:driver_event %% tracepoints %% @@ -100,7 +99,6 @@ end_per_testcase(Case, _Config) -> %% org_erlang_otp:driver_flush %% org_erlang_otp:driver_stop_select %% org_erlang_otp:driver_timeout -%% org_erlang_otp:driver_event %% org_erlang_otp:driver_ready_output %% org_erlang_otp:driver_ready_input %% org_erlang_otp:driver_output @@ -431,7 +429,6 @@ txt() -> "%% org_erlang_otp:driver_flush\n" "%% org_erlang_otp:driver_stop_select\n" "%% org_erlang_otp:driver_timeout\n" - "%% org_erlang_otp:driver_event\n" "%% org_erlang_otp:driver_ready_output\n" "%% org_erlang_otp:driver_ready_input\n" "%% org_erlang_otp:driver_output\n" diff --git a/erts/emulator/test/match_spec_SUITE.erl b/erts/emulator/test/match_spec_SUITE.erl index 92ddc23592..08a7b4560c 100644 --- a/erts/emulator/test/match_spec_SUITE.erl +++ b/erts/emulator/test/match_spec_SUITE.erl @@ -21,7 +21,7 @@ -module(match_spec_SUITE). -export([all/0, suite/0, not_run/1]). --export([test_1/1, test_2/1, test_3/1, bad_match_spec_bin/1, +-export([test_1/1, test_2/1, test_3/1, caller_and_return_to/1, bad_match_spec_bin/1, trace_control_word/1, silent/1, silent_no_ms/1, silent_test/1, ms_trace2/1, ms_trace3/1, ms_trace_dead/1, boxed_and_small/1, destructive_in_test_bif/1, guard_exceptions/1, @@ -47,7 +47,7 @@ suite() -> all() -> case test_server:is_native(match_spec_SUITE) of false -> - [test_1, test_2, test_3, bad_match_spec_bin, + [test_1, test_2, test_3, caller_and_return_to, bad_match_spec_bin, trace_control_word, silent, silent_no_ms, silent_test, ms_trace2, ms_trace3, ms_trace_dead, boxed_and_small, destructive_in_test_bif, guard_exceptions, unary_plus, unary_minus, fpe, @@ -180,6 +180,50 @@ test_3(Config) when is_list(Config) -> collect(P1, [{trace, P1, call, {?MODULE, f2, [a, b]}, [true]}]), ok. +%% Test that caller and return to work as they should +%% There was a bug where caller would be undefined when return_to was set +%% for r the bif erlang:put(). +caller_and_return_to(Config) -> + tr( + fun do_put_wrapper/0, + fun (Tracee) -> + MsgCaller = [{'_',[],[{message,{caller}}]}], + 1 = erlang:trace(Tracee, true, [call,return_to]), + 1 = erlang:trace_pattern( {?MODULE,do_put,1}, MsgCaller, [local]), + 1 = erlang:trace_pattern( {?MODULE,do_the_put,1}, MsgCaller, [local]), + 1 = erlang:trace_pattern( {erlang,integer_to_list,1}, MsgCaller, [local]), + 1 = erlang:trace_pattern( {erlang,put,2}, MsgCaller, [local]), + + [{trace,Tracee,call,{?MODULE,do_put,[test]},{?MODULE,do_put_wrapper,0}}, + {trace,Tracee,call,{?MODULE,do_the_put,[test]},{?MODULE,do_put,1}}, + {trace,Tracee,call,{erlang,integer_to_list,[1]},{?MODULE,do_the_put,1}}, + {trace,Tracee,return_to,{?MODULE,do_the_put,1}}, + {trace,Tracee,call,{erlang,put,[test,"1"]},{?MODULE,do_put,1}}, + {trace,Tracee,return_to,{?MODULE,do_put,1}}, + + %% These last trace messages are a bit strange... + %% if call tracing had been enabled for do_put_wrapper + %% then caller and return_to would have been {?MODULE,do_put_wrapper,1} + %% but since it is not, they are set to do_put instead, but we still + %% get the do_put_wrapper return_to message... + {trace,Tracee,call,{erlang,integer_to_list,[2]},{?MODULE,do_put,1}}, + {trace,Tracee,return_to,{?MODULE,do_put,1}}, + {trace,Tracee,return_to,{?MODULE,do_put_wrapper,0}} + ] + end), + ok. + +do_put_wrapper() -> + do_put(test), + ok. + +do_put(Var) -> + do_the_put(Var), + erlang:integer_to_list(id(2)). +do_the_put(Var) -> + Lst = erlang:integer_to_list(id(1)), + erlang:put(Var, Lst). + otp_9422(Config) when is_list(Config) -> Laps = 10000, Fun1 = fun() -> otp_9422_tracee() end, diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index ef66f0bbfc..bec2291867 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -25,13 +25,14 @@ %%-define(CHECK(Exp,Got), Exp = Got). -include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). -export([all/0, suite/0, groups/0, init_per_group/2, end_per_group/2, init_per_testcase/2, end_per_testcase/2, basic/1, reload_error/1, upgrade/1, heap_frag/1, t_on_load/1, - select/1, + select/1, select_steal/1, monitor_process_a/1, monitor_process_b/1, monitor_process_c/1, @@ -42,9 +43,9 @@ types/1, many_args/1, binaries/1, get_string/1, get_atom/1, maps/1, api_macros/1, - from_array/1, iolist_as_binary/1, resource/1, resource_binary/1, + from_array/1, iolist_as_binary/1, resource/1, resource_binary/1, resource_takeover/1, - threading/1, send/1, send2/1, send3/1, send_threaded/1, neg/1, + threading/1, send/1, send2/1, send3/1, send_threaded/1, neg/1, is_checks/1, get_length/1, make_atom/1, make_string/1, reverse_list_test/1, otp_9828/1, @@ -61,7 +62,8 @@ nif_internal_hash_salted/1, nif_phash2/1, nif_whereis/1, nif_whereis_parallel/1, - nif_whereis_threaded/1, nif_whereis_proxy/1 + nif_whereis_threaded/1, nif_whereis_proxy/1, + nif_ioq/1 ]). -export([many_args_100/100]). @@ -78,7 +80,7 @@ all() -> [{group, G} || G <- api_groups()] ++ [reload_error, heap_frag, types, many_args, - select, + select, select_steal, {group, monitor}, monitor_frenzy, hipe, @@ -99,7 +101,8 @@ all() -> nif_internal_hash, nif_internal_hash_salted, nif_phash2, - nif_whereis, nif_whereis_parallel, nif_whereis_threaded]. + nif_whereis, nif_whereis_parallel, nif_whereis_threaded, + nif_ioq]. groups() -> [{G, [], api_repeaters()} || G <- api_groups()] @@ -142,7 +145,8 @@ init_per_testcase(nif_whereis_threaded, Config) -> true -> Config; false -> {skip, "No thread support"} end; -init_per_testcase(select, Config) -> +init_per_testcase(Select, Config) when Select =:= select; + Select =:= select_steal -> case os:type() of {win32,_} -> {skip, "Test not yet implemented for windows"}; @@ -150,6 +154,9 @@ init_per_testcase(select, Config) -> Config end; init_per_testcase(_Case, Config) -> + %% Clear any resource dtor data before test starts in case another tc + %% left it in a bad state + catch last_resource_dtor_call(), Config. end_per_testcase(t_on_load, _Config) -> @@ -588,7 +595,71 @@ select_3(_Config) -> {_,_,2} = last_resource_dtor_call(), ok. -check_stop_ret(?ERL_NIF_SELECT_STOP_CALLED) -> ok; +%% @doc The stealing child process for the select_steal test. Duplicates given +%% W/RFds and runs select on them to steal +select_steal_child_process(Parent, RFd) -> + %% Duplicate the resource with the same FD + {R2Fd, _R2Ptr} = dupe_resource_nif(RFd), + Ref2 = make_ref(), + + %% Try to select from the child pid (steal from parent) + ?assertEqual(0, select_nif(R2Fd, ?ERL_NIF_SELECT_READ, R2Fd, null, Ref2)), + ?assertEqual([], flush(0)), + ?assertEqual(eagain, read_nif(R2Fd, 1)), + + %% Check that now events arrive to this temporary process + Parent ! {self(), stage1}, % signal parent to send the <<"stolen1">> + + %% Receive <<"stolen1">> via enif_select + ?assertEqual(0, select_nif(R2Fd, ?ERL_NIF_SELECT_READ, R2Fd, null, Ref2)), + ?assertMatch([{select, R2Fd, Ref2, ready_input}], flush()), + ?assertEqual(<<"stolen1">>, read_nif(R2Fd, 7)), + + clear_select_nif(R2Fd), + + % do not do this here - stop_selecting(R2Fd, R2Rsrc, Ref2), + Parent ! {self(), done}. + +%% @doc Similar to select/1 test, make a double ended pipe. Then try to steal +%% the socket, see what happens. +select_steal(Config) when is_list(Config) -> + ensure_lib_loaded(Config), + + Ref = make_ref(), + {{RFd, RPtr}, {WFd, WPtr}} = pipe_nif(), + + %% Bind the socket to current pid in enif_select + ?assertEqual(0, select_nif(RFd, ?ERL_NIF_SELECT_READ, RFd, null, Ref)), + ?assertEqual([], flush(0)), + + %% Spawn a process and do some stealing + Parent = self(), + Pid = spawn_link(fun() -> select_steal_child_process(Parent, RFd) end), + + %% Signal from the child to send the first message + {Pid, stage1} = receive_any(), + ?assertEqual(ok, write_nif(WFd, <<"stolen1">>)), + + ?assertMatch([{Pid, done}], flush(1)), % synchronize with the child + + %% Try to select from the parent pid (steal back) + ?assertEqual(0, select_nif(RFd, ?ERL_NIF_SELECT_READ, RFd, Pid, Ref)), + + %% Ensure that no data is hanging and close. + %% Rfd is stolen at this point. + check_stop_ret(select_nif(WFd, ?ERL_NIF_SELECT_STOP, WFd, null, Ref)), + ?assertMatch([{fd_resource_stop, WPtr, _}], flush()), + {1, {WPtr, 1}} = last_fd_stop_call(), + + check_stop_ret(select_nif(RFd, ?ERL_NIF_SELECT_STOP, RFd, null, Ref)), + ?assertMatch([{fd_resource_stop, RPtr, _}], flush()), + {1, {RPtr, 1}} = last_fd_stop_call(), + + ?assert(is_closed_nif(WFd)), + + ok. + +check_stop_ret(?ERL_NIF_SELECT_STOP_CALLED) -> ok; check_stop_ret(?ERL_NIF_SELECT_STOP_SCHEDULED) -> ok. write_full(W, C) -> @@ -2952,6 +3023,180 @@ nif_whereis_proxy(Ref) -> {Ref, quit} -> ok end. +nif_ioq(Config) -> + ensure_lib_loaded(Config), + + Script = + [{create, a}, + + %% Test enq of erlang term binary + {enqb, a}, + {enqb, a, 3}, + + %% Test enq of non-erlang term binary + {enqbraw,a}, + {enqbraw,a, 5}, + {peek, a}, + {deq, a, 42}, + + %% Test enqv + {enqv, a, 2, 100}, + {deq, a, all}, + + %% This skips all elements but one in the iolist + {enqv, a, 5, iolist_size(nif_ioq_payload(5)) - 1}, + {peek, a}, + + %% Test to enqueue a bunch of refc binaries + {enqv, a, [nif_ioq_payload(refcbin) || _ <- lists:seq(1,20)], 0}, + + %% Enq stuff to destroy with data in queue + {enqv, a, 2, 100}, + {destroy,a}, + + %% Test destroy of new queue + {create, a}, + {destroy,a} + ], + + nif_ioq_run(Script), + + %% Test that only enif_inspect_as_vec works + Payload = nif_ioq_payload(5), + PayloadBin = iolist_to_binary(Payload), + + [begin + PayloadBin = iolist_to_binary(ioq_nif(inspect,Payload,Stack,Env)), + <<>> = iolist_to_binary(ioq_nif(inspect,[],Stack,Env)) + end || Stack <- [no_stack, use_stack], Env <- [use_env, no_env]], + + %% Test error cases + + Q = ioq_nif(create), + + {'EXIT', {badarg, _}} = (catch ioq_nif(deq, Q, 1)), + {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, 1, 1234)), + + {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [atom_in_list], 0)), + {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [make_ref()], 0)), + {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [256], 0)), + {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [-1], 0)), + {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [#{}], 0)), + {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [1 bsl 64], 0)), + {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [{tuple}], 0)), + + {'EXIT', {badarg, _}} = (catch ioq_nif(inspect, [atom_in_list], use_stack)), + {'EXIT', {badarg, _}} = (catch ioq_nif(inspect, [make_ref()], no_stack)), + {'EXIT', {badarg, _}} = (catch ioq_nif(inspect, [256], use_stack)), + {'EXIT', {badarg, _}} = (catch ioq_nif(inspect, [-1], no_stack)), + {'EXIT', {badarg, _}} = (catch ioq_nif(inspect, [#{}], use_stack)), + {'EXIT', {badarg, _}} = (catch ioq_nif(inspect, [1 bsl 64], no_stack)), + {'EXIT', {badarg, _}} = (catch ioq_nif(inspect, [{tuple}], use_stack)), + {'EXIT', {badarg, _}} = (catch ioq_nif(inspect, <<"binary">>, use_stack)), + + ioq_nif(destroy, Q), + + %% Test that the example in the docs works + ExampleQ = ioq_nif(create), + true = ioq_nif(example, ExampleQ, nif_ioq_payload(5)), + ioq_nif(destroy, ExampleQ), + + ok. + + +nif_ioq_run(Script) -> + nif_ioq_run(Script, #{}). + +nif_ioq_run([{Action, Name}|T], State) + when Action =:= enqb; Action =:= enqbraw -> + nif_ioq_run([{Action, Name, heapbin}|T], State); +nif_ioq_run([{Action, Name, Skip}|T], State) + when Action =:= enqb, is_integer(Skip); + Action =:= enqbraw, is_integer(Skip) -> + nif_ioq_run([{Action, Name, heapbin, Skip}|T], State); +nif_ioq_run([{Action, Name, N}|T], State) + when Action =:= enqv; Action =:= enqb; Action =:= enqbraw -> + nif_ioq_run([{Action, Name, N, 0}|T], State); +nif_ioq_run([{Action, Name, N, Skip}|T], State) + when Action =:= enqv; Action =:= enqb; Action =:= enqbraw -> + + #{ q := IOQ, b := B } = Q = maps:get(Name, State), + true = ioq_nif(size, IOQ) == iolist_size(B), + + %% Sanitize the log output a bit so that it doesn't become too large. + H = {Action, Name, try iolist_size(N) of Sz -> Sz catch _:_ -> N end, Skip}, + ct:log("~p", [H]), + + Data = nif_ioq_payload(N), + ioq_nif(Action, IOQ, Data, Skip), + + <<_:Skip/binary, SkippedData/binary>> = iolist_to_binary(Data), + + true = ioq_nif(size, IOQ) == (iolist_size([B|SkippedData])), + + nif_ioq_run(T, State#{ Name := Q#{ b := [B|SkippedData]}}); +nif_ioq_run([{peek, Name} = H|T], State) -> + #{ q := IOQ, b := B } = maps:get(Name, State), + true = ioq_nif(size, IOQ) == iolist_size(B), + + ct:log("~p", [H]), + + Data = ioq_nif(peek, IOQ, ioq_nif(size, IOQ)), + + true = iolist_to_binary(B) == iolist_to_binary(Data), + nif_ioq_run(T, State); +nif_ioq_run([{deq, Name, all}|T], State) -> + #{ q := IOQ, b := B } = maps:get(Name, State), + Size = ioq_nif(size, IOQ), + true = Size == iolist_size(B), + nif_ioq_run([{deq, Name, Size}|T], State); +nif_ioq_run([{deq, Name, N} = H|T], State) -> + #{ q := IOQ, b := B } = Q = maps:get(Name, State), + true = ioq_nif(size, IOQ) == iolist_size(B), + + ct:log("~p", [H]), + + <<_:N/binary,Remain/binary>> = iolist_to_binary(B), + NewQ = Q#{ b := Remain }, + + Sz = ioq_nif(deq, IOQ, N), + + true = Sz == iolist_size(Remain), + true = ioq_nif(size, IOQ) == iolist_size(Remain), + + nif_ioq_run(T, State#{ Name := NewQ }); +nif_ioq_run([{create, Name} = H|T], State) -> + ct:log("~p", [H]), + nif_ioq_run(T, State#{ Name => #{ q => ioq_nif(create), b => [] } }); +nif_ioq_run([{destroy, Name} = H|T], State) -> + #{ q := IOQ, b := B } = maps:get(Name, State), + true = ioq_nif(size, IOQ) == iolist_size(B), + + ct:log("~p", [H]), + + ioq_nif(destroy, IOQ), + + nif_ioq_run(T, maps:remove(Name, State)); +nif_ioq_run([], State) -> + State. + +nif_ioq_payload(N) when is_integer(N) -> + Tail = if N > 3 -> nif_ioq_payload(N-3); true -> [] end, + Head = element(1, lists:split(N,[nif_ioq_payload(subbin), + nif_ioq_payload(heapbin), + nif_ioq_payload(refcbin) | Tail])), + erlang:iolist_to_iovec(Head); +nif_ioq_payload(subbin) -> + Bin = nif_ioq_payload(refcbin), + Sz = size(Bin) - 1, + <<_:8,SubBin:Sz/binary,_/bits>> = Bin, + SubBin; +nif_ioq_payload(heapbin) -> + <<"a literal heap binary">>; +nif_ioq_payload(refcbin) -> + iolist_to_binary([lists:seq(1,255) || _ <- lists:seq(1,255)]); +nif_ioq_payload(Else) -> + Else. %% The NIFs: lib_version() -> undefined. @@ -3017,16 +3262,22 @@ binary_to_term_nif(_, _, _) -> ?nif_stub. port_command_nif(_, _) -> ?nif_stub. format_term_nif(_,_) -> ?nif_stub. select_nif(_,_,_,_,_) -> ?nif_stub. +dupe_resource_nif(_) -> ?nif_stub. pipe_nif() -> ?nif_stub. write_nif(_,_) -> ?nif_stub. read_nif(_,_) -> ?nif_stub. is_closed_nif(_) -> ?nif_stub. +clear_select_nif(_) -> ?nif_stub. last_fd_stop_call() -> ?nif_stub. alloc_monitor_resource_nif() -> ?nif_stub. monitor_process_nif(_,_,_,_) -> ?nif_stub. demonitor_process_nif(_,_) -> ?nif_stub. compare_monitors_nif(_,_) -> ?nif_stub. monitor_frenzy_nif(_,_,_,_) -> ?nif_stub. +ioq_nif(_) -> ?nif_stub. +ioq_nif(_,_) -> ?nif_stub. +ioq_nif(_,_,_) -> ?nif_stub. +ioq_nif(_,_,_,_) -> ?nif_stub. %% whereis whereis_send(_Type,_Name,_Msg) -> ?nif_stub. diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c index 307d1c390f..79560a38aa 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c +++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c @@ -186,6 +186,12 @@ static ErlNifResourceTypeInit frenzy_rt_init = { static ErlNifResourceType* whereis_resource_type; static void whereis_thread_resource_dtor(ErlNifEnv* env, void* obj); +static ErlNifResourceType* ioq_resource_type; + +static void ioq_resource_dtor(ErlNifEnv* env, void* obj); +struct ioq_resource { + ErlNifIOQueue *q; +}; static int get_pointer(ErlNifEnv* env, ERL_NIF_TERM term, void** pp) { @@ -243,6 +249,10 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) whereis_resource_type = enif_open_resource_type(env, NULL, "nif_SUITE.whereis", whereis_thread_resource_dtor, ERL_NIF_RT_CREATE, NULL); + ioq_resource_type = enif_open_resource_type(env,NULL,"ioq", + ioq_resource_dtor, + ERL_NIF_RT_CREATE, NULL); + atom_false = enif_make_atom(env,"false"); atom_true = enif_make_atom(env,"true"); atom_self = enif_make_atom(env,"self"); @@ -2430,7 +2440,6 @@ static ERL_NIF_TERM format_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg return enif_make_binary(env,&obin); } - static int get_fd(ErlNifEnv* env, ERL_NIF_TERM term, struct fd_resource** rsrc) { if (!enif_get_resource(env, term, fd_resource_type, (void**)rsrc)) { @@ -2439,6 +2448,13 @@ static int get_fd(ErlNifEnv* env, ERL_NIF_TERM term, struct fd_resource** rsrc) return 1; } +/* Returns: badarg + * Or an enif_select result, which is a combination of bits: + * ERL_NIF_SELECT_STOP_CALLED = 1 + * ERL_NIF_SELECT_STOP_SCHEDULED = 2 + * ERL_NIF_SELECT_INVALID_EVENT = 4 + * ERL_NIF_SELECT_FAILED = 8 + */ static ERL_NIF_TERM select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { struct fd_resource* fdr; @@ -2470,6 +2486,9 @@ static ERL_NIF_TERM select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv } #ifndef __WIN32__ +/* + * Create a read-write pipe with two fds (to read and to write) + */ static ERL_NIF_TERM pipe_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { struct fd_resource* read_rsrc; @@ -2505,6 +2524,30 @@ static ERL_NIF_TERM pipe_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[] enif_make_tuple2(env, write_fd, make_pointer(env, write_rsrc))); } +/* + * Create (dupe) of a resource with the same fd, to test stealing + */ +static ERL_NIF_TERM dupe_resource_nif(ErlNifEnv* env, int argc, + const ERL_NIF_TERM argv[]) { + struct fd_resource* orig_rsrc; + + if (!get_fd(env, argv[0], &orig_rsrc)) { + return enif_make_badarg(env); + } else { + struct fd_resource* new_rsrc; + ERL_NIF_TERM new_fd; + + new_rsrc = enif_alloc_resource(fd_resource_type, + sizeof(struct fd_resource)); + new_rsrc->fd = orig_rsrc->fd; + new_rsrc->was_selected = 0; + new_fd = enif_make_resource(env, new_rsrc); + enif_release_resource(new_rsrc); + + return enif_make_tuple2(env, new_fd, make_pointer(env, new_rsrc)); + } +} + static ERL_NIF_TERM write_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { struct fd_resource* fdr; @@ -2580,6 +2623,20 @@ static ERL_NIF_TERM is_closed_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM a return fdr->fd < 0 ? atom_true : atom_false; } + +static ERL_NIF_TERM clear_select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + struct fd_resource* fdr = NULL; + + if (!get_fd(env, argv[0], &fdr)) + return enif_make_badarg(env); + + fdr->fd = -1; + fdr->was_selected = 0; + + return atom_ok; +} + #endif /* !__WIN32__ */ @@ -3158,7 +3215,231 @@ static void frenzy_resource_down(ErlNifEnv* env, void* obj, ErlNifPid* pid, abort(); } +/*********** testing ioq ************/ + +static void ioq_resource_dtor(ErlNifEnv* env, void* obj) { + +} + +#ifndef __WIN32__ +static int writeiovec(ErlNifEnv *env, ERL_NIF_TERM term, ERL_NIF_TERM *tail, ErlNifIOQueue *q, int fd) { + ErlNifIOVec vec, *iovec = &vec; + SysIOVec *sysiovec; + int saved_errno; + int iovcnt, n; + + if (!enif_inspect_iovec(env, 64, term, tail, &iovec)) + return -2; + + if (enif_ioq_size(q) > 0) { + /* If the I/O queue contains data we enqueue the iovec and then + peek the data to write out of the queue. */ + if (!enif_ioq_enqv(q, iovec, 0)) + return -3; + + sysiovec = enif_ioq_peek(q, &iovcnt); + } else { + /* If the I/O queue is empty we skip the trip through it. */ + iovcnt = iovec->iovcnt; + sysiovec = iovec->iov; + } + + /* Attempt to write the data */ + n = writev(fd, sysiovec, iovcnt); + saved_errno = errno; + + if (enif_ioq_size(q) == 0) { + /* If the I/O queue was initially empty we enqueue any + remaining data into the queue for writing later. */ + if (n >= 0 && !enif_ioq_enqv(q, iovec, n)) + return -3; + } else { + /* Dequeue any data that was written from the queue. */ + if (n > 0 && !enif_ioq_deq(q, n, NULL)) + return -4; + } + + /* return n, which is either number of bytes written or -1 if + some error happened */ + errno = saved_errno; + return n; +} +#endif + +static ERL_NIF_TERM ioq(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + struct ioq_resource *ioq; + ERL_NIF_TERM ret; + if (enif_is_identical(argv[0], enif_make_atom(env, "create"))) { + ErlNifIOQueue *q = enif_ioq_create(ERL_NIF_IOQ_NORMAL); + ioq = (struct ioq_resource *)enif_alloc_resource(ioq_resource_type, + sizeof(*ioq)); + ioq->q = q; + ret = enif_make_resource(env, ioq); + enif_release_resource(ioq); + return ret; + } else if (enif_is_identical(argv[0], enif_make_atom(env, "inspect"))) { + ErlNifIOVec vec, *iovec = NULL; + int i, iovcnt; + ERL_NIF_TERM *elems, tail, list; + ErlNifEnv *myenv = NULL; + + if (enif_is_identical(argv[2], enif_make_atom(env, "use_stack"))) + iovec = &vec; + if (enif_is_identical(argv[3], enif_make_atom(env, "use_env"))) + myenv = env; + if (!enif_inspect_iovec(myenv, ~(size_t)0, argv[1], &tail, &iovec)) + return enif_make_badarg(env); + + iovcnt = iovec->iovcnt; + elems = enif_alloc(sizeof(ERL_NIF_TERM) * iovcnt); + for (i = 0; i < iovcnt; i++) { + ErlNifBinary bin; + if (!enif_alloc_binary(iovec->iov[i].iov_len, &bin)) { + enif_free_iovec(iovec); + enif_free(elems); + return enif_make_badarg(env); + } + memcpy(bin.data, iovec->iov[i].iov_base, iovec->iov[i].iov_len); + elems[i] = enif_make_binary(env, &bin); + } + + if (!myenv) + enif_free_iovec(iovec); + + list = enif_make_list_from_array(env, elems, iovcnt); + enif_free(elems); + return list; + } else { + unsigned skip; + if (!enif_get_resource(env, argv[1], ioq_resource_type, (void**)&ioq) + || !ioq->q) + return enif_make_badarg(env); + + if (enif_is_identical(argv[0], enif_make_atom(env, "example"))) { +#ifndef __WIN32__ + int fd[2], res = 0, cnt = 0, queue_cnt; + ERL_NIF_TERM tail; + char buff[255]; + pipe(fd); + fcntl(fd[0], F_SETFL, fcntl(fd[0], F_GETFL) | O_NONBLOCK); + fcntl(fd[1], F_SETFL, fcntl(fd[1], F_GETFL) | O_NONBLOCK); + + /* Write until the pipe buffer is full, which should result in data + * being queued up. */ + for (res = 0; res >= 0; ) { + cnt += res; + res = writeiovec(env, argv[2], &tail, ioq->q, fd[1]); + } + + /* Flush the queue while reading from the other end of the pipe. */ + tail = enif_make_list(env, 0); + while (enif_ioq_size(ioq->q) > 0) { + res = writeiovec(env, tail, &tail, ioq->q, fd[1]); + + if (res < 0 && errno != EAGAIN) { + break; + } else if (res > 0) { + cnt += res; + } + + for (res = 0; res >= 0; ) { + cnt -= res; + res = read(fd[0], buff, sizeof(buff)); + } + } + + close(fd[0]); + close(fd[1]); + + /* Check that we read as much as we wrote */ + if (cnt == 0 && enif_ioq_size(ioq->q) == 0) + return enif_make_atom(env, "true"); + + return enif_make_int(env, cnt); +#else + return enif_make_atom(env, "true"); +#endif + } + if (enif_is_identical(argv[0], enif_make_atom(env, "destroy"))) { + enif_ioq_destroy(ioq->q); + ioq->q = NULL; + return enif_make_atom(env, "false"); + } else if (enif_is_identical(argv[0], enif_make_atom(env, "enqv"))) { + ErlNifIOVec vec, *iovec = &vec; + ERL_NIF_TERM tail; + + if (!enif_get_uint(env, argv[3], &skip)) + return enif_make_badarg(env); + if (!enif_inspect_iovec(env, ~0ul, argv[2], &tail, &iovec)) + return enif_make_badarg(env); + if (!enif_ioq_enqv(ioq->q, iovec, skip)) + return enif_make_badarg(env); + + return enif_make_atom(env, "true"); + } else if (enif_is_identical(argv[0], enif_make_atom(env, "enqb"))) { + ErlNifBinary bin; + if (!enif_get_uint(env, argv[3], &skip) || + !enif_inspect_binary(env, argv[2], &bin)) + return enif_make_badarg(env); + + if (!enif_ioq_enq_binary(ioq->q, &bin, skip)) + return enif_make_badarg(env); + + return enif_make_atom(env, "true"); + } else if (enif_is_identical(argv[0], enif_make_atom(env, "enqbraw"))) { + ErlNifBinary bin; + ErlNifBinary localbin; + int i; + if (!enif_get_uint(env, argv[3], &skip) || + !enif_inspect_binary(env, argv[2], &bin) || + !enif_alloc_binary(bin.size, &localbin)) + return enif_make_badarg(env); + + memcpy(localbin.data, bin.data, bin.size); + i = enif_ioq_enq_binary(ioq->q, &localbin, skip); + if (!i) + return enif_make_badarg(env); + else + return enif_make_atom(env, "true"); + } else if (enif_is_identical(argv[0], enif_make_atom(env, "peek"))) { + int iovlen, num, i, off = 0; + SysIOVec *iov = enif_ioq_peek(ioq->q, &iovlen); + ErlNifBinary bin; + + if (!enif_get_int(env, argv[2], &num) || !enif_alloc_binary(num, &bin)) + return enif_make_badarg(env); + + for (i = 0; i < iovlen && num > 0; i++) { + int to_copy = num < iov[i].iov_len ? num : iov[i].iov_len; + memcpy(bin.data + off, iov[i].iov_base, to_copy); + num -= to_copy; + off += to_copy; + } + + return enif_make_binary(env, &bin); + } else if (enif_is_identical(argv[0], enif_make_atom(env, "deq"))) { + int num; + size_t sz; + ErlNifUInt64 sz64; + if (!enif_get_int(env, argv[2], &num)) + return enif_make_badarg(env); + + if (!enif_ioq_deq(ioq->q, num, &sz)) + return enif_make_badarg(env); + + sz64 = sz; + + return enif_make_uint64(env, sz64); + } else if (enif_is_identical(argv[0], enif_make_atom(env, "size"))) { + ErlNifUInt64 size = enif_ioq_size(ioq->q); + return enif_make_uint64(env, size); + } + } + + return enif_make_badarg(env); +} static ErlNifFunc nif_funcs[] = { @@ -3243,8 +3524,10 @@ static ErlNifFunc nif_funcs[] = #ifndef __WIN32__ {"pipe_nif", 0, pipe_nif}, {"write_nif", 2, write_nif}, + {"dupe_resource_nif", 1, dupe_resource_nif}, {"read_nif", 2, read_nif}, {"is_closed_nif", 1, is_closed_nif}, + {"clear_select_nif", 1, clear_select_nif}, #endif {"last_fd_stop_call", 0, last_fd_stop_call}, {"alloc_monitor_resource_nif", 0, alloc_monitor_resource_nif}, @@ -3255,7 +3538,11 @@ static ErlNifFunc nif_funcs[] = {"whereis_send", 3, whereis_send}, {"whereis_term", 2, whereis_term}, {"whereis_thd_lookup", 2, whereis_thd_lookup}, - {"whereis_thd_result", 1, whereis_thd_result} + {"whereis_thd_result", 1, whereis_thd_result}, + {"ioq_nif", 1, ioq}, + {"ioq_nif", 2, ioq}, + {"ioq_nif", 3, ioq}, + {"ioq_nif", 4, ioq} }; ERL_NIF_INIT(nif_SUITE,nif_funcs,load,NULL,upgrade,unload) diff --git a/erts/emulator/test/node_container_SUITE.erl b/erts/emulator/test/node_container_SUITE.erl index 8e9e3cb05a..be90f929df 100644 --- a/erts/emulator/test/node_container_SUITE.erl +++ b/erts/emulator/test/node_container_SUITE.erl @@ -405,6 +405,7 @@ node_table_gc(Config) when is_list(Config) -> PreKnown = nodes(known), io:format("PreKnown = ~p~n", [PreKnown]), make_node_garbage(0, 200000, 1000, []), + receive after 1000 -> ok end, %% Wait for thread progress... PostKnown = nodes(known), PostAreas = erlang:system_info(allocated_areas), io:format("PostKnown = ~p~n", [PostKnown]), diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl index ab0b1a82bd..730a17d7e8 100644 --- a/erts/emulator/test/port_SUITE.erl +++ b/erts/emulator/test/port_SUITE.erl @@ -159,7 +159,7 @@ suite() -> all() -> [otp_6224, {group, stream}, basic_ping, slow_writes, bad_packet, bad_port_messages, {group, options}, - {group, multiple_packets}, parallell, dying_port, + {group, multiple_packets}, parallell, dying_port, dropped_commands, port_program_with_path, open_input_file_port, open_output_file_port, name1, env, huge_env, bad_env, cd, cd_relative, pipe_limit_env, bad_args, @@ -569,12 +569,14 @@ dropped_commands(Config, Outputv, Cmd) -> [dropped_commands_test(Cmd) || _ <- lists:seq(1, 100)], timer:sleep(100), erl_ddll:unload_driver("echo_drv"), + os:unsetenv("ECHO_DRV_USE_OUTPUTV"), ok. dropped_commands_test(Cmd) -> - Port = erlang:open_port({spawn_driver, "echo_drv"}, [{parallelism, true}]), spawn_monitor( fun() -> + Port = erlang:open_port({spawn_driver, "echo_drv"}, + [{parallelism, true}]), [spawn_link(fun() -> spin(Port, Cmd) end) || _ <- lists:seq(1,8)], timer:sleep(5), port_close(Port), diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl index 6ded7ff1c9..a9f20f9928 100644 --- a/erts/emulator/test/process_SUITE.erl +++ b/erts/emulator/test/process_SUITE.erl @@ -152,7 +152,11 @@ spawn_with_binaries(Config) when is_list(Config) -> TwoMeg = lists:duplicate(1024, L), Fun = fun() -> spawn(?MODULE, binary_owner, [list_to_binary(TwoMeg)]), receive after 1 -> ok end end, - test_server:do_times(150, Fun), + Iter = case test_server:is_valgrind() of + true -> 10; + false -> 150 + end, + test_server:do_times(Iter, Fun), ok. binary_owner(Bin) when is_binary(Bin) -> diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl index 12e26671c2..7afb82a1b8 100644 --- a/erts/emulator/test/scheduler_SUITE.erl +++ b/erts/emulator/test/scheduler_SUITE.erl @@ -57,6 +57,7 @@ scheduler_suspend_basic/1, scheduler_suspend/1, dirty_scheduler_threads/1, + poll_threads/1, reader_groups/1]). suite() -> @@ -72,6 +73,7 @@ all() -> {group, scheduler_bind}, scheduler_threads, scheduler_suspend_basic, scheduler_suspend, dirty_scheduler_threads, + poll_threads, reader_groups]. groups() -> @@ -1446,6 +1448,79 @@ sst5_loop(N) -> erlang:system_flag(multi_scheduling, unblock_normal), sst5_loop(N-1). +poll_threads(Config) when is_list(Config) -> + {Conc, PollType, KP} = get_ioconfig(Config), + {Sched, SchedOnln, _} = get_sstate(Config, ""), + + [1, 1] = get_ionum(Config,"+IOt 2 +IOp 2"), + [1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 5 +IOp 5"), + + [1, 1] = get_ionum(Config, "+S 2 +IOPt 100 +IOPp 100"), + + if + Conc -> + [5] = get_ionum(Config,"+IOt 5 +IOp 1"), + [3, 2] = get_ionum(Config,"+IOt 5 +IOp 2"), + [2, 2, 2, 2, 2] = get_ionum(Config,"+IOt 10 +IOPp 50"), + + [2] = get_ionum(Config, "+S 2 +IOPt 100"), + [4] = get_ionum(Config, "+S 4 +IOPt 100"), + [4] = get_ionum(Config, "+S 4:2 +IOPt 100"), + [4, 4] = get_ionum(Config, "+S 8 +IOPt 100 +IOPp 25"), + + fail = get_ionum(Config, "+IOt 1 +IOp 2"), + + ok; + not Conc -> + [1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 5 +IOp 1"), + [1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 5 +IOp 2"), + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 10 +IOPp 50"), + + [1, 1] = get_ionum(Config, "+S 2 +IOPt 100"), + [1, 1, 1, 1] = get_ionum(Config, "+S 4 +IOPt 100"), + [1, 1, 1, 1] = get_ionum(Config, "+S 4:2 +IOPt 100"), + [1, 1, 1, 1, 1, 1, 1, 1] = get_ionum(Config, "+S 8 +IOPt 100 +IOPp 25"), + + [1] = get_ionum(Config, "+IOt 1 +IOp 2"), + + ok + end, + + fail = get_ionum(Config, "+IOt 1 +IOPp 101"), + fail = get_ionum(Config, "+IOt 0"), + fail = get_ionum(Config, "+IOPt 101"), + + ok. + +get_ioconfig(Config) -> + [PS | _] = get_iostate(Config, ""), + {proplists:get_value(concurrent_updates, PS), + proplists:get_value(primary, PS), + proplists:get_value(kernel_poll, PS)}. + +get_ionum(Config, Cmd) -> + case get_iostate(Config, Cmd) of + fail -> fail; + PSs -> + lists:reverse( + lists:sort( + [proplists:get_value(poll_threads, PS) || PS <- PSs])) + end. + +get_iostate(Config, Cmd)-> + case start_node(Config, Cmd) of + {ok, Node} -> + [IOStates] = mcall(Node,[fun () -> + erlang:system_info(check_io) + end]), + IO = [IOState || IOState <- IOStates, + proplists:get_value(fallback, IOState) == false], + stop_node(Node), + IO; + {error,timeout} -> + fail + end. + reader_groups(Config) when is_list(Config) -> %% White box testing. These results are correct, but other results %% could be too... @@ -1770,18 +1845,24 @@ mcall(Node, Funs) -> Parent = self(), Refs = lists:map(fun (Fun) -> Ref = make_ref(), - spawn_link(Node, - fun () -> - Res = Fun(), - unlink(Parent), - Parent ! {Ref, Res} - end), - Ref + Pid = spawn(Node, + fun () -> + Res = Fun(), + unlink(Parent), + Parent ! {Ref, Res} + end), + MRef = erlang:monitor(process, Pid), + {Ref, MRef} end, Funs), - lists:map(fun (Ref) -> + lists:map(fun ({Ref, MRef}) -> receive {Ref, Res} -> - Res + receive + {'DOWN',MRef,_,_,_} -> + Res + end; + {'DOWN',MRef,_,_,Reason} -> + Reason end end, Refs). diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl index 6c01bfd45c..029a6de897 100644 --- a/erts/emulator/test/statistics_SUITE.erl +++ b/erts/emulator/test/statistics_SUITE.erl @@ -23,8 +23,10 @@ %% Tests the statistics/1 bif. -export([all/0, suite/0, groups/0, + wall_clock_sanity/1, wall_clock_zero_diff/1, wall_clock_update/1, - runtime_zero_diff/1, + runtime_sanity/1, + runtime_zero_diff/1, runtime_update/1, runtime_diff/1, run_queue_one/1, scheduler_wall_time/1, @@ -54,11 +56,23 @@ all() -> groups() -> [{wall_clock, [], - [wall_clock_zero_diff, wall_clock_update]}, + [wall_clock_sanity, wall_clock_zero_diff, wall_clock_update]}, {runtime, [], - [runtime_zero_diff, runtime_update, runtime_diff]}, + [runtime_sanity, runtime_zero_diff, runtime_update, runtime_diff]}, {run_queue, [], [run_queue_one]}]. +wall_clock_sanity(Config) when is_list(Config) -> + erlang:yield(), + {WallClock, _} = statistics(wall_clock), + MT = erlang:monotonic_time(), + Time = erlang:convert_time_unit(MT - erlang:system_info(start_time), + native, millisecond), + io:format("Time=~p WallClock=~p~n", + [Time, WallClock]), + true = WallClock =< Time, + true = Time - 100 =< WallClock, + ok. + %%% Testing statistics(wall_clock). %% Tests that the 'Wall clock since last call' element of the result @@ -102,6 +116,20 @@ wall_clock_update1(0) -> %%% Test statistics(runtime). +runtime_sanity(Config) when is_list(Config) -> + case erlang:system_info(logical_processors_available) of + unknown -> + {skipped, "Don't know available logical processors"}; + LP when is_integer(LP) -> + erlang:yield(), + {RunTime, _} = statistics(runtime), + MT = erlang:monotonic_time(), + Time = erlang:convert_time_unit(MT - erlang:system_info(start_time), + native, millisecond), + io:format("Time=~p RunTime=~p~n", + [Time, RunTime]), + true = RunTime =< Time*LP + end. %% Tests that the difference between the times returned from two consectuitive %% calls to statistics(runtime) is zero. @@ -646,6 +674,16 @@ msacc_test(TmpFile) -> ets:insert(Tid, {1, hello}), ets:delete(Tid), + %% Check some IO + {ok, L} = gen_tcp:listen(0, [{active, true},{reuseaddr,true}]), + {ok, Port} = inet:port(L), + Pid = spawn(fun() -> + {ok, S} = gen_tcp:accept(L), + (fun F() -> receive M -> F() end end)() + end), + {ok, C} = gen_tcp:connect("localhost", Port, []), + [begin gen_tcp:send(C,"hello"),timer:sleep(1) end || _ <- lists:seq(1,100)], + %% Collect some garbage [erlang:garbage_collect() || _ <- lists:seq(1,100)], diff --git a/erts/emulator/test/trace_SUITE.erl b/erts/emulator/test/trace_SUITE.erl index 72acd33033..a81aa64057 100644 --- a/erts/emulator/test/trace_SUITE.erl +++ b/erts/emulator/test/trace_SUITE.erl @@ -38,7 +38,7 @@ system_monitor_long_gc_1/1, system_monitor_long_gc_2/1, system_monitor_large_heap_1/1, system_monitor_large_heap_2/1, system_monitor_long_schedule/1, - bad_flag/1, trace_delivered/1]). + bad_flag/1, trace_delivered/1, trap_exit_self_receive/1]). -include_lib("common_test/include/ct.hrl"). @@ -61,7 +61,8 @@ all() -> more_system_monitor_args, system_monitor_long_gc_1, system_monitor_long_gc_2, system_monitor_large_heap_1, system_monitor_long_schedule, - system_monitor_large_heap_2, bad_flag, trace_delivered]. + system_monitor_large_heap_2, bad_flag, trace_delivered, + trap_exit_self_receive]. init_per_testcase(_Case, Config) -> [{receiver,spawn(fun receiver/0)}|Config]. @@ -1709,6 +1710,31 @@ trace_delivered(Config) when is_list(Config) -> ok end. +%% This testcase checks that receive trace works on exit signal messages +%% when the sender of the exit signal is the process itself. +trap_exit_self_receive(Config) -> + Parent = self(), + Proc = spawn_link(fun() -> process(Parent) end), + + 1 = erlang:trace(Proc, true, ['receive']), + Proc ! {trap_exit_please, true}, + {trace, Proc, 'receive', {trap_exit_please, true}} = receive_first_trace(), + + %% Make the process call exit(self(), signal) + Reason1 = make_ref(), + Proc ! {exit_signal_please, Reason1}, + {trace, Proc, 'receive', {exit_signal_please, Reason1}} = receive_first_trace(), + {trace, Proc, 'receive', {'EXIT', Proc, Reason1}} = receive_first_trace(), + receive {Proc, {'EXIT', Proc, Reason1}} -> ok end, + receive_nothing(), + + unlink(Proc), + Reason2 = make_ref(), + Proc ! {exit_please, Reason2}, + {trace, Proc, 'receive', {exit_please, Reason2}} = receive_first_trace(), + receive_nothing(), + ok. + drop_trace_until_down(Proc, Mon) -> drop_trace_until_down(Proc, Mon, false, 0, 0). @@ -1791,6 +1817,9 @@ process(Dest) -> process(Dest); {exit_please, Reason} -> exit(Reason); + {exit_signal_please, Reason} -> + exit(self(), Reason), + process(Dest); {trap_exit_please, State} -> process_flag(trap_exit, State), process(Dest); diff --git a/erts/emulator/test/tuple_SUITE.erl b/erts/emulator/test/tuple_SUITE.erl index 79b681b4d1..baf41180e0 100644 --- a/erts/emulator/test/tuple_SUITE.erl +++ b/erts/emulator/test/tuple_SUITE.erl @@ -134,6 +134,13 @@ t_element(Config) when is_list(Config) -> {'EXIT', {badarg, _}} = (catch element(1, id(42))), {'EXIT', {badarg, _}} = (catch element(id(1.5), id({a,b}))), + %% Make sure that the loader does not reject the module when + %% huge literal index values are used. + {'EXIT', {badarg, _}} = (catch element((1 bsl 24)-1, id({a,b,c}))), + {'EXIT', {badarg, _}} = (catch element(1 bsl 24, id({a,b,c}))), + {'EXIT', {badarg, _}} = (catch element(1 bsl 32, id({a,b,c}))), + {'EXIT', {badarg, _}} = (catch element(1 bsl 64, id({a,b,c}))), + ok. get_elements([Element|Rest], Tuple, Pos) -> diff --git a/erts/emulator/test/z_SUITE.erl b/erts/emulator/test/z_SUITE.erl index feea7432a9..ac3df8bfbf 100644 --- a/erts/emulator/test/z_SUITE.erl +++ b/erts/emulator/test/z_SUITE.erl @@ -330,7 +330,7 @@ display_check_io(ChkIo) -> ok. get_check_io_info() -> - ChkIo = erlang:system_info(check_io), + ChkIo = driver_SUITE:get_check_io_total(erlang:system_info(check_io)), PendUpdNo = case lists:keysearch(pending_updates, 1, ChkIo) of {value, {pending_updates, PendNo}} -> PendNo; diff --git a/erts/emulator/utils/beam_emu_vars b/erts/emulator/utils/beam_emu_vars new file mode 100755 index 0000000000..c798a4dada --- /dev/null +++ b/erts/emulator/utils/beam_emu_vars @@ -0,0 +1,122 @@ +#!/usr/bin/perl -w +use strict; + +# Analyse beam_emu.s and try to find out the registers +# used for the important variables in process_main(). +# +# Works for .s files from clang or gcc. For gcc, the -fverbose-asm +# option must be used. +# +# Example: +# +# $ beam-emu-vars -vars 'c_p E HTOP FCALLS I reg freg' beam_emu.s +# E: %r13 +# FCALLS: %rcx:98 %rax:88 16(%rsp):50 %rdi:6 +# HTOP: %r10:382 64(%rsp):88 72(%rsp):9 24(%rsp):7 %rcx:6 %r15:6 80(%rsp):3 88(%rsp):2 +# I: %rbx +# c_p: %rbp +# freg: 48(%rsp):11 %rcx:8 %rdi:5 %rax:4 +# reg: %r12 +# +# That means that E, I, c_p, reg seems to be assigned to permanent registers. +# HTOP seems to be assigned %r10, but it is saved to a scratch location +# before any function calls. FCALLS and freg seems to be saved in a location on +# the stack and loaded into a register when used. +# +# The exit status will be 0 if all variables are assigned to registers (most of +# the time), and 1 if one or more variables are assigned to a stack location. + +my $vars = 'c_p E FCALLS freg HTOP I reg'; + +while (@ARGV and $ARGV[0] =~ /^-(.*)/) { + $_ = $1; + shift; + ($vars = shift), next if /^vars/; + die "$0: Bad option: -$_\n"; +} + +my @vars = split(" ", $vars); +my %vars; +@vars{@vars} = @vars; + +my $inside; +my %count; + +if (@ARGV != 1) { + usage(); +} + +while (<>) { + if (!$inside && /[.]globl\s*_?process_main/) { + $inside = 1; + } elsif ($inside && /[.]globl/) { + last; + } + if ($inside) { + if (/##DEBUG_VALUE:\s*process_main:([A-Za-z]*)\s*<-\s*(.*)/) { + # clang + my($var,$reg) = ($1,$2); + next if $reg =~ /^[-\d]+$/; # Ignore if number. + $count{$var}->{$reg}++ if $vars{$var}; + next; + } + + # Parse gcc verbose arguments. Comments are marked with + # one '#' (clang marks its comments with two '#'). + my($src,$dst,$comment) = /movq\s+([^#]+), ([^#]+)#(?!#)\s*(.*)/; + next unless $comment; + $dst =~ s/\s*$//; + my($vsrc,$vdst) = split /,/, $comment, 2; + $vdst =~ s/^\s//; + update_count(\%count, $vsrc, $src); + update_count(\%count, $vdst, $dst); + if ($vars{$vdst} and $vsrc eq '%sfp') { + $count{$vdst}->{$src}++; + } + } +} + +my @first; + +OUTER: +for my $var (sort keys %count) { + my $total = 0; + + foreach my $reg (keys %{$count{$var}}) { + $total += $count{$var}->{$reg}++; + } + + foreach my $reg (keys %{$count{$var}}) { + if ($count{$var}->{$reg} > 0.9*$total) { + print "$var: $reg\n"; + push @first, $var; + next OUTER; + } + } + + my @r; + foreach my $reg (keys %{$count{$var}}) { + push @r, $reg; + } + @r = sort { $count{$var}->{$b} <=> $count{$var}->{$a} } @r; + @r = map { "$_:$count{$var}->{$_}" } @r; + push @first, $r[0]; + print "$var: ", join(' ', @r), "\n"; +} + +foreach (@first) { + exit 1 if /%rsp/; +} +exit 0; + +sub update_count { + my($count_ref,$var,$reg) = @_; + return unless $vars{$var}; + ${${$count_ref}{$var}}{$reg}++; +} + +sub usage { + die qq[usage: beam_emu_vars [ -vars "var1 var2..." ] <filename>.s\n\n] . + "The exit status is 0 if all variables are assigned to registers,\n" . + "and 1 if one or more variables are allocated to a stack location.\n"; +} diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops index da69b13e87..d7791d23fa 100755 --- a/erts/emulator/utils/beam_makeops +++ b/erts/emulator/utils/beam_makeops @@ -20,44 +20,38 @@ # use strict; use vars qw($BEAM_FORMAT_NUMBER); +use constant COLD => 0; +use constant WARM => 1; +use constant HOT => 2; + +# Instructions for packing +use constant PACK_JUMP => 1; +use constant PACK_IN_INSTR_WORD => 2; +use constant PACK_OPT_IN_INSTR_WORD => 4; + +# Packing commands +use constant PACK_CMD_TIGHTEST => '1'; +use constant PACK_CMD_TIGHT => '2'; +use constant PACK_CMD_LOOSE => '3'; +use constant PACK_CMD_WIDE => '4'; $BEAM_FORMAT_NUMBER = undef; my $target = \&emulator_output; my $outdir = "."; # Directory for output files. my $verbose = 0; -my $hot = 1; +my $hotness = 1; my $num_file_opcodes = 0; my $wordsize = 32; -my %defs; # Defines (from command line). +my $code_pointers_are_short = 0; # Whether code pointers (to C code) are short. +my $code_model = 'unknown'; +my %defs; # Defines (from command line). # This is shift counts and mask for the packer. my $WHOLE_WORD = ''; -my @pack_instr; -my @pack_shift; -my @pack_mask; - -$pack_instr[2] = ['6', 'i']; -$pack_instr[3] = ['0', '0', 'i']; -$pack_instr[4] = ['6', '6', '6', 'i']; # Only for 64 bit wordsize - -$pack_shift[2] = ['0', 'BEAM_LOOSE_SHIFT']; -$pack_shift[3] = ['0', 'BEAM_TIGHT_SHIFT', '(2*BEAM_TIGHT_SHIFT)']; -$pack_shift[4] = ['0', 'BEAM_LOOSE_SHIFT', # Only for 64 bit wordsize - '(2*BEAM_LOOSE_SHIFT)', - '(3*BEAM_LOOSE_SHIFT)']; -$pack_mask[2] = ['BEAM_LOOSE_MASK', $WHOLE_WORD]; -$pack_mask[3] = ['BEAM_TIGHT_MASK', 'BEAM_TIGHT_MASK', 'BEAM_TIGHT_MASK']; -$pack_mask[4] = ['BEAM_LOOSE_MASK', # Only for 64 bit wordsize - 'BEAM_LOOSE_MASK', - 'BEAM_LOOSE_MASK', - $WHOLE_WORD]; - -# Mapping from packagable arguments to number of packed arguments per -# word. Initialized after the wordsize is known. - -my @args_per_word; +my @basic_pack_options = (0); +my @extended_pack_options = @basic_pack_options; # There are two types of instructions: generic and specific. # The generic instructions are those generated by the Beam compiler. @@ -83,6 +77,10 @@ my %num_specific; my %gen_to_spec; my %specific_op; +# Information about each specific operator. Key is the print name (e.g. get_list_xxy). +# Value is a hash. +my %spec_op_info; + my %gen_arity; my @gen_arity; @@ -97,10 +95,9 @@ my %c_code_used; # Used or not. # Definitions for instructions combined from micro instructions. my %combined_instrs; -my %combined_code; # Combined micro instructions. -my %hot_code; -my %cold_code; +my @generated_code; # Generated code. +my %sort_order; my @unnumbered_generic; my %unnumbered; @@ -144,13 +141,15 @@ my %arg_size = ('r' => 0, # x(0) - x register zero 'n' => 0, # NIL (implicit) 'c' => 1, # tagged constant (integer, atom, nil) 's' => 1, # tagged source; any of the above + 'S' => 1, # tagged source register (x or y) 'd' => 1, # tagged destination register (r, x, y) 'f' => 1, # failure label 'j' => 1, # either 'f' or 'p' 'e' => 1, # pointer to export entry 'L' => 0, # label - 'I' => 1, # untagged integer - 't' => 1, # untagged integer -- can be packed + 't' => 1, # untagged integer (12 bits) -- can be packed + 'I' => 1, # untagged integer (32 bits) -- can be packed + 'W' => 1, # untagged integer/pointer (one word) 'b' => 1, # pointer to bif 'A' => 1, # arity value 'P' => 1, # byte offset into tuple or stack @@ -192,16 +191,16 @@ sub define_type_bit { define_type_bit('s', $type_bit{'d'} | $type_bit{'i'} | $type_bit{'a'} | $type_bit{'n'} | $type_bit{'q'}); + define_type_bit('S', $type_bit{'d'}); define_type_bit('j', $type_bit{'f'} | $type_bit{'p'}); # Aliases (for matching purposes). - define_type_bit('I', $type_bit{'u'}); define_type_bit('t', $type_bit{'u'}); + define_type_bit('I', $type_bit{'u'}); + define_type_bit('W', $type_bit{'u'}); define_type_bit('A', $type_bit{'u'}); define_type_bit('L', $type_bit{'u'}); define_type_bit('b', $type_bit{'u'}); - define_type_bit('N', $type_bit{'u'}); - define_type_bit('U', $type_bit{'u'}); define_type_bit('e', $type_bit{'u'}); define_type_bit('P', $type_bit{'u'}); define_type_bit('Q', $type_bit{'u'}); @@ -228,6 +227,12 @@ $match_engine_ops{'TOP_fail'} = 1; sanity("tag '$tag': primitive tags must be named with lowercase letters") unless $tag =~ /^[a-z]$/; } + + foreach my $tag (keys %arg_size) { + defined $type_bit{$tag} or + sanity("the tag '$tag' has a size in %arg_size, " . + "but has no defined bit pattern"); + } } # @@ -241,6 +246,7 @@ while (@ARGV && $ARGV[0] =~ /^-(.*)/) { ($target = \&compiler_output), next if /^compiler/; ($outdir = shift), next if /^outdir/; ($wordsize = shift), next if /^wordsize/; + ($code_model = shift), next if /^code-model/; ($verbose = 1), next if /^v/; ($defs{$1} = $2), next if /^D(\w+)=(\w+)/; die "$0: Bad option: -$_\n"; @@ -252,21 +258,31 @@ if ($wordsize == 32) { } elsif ($wordsize == 64) { $defs{'ARCH_32'} = 0; $defs{'ARCH_64'} = 1; + $code_pointers_are_short = $code_model eq 'small'; } # -# Initialize number of arguments per packed word. +# Initialize pack options. # -$args_per_word[2] = 2; -$args_per_word[3] = 3; -$args_per_word[4] = 2; -$args_per_word[5] = 3; -$args_per_word[6] = 3; - if ($wordsize == 64) { - $pack_mask[3] = ['BEAM_TIGHT_MASK', 'BEAM_TIGHT_MASK', $WHOLE_WORD]; - $args_per_word[4] = 4; + @basic_pack_options = (0,PACK_JUMP); + @extended_pack_options = @basic_pack_options; + if ($code_pointers_are_short) { + foreach (@basic_pack_options) { + push @extended_pack_options, $_ | PACK_IN_INSTR_WORD; + } + } +} + +# +# Add placeholders for built-in macros. +# + +$c_code{'IS_PACKED'} = ['$Expr',"built-in macro",('Expr')]; +$c_code{'ARG_POSITION'} = ['$Expr',"built-in macro",('Expr')]; +foreach my $name (keys %c_code) { + $c_code_used{$name} = 1; } # @@ -278,11 +294,6 @@ my $c_code_block; my $c_code_loc; my @c_args; -sub save_c_code { - my($name,$block,$loc,@args) = @_; - -} - while (<>) { my($op_num); if ($in_c_code) { @@ -354,13 +365,16 @@ while (<>) { } # - # Handle %hot/%cold. + # Handle %hot, %warm, and %cold. # if (/^\%hot/) { - $hot = 1; + $hotness = HOT; next; + } elsif (/^\%warm/) { + $hotness = WARM; + next; } elsif (/^\%cold/) { - $hot = 0; + $hotness = COLD; next; } @@ -393,7 +407,7 @@ while (<>) { # micro instructions. # if (/^(\w+)\s*:=\s*([\w.]+)\s*;\s*$/) { - $combined_instrs{$1} = ["$ARGV($.)","beam_instrs.h",$2]; + $combined_instrs{$1} = ["$ARGV($.)",$2]; next; } @@ -440,15 +454,7 @@ while (<>) { # Parse specific instructions (only present in emulator/loader): # Name Arg1 Arg2... # - my($name, @args) = split; - error("too many operands") - if @args > $max_spec_operands; - syntax_check($name, @args); - my $arity = @args; - if (defined $gen_opnum{$name,$arity} and $obsolete[$gen_opnum{$name,$arity}]) { - error("specific instructions may not be specified for obsolete instructions"); - } - save_specific_ops($name, $arity, $hot, @args); + my($name,$arity) = parse_specific_op($_); if (defined $op_num) { error("specific instructions must not be numbered"); } elsif (!defined($gen_arity{$name}) && !defined($unnumbered{$name,$arity})) { @@ -516,6 +522,36 @@ sub emulator_output { my $key; # Loop variable. # + # Generate code and meta information for all instructions. + # + foreach $key (keys %specific_op) { + foreach (@{$specific_op{$key}}) { + my($name, $hotness, @args) = @$_; + my $print_name = print_name($name, @args); + + my($size, $code, $pack_spec) = cg_basic(name => $name, args => \@args); + if (defined $code) { + $code = "OpCase($print_name):\n$code"; + push @generated_code, [$hotness,$code,($print_name)]; + } + + # Note: Some of the information below will be modified + # for combined instructions. + my %info = ('size' => $size, + 'pack_spec' => $pack_spec, + 'adj' => 0, + 'args' => \@args); + $spec_op_info{$print_name} = \%info; + } + } + + # + # Combine micro instruction into instruction blocks and generate + # code for them. + # + combine_micro_instructions(); + + # # Information about opcodes (beam_opcodes.c). # $name = "$outdir/beam_opcodes.c"; @@ -544,14 +580,9 @@ sub emulator_output { print "\n"; # - # Combine micro instruction into instruction blocks. - # - combine_micro_instructions(); - - # # Generate code for specific ops. # - my($spec_opnum) = 0; + my $spec_opnum = 0; print "const OpEntry opc[] = {\n"; foreach $key (sort keys %specific_op) { $gen_to_spec{$key} = $spec_opnum; @@ -566,41 +597,26 @@ sub emulator_output { foreach (@{$specific_op{$key}}) { my($name, $hot, @args) = @{$_}; my($sign) = join('', @args); + $sign =~ s/[?]//g; # The primitive types should sort before other types. - my($sort_key) = $sign; + my $sort_key = $sign; eval "\$sort_key =~ tr/$genop_types/./"; $sort_key .= ":$sign"; - $items{$sort_key} = [$name, $hot, $sign, @args]; + my $print_name = print_name($name, @args); + $items{$sort_key} = $print_name; } # # Now call the generator for the sorted result. # - foreach (sort keys %items) { - my($name, $hot, $sign, @args) = @{$items{$_}}; + foreach my $sort_key (sort keys %items) { + my $print_name = $items{$sort_key}; + my $info = $spec_op_info{$print_name}; + my(@args) = @{$info->{'args'}}; + @args = map { s/[?]$//; $_ } @args; my $arity = @args; - my($instr) = "${name}_$sign"; - $instr =~ s/_$//; - - # - # Call a generator to calculate size and generate macros - # for the emulator. - # - my($size, $code, $pack) = - basic_generator($name, $hot, '', 0, undef, @args); - - # - # Save the generated $code for later. - # - if (defined $code) { - if ($hot) { - push(@{$hot_code{$code}}, $instr); - } else { - push(@{$cold_code{$code}}, $instr); - } - } # # Calculate the bit mask which should be used to match this @@ -622,7 +638,6 @@ sub emulator_output { } printf "/* %3d */ ", $spec_opnum; - my $print_name = $sign ne '' ? "${name}_$sign" : $name; my $init = "{"; my $sep = ""; foreach (@bits) { @@ -630,8 +645,12 @@ sub emulator_output { $sep = ","; } $init .= "}"; - init_item($print_name, $init, $involves_r, $size, $pack, $sign); - $op_to_name[$spec_opnum] = $instr; + my $adj = $info->{'adj'}; + my $size = $info->{'size'}; + my $pack_spec = $info->{'pack_spec'}; + my $sign = join '', @args; + init_item($print_name, $init, $involves_r, $size, $adj, $pack_spec, $sign); + $op_to_name[$spec_opnum] = $print_name; $spec_opnum++; } } @@ -710,12 +729,19 @@ sub emulator_output { print "#if !defined(ARCH_64)\n"; print qq[ #error "64-bit architecture assumed, but ARCH_64 not defined"\n]; print "#endif\n"; - print "#define BEAM_WIDE_MASK 0xFFFFUL\n"; - print "#define BEAM_LOOSE_MASK 0xFFFFUL\n"; - print "#define BEAM_TIGHT_MASK 0xFFFFUL\n"; + if ($code_pointers_are_short) { + print "#if !defined(CODE_MODEL_SMALL)\n"; + print qq[ #error "small code model assumed, but CODE_MODEL_SMALL not defined"\n]; + print "#endif\n"; + } + print "#define BEAM_WIDE_MASK 0xFFFFFFFFull\n"; + print "#define BEAM_LOOSE_MASK 0xFFFFull\n"; + print "#define BEAM_TIGHT_MASK 0xFFFFull\n"; + print "#define BEAM_TIGHTEST_MASK 0x3FFull\n"; print "#define BEAM_WIDE_SHIFT 32\n"; print "#define BEAM_LOOSE_SHIFT 16\n"; print "#define BEAM_TIGHT_SHIFT 16\n"; + print "#define BEAM_TIGHTEST_SHIFT 10\n"; } print "\n"; @@ -814,19 +840,24 @@ sub emulator_output { $name = "$outdir/beam_hot.h"; open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n"; comment('C'); - print_code(\%hot_code); + print_code(HOT); + + $name = "$outdir/beam_warm.h"; + open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n"; + comment('C'); + print_code(WARM); $name = "$outdir/beam_cold.h"; open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n"; comment('C'); - print_code(\%cold_code); + print_code(COLD); +} - foreach my $key (keys %combined_code) { - my $name = "$outdir/$key"; - open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n"; - comment('C'); - print_indented_code(@{$combined_code{$key}}); - } +sub print_name { + my($name,@args) = @_; + my $sign = join '', @args; + $sign =~ s/[?]//g; + $sign ne '' ? "${name}_$sign" : $name; } sub init_item { @@ -854,19 +885,14 @@ sub q { } sub print_code { - my($ref) = @_; - my(%sorted); - my($key, $label); # Loop variables. - - foreach $key (keys %$ref) { - my($sort_key); - my($code) = ''; - foreach $label (@{$ref->{$key}}) { - $code .= "OpCase($label):\n"; - $sort_key = $label; - } - $code .= "$key\n"; - $sorted{$sort_key} = $code; + my($include_hot) = @_; + my %sorted; + + foreach my $ref (@generated_code) { + my($hot,$code,@labels) = @$ref; + next unless $hot == $include_hot; + my($sort_key) = @labels; # Use the first label as sort key. + $sorted{$sort_key} = $code; } foreach (sort keys %sorted) { @@ -950,40 +976,54 @@ sub compiler_output { } # -# Check an operation for validity. +# Parse and store a specific operation. # -sub syntax_check { - my($name, @args) = @_; - my($i); +sub parse_specific_op { + my($name, @args) = split " ", shift; + my $arity = @args; + # Check for various errors. error("Bad opcode name '$name'") unless $name =~ /^[a-z][\w\d_]*$/; - for ($i = 0; $i < @args; $i++) { - foreach my $type (split(//, $args[$i])) { + error("too many operands") + if @args > $max_spec_operands; + for (my $i = 0; $i < $arity; $i++) { + my $arg = $args[$i]; + $arg =~ s/[?]$//; + foreach my $type (split(//, $arg)) { error("Argument " . ($i+1) . ": invalid type '$type'") unless defined $arg_size{$type}; } } -} - -sub save_specific_ops { - my($name,$arity,$hot,@args) = @_; - my(@res) = (""); + if (defined $gen_opnum{$name,$arity} and $obsolete[$gen_opnum{$name,$arity}]) { + error("specific instructions may not be specified for obsolete instructions"); + } + # Expand operands with multiple types to multiple instructions. + # (For example, "get_list xy xy xy" will be expanded to six instructions.) + my @res = ([]); foreach my $arg (@args) { - my @new_res = (); + my @old_res = @res; + @res = (); + my $marker = ($arg =~ s/[?]$//) ? '?' : ''; foreach my $type (split(//, $arg)) { - foreach my $args (@res) { - push @new_res, "$args$type"; + foreach my $args_ref (@old_res) { + my @args = @$args_ref; + push @args, "$type$marker"; + push @res, \@args; } } - @res = @new_res; } + + # Store each specific instruction. my $key = "$name/$arity"; - foreach my $args (@res) { - @args = split //, $args; - push @{$specific_op{$key}}, [$name,$hot,@args]; + foreach my $args_ref (@res) { + @args = @$args_ref; + push @{$specific_op{$key}}, [$name,$hotness,@args]; } + + # Done. + ($name,$arity); } sub parse_c_args { @@ -1043,12 +1083,11 @@ sub comment { # sub combine_micro_instructions { my %groups; - my %group_file; # Sanity check, normalize micro instructions. foreach my $instr (keys %combined_instrs) { my $ref = $combined_instrs{$instr}; - my($def_loc,$outfile,$def) = @$ref; + my($def_loc,$def) = @$ref; my($group,@subs) = split /[.]/, $def; my $arity = 0; @subs = map { "$group.$_" } @subs; @@ -1061,20 +1100,20 @@ sub combine_micro_instructions { $arity += scalar(@c_args); } push @{$groups{$group}}, [$instr,$arity,@subs]; - $group_file{$group} = $outfile; } # Now generate code for each group. foreach my $group (sort keys %groups) { - my $code = combine_instruction_group($group, @{$groups{$group}}); - my $outfile = $group_file{$group}; - push @{$combined_code{$outfile}}, $code; + my($hotness,$code,@labels) = + combine_instruction_group($group, @{$groups{$group}}); + push @generated_code, [$hotness,$code,@labels]; } } sub combine_instruction_group { my($group,@in_instrs) = @_; my $gcode = ''; # Code for the entire group. + my $group_hotness = COLD; # Get code for the head of the group (if any). my $head_name = "$group.head"; @@ -1090,8 +1129,27 @@ sub combine_instruction_group { # Variables. my %offsets; my @instrs; - my %num_references; - my $group_size = 0; + my %num_references; # Number of references from other sub instructions. + my $group_size = 999; + + # + # Calculate the number of references from other sub instructions. + # This number is useful in several ways: + # + # * If this number is 0, it is only used as the entry point for a + # function, implying that it does not need a label and that operands + # can be packed into the instruction word. + # + # * We'll use this number in the sort key, as a tie breaker for sub instructions + # at the same instruction offset. + # + foreach my $ref_instr (@in_instrs) { + my(undef,undef,$first_sub,@other_subs) = @$ref_instr; + $num_references{$first_sub} += 0; # Make sure it is defined. + foreach my $sub (@other_subs) { + $num_references{$sub}++; + } + } # Do basic error checking. Associate operands of instructions # with the correct micro instructions. Calculate offsets for micro @@ -1103,12 +1161,14 @@ sub combine_instruction_group { error("no $specific_key instruction") unless defined $specific_op_ref; foreach my $specific_op (@$specific_op_ref) { - my($name, $hot, @args) = @{$specific_op}; + my($name, $hotness, @args) = @{$specific_op}; + $group_hotness = $hotness unless $group_hotness >= $hotness; my $offset = 0; my @rest = @args; my @new_subs; - my $opcase = $specific; - $opcase .= "_" . join '', @args if @args; + my $print_name = print_name($specific, @args); + my $opcase = $print_name; + my $last = $subs[$#subs]; foreach my $s (@subs) { my $code = $c_code{$s}; my(undef,undef,@c_args) = @{$code}; @@ -1116,16 +1176,18 @@ sub combine_instruction_group { foreach (0..$#c_args) { push @first, shift @rest; } - my($size,undef) = basic_generator($s, 0, '', 0, undef, @first); + my $size = cg_combined_size(name => $s, + first => $num_references{$s} == 0, + args => \@first); $offsets{$s} = $offset - unless defined $offsets{$s} and $offsets{$s} >= $offset; + unless defined $offsets{$s} and $offsets{$s} < $offset; $offset += $size - 1; my $label = micro_label($s); - $num_references{$label} = 0; push @new_subs, [$opcase,$label,$s,$size-1,@first]; $opcase = ''; } - $group_size = $offset if $group_size < $offset; + $spec_op_info{$print_name}->{'size'} = $offset + 1; + $group_size = $offset if $group_size >= $offset; push @instrs, [$specific_key,@new_subs]; } } @@ -1139,9 +1201,8 @@ sub combine_instruction_group { my($opcase,$label,$s,$size,@args) = @{$subs[$i]}; my $next = ''; (undef,$next) = @{$subs[$i+1]} if $i < $#subs; - $num_references{$next}++ if $next; my $instr_info = "$opcase:$label:$next:$s:$size:@args"; - push @all_instrs, [$label,$offsets{$s},$instr_info]; + push @all_instrs, [$label,$s,$offsets{$s},$instr_info]; } } @@ -1149,8 +1210,8 @@ sub combine_instruction_group { my %label_to_offset; my %order_to_offset; foreach my $instr (@all_instrs) { - my($label,$offset,$instr_info) = @$instr; - my $sort_key = sprintf("%02d.%02d", $offset, $num_references{$label}); + my($label,$s,$offset,$instr_info) = @$instr; + my $sort_key = sprintf("%02d.%02d", $offset, $num_references{$s}); push @{$order_to_instrs{$sort_key}}, $instr_info; $label_to_offset{$label} = $offset; $order_to_offset{$sort_key} = $offset; @@ -1160,6 +1221,9 @@ sub combine_instruction_group { # Now generate the code for the entire group. my $offset = 0; + my @opcase_labels; + my %down; + my %up; for(my $i = 0; $i < @slots; $i++) { my $key = $slots[$i]; @@ -1180,35 +1244,70 @@ sub combine_instruction_group { my $seen_key = "$label:$next:" . scalar(@first); next if $opcase eq '' and $seen{$seen_key}; $seen{$seen_key} = 1; + $seen_key .= $opcase; if ($opcase ne '') { $gcode .= "OpCase($opcase):\n"; + push @opcase_labels, $opcase; } - if ($num_references{$label}) { + if ($num_references{$s}) { $gcode .= "$label:\n"; } my $flags = ''; my $transfer_to_next = ''; - my $dec = 0; + my $inc = 0; unless ($i == $#slots) { $flags = "-no_next"; my $next_offset = $label_to_offset{$next}; - $dec = $next_offset - ($offset + $size); - $transfer_to_next = "I -= $dec;\n" if $dec; + $inc = ($offset + $size) - $next_offset; + $transfer_to_next = "I += $inc;\n" if $inc; $transfer_to_next .= "goto $next;\n\n"; } - my(undef,$gen_code) = - basic_generator($s, 0, $flags, $offset, - $group_size-$offset-$dec, @first); + my($gen_code,$down,$up) = + cg_combined_code(name => $s, + first => $num_references{$s} == 0, + extra_comments => $flags, + offset => $offset, + comp_size => $group_size-$offset, + inc => $inc, + args =>\@first); + my $spec_label = "$opcase$label"; + $down{$spec_label} = $down; + $up{$spec_label} = $up; $gcode .= $gen_code . $transfer_to_next; } $offset = $order_to_offset{$slots[$i+1]} if $i < $#slots; } - "{\n$gcode\n}\n\n"; + foreach my $print_name (@opcase_labels) { + my $info = $spec_op_info{$print_name}; + $info->{'adj'} = $info->{'size'} - $group_size - 1; + } + + # + # Assemble pack specifications for all instructions in the group. + # + foreach my $instr (@instrs) { + my(undef,@subs) = @{$instr}; + my $down = ''; + my $up = ''; + for (my $i = 0; $i < @subs; $i++) { + my($opcase,$label) = @{$subs[$i]}; + my $spec_label = "$opcase$label"; + if (defined $down{$spec_label}) { + $down = $down{$spec_label} . $down; + $up = $up . $up{$spec_label}; + } + } + my $print_name = $subs[0]->[0]; + my $info = $spec_op_info{$print_name}; + $info->{'pack_spec'} = build_pack_spec("$down:$up"); + } + + ($group_hotness,"{\n$gcode\n}\n\n",@opcase_labels); } sub micro_label { @@ -1219,12 +1318,56 @@ sub micro_label { # -# Basic implementation of instruction in emulator loop -# (assuming no packing). +# Basic code generation for one instruction. +# + +sub cg_basic { + my %params = (@_, pack_options => \@extended_pack_options); + my($size,$code,$pack_spec) = code_gen(%params); + $pack_spec = build_pack_spec($pack_spec); + ($size,$code,$pack_spec); +} + +# +# Calculate size for a micro instruction. # -sub basic_generator { - my($name,$hot,$extra_comments,$offset,$group_size,@args) = @_; +sub cg_combined_size { + my %params = (@_, pack_options => \@basic_pack_options); + $params{pack_options} = \@extended_pack_options + if $params{first}; + my($size) = code_gen(%params); + $size; +} + +# +# Generate code for a micro instruction. +# + +sub cg_combined_code { + my %params = (@_, pack_options => \@basic_pack_options); + $params{pack_options} = \@extended_pack_options + if $params{first}; + my($size,$code,$pack_spec) = code_gen(%params); + if ($pack_spec eq '') { + ($code,'',''); + } else { + my($down,$up) = split /:/, $pack_spec; + ($code,$down,$up); + } +} + +sub code_gen { + my %params = (extra_comments => '', + offset => 0, + inc => 0, + @_); + my $name = $params{name}; + my $extra_comments = $params{extra_comments}; + my $offset = $params{offset}; + my $inc = $params{inc}; + my @args = @{$params{args}}; + my $size = 0; my $flags = ''; my @f; @@ -1238,8 +1381,9 @@ sub basic_generator { # my $c_code_ref = $c_code{$name}; - if ($hot and defined $c_code_ref) { - ($prefix, $pack_spec, @args) = do_pack(@args); + if (defined $c_code_ref and $name ne 'catch') { + my $pack_options = $params{pack_options}; + ($var_decls, $pack_spec, @args) = do_pack($name, $offset, $pack_options, @args); } # @@ -1249,11 +1393,19 @@ sub basic_generator { my $need_block = 0; my $arg_offset = $offset; + @args = map { s/[?]$//g; $_ } @args; foreach (@args) { my($this_size) = $arg_size{$_}; SWITCH: { - /^pack:(\d):(.*)/ and do { + /^packed:d:(\d):(.*)/ and do { + $var_decls .= "Eterm dst = $2;\n" . + "Eterm* dst_ptr = REG_TARGET_PTR(dst);\n"; + push(@f, "*dst_ptr"); + $this_size = $1; + last SWITCH; + }; + /^packed:[a-zA-z]:(\d):(.*)/ and do { push(@f, $2); $this_size = $1; last SWITCH; @@ -1262,8 +1414,8 @@ sub basic_generator { push(@f, "r(0)"); last SWITCH; }; - /[lxy]/ and do { - push(@f, $_ . "b(Arg($arg_offset))"); + /[lxyS]/ and do { + push(@f, $_ . "b(" . arg_offset($arg_offset) . ")"); last SWITCH; }; /n/ and do { @@ -1280,13 +1432,13 @@ sub basic_generator { last SWITCH; }; /d/ and do { - $var_decls .= "Eterm dst = Arg($arg_offset);\n" . + $var_decls .= "Eterm dst = " . arg_offset($arg_offset) . ";\n" . "Eterm* dst_ptr = REG_TARGET_PTR(dst);\n"; push(@f, "*dst_ptr"); last SWITCH; }; defined $arg_size{$_} and do { - push(@f, "Arg($arg_offset)"); + push @f, arg_offset($arg_offset); last SWITCH; }; @@ -1304,7 +1456,7 @@ sub basic_generator { return ($size+1, undef, ''); } - $group_size = $size unless defined $group_size; + my $group_size = ($params{comp_size} || $size) + $inc; # # Generate main body of the implementation. @@ -1323,6 +1475,7 @@ sub basic_generator { $bindings{$var} = $f[$i]; } $bindings{'NEXT_INSTRUCTION'} = "I+" . ($group_size+$offset+1); + $bindings{'IP_ADJUSTMENT'} = $inc; $c_code = eval { expand_all($c_code, \%bindings) }; unless (defined $c_code) { warn $@; @@ -1345,11 +1498,10 @@ sub basic_generator { "ASSERT(VALID_INSTR(*I));\n" . "Goto(*I);"; } else { - $var_decls .= "BeamInstr* _nextpf = " . - "(BeamInstr *) I[$instr_offset];\n"; + $var_decls .= "BeamInstr next_pf = BeamCodeAddr(I[$instr_offset]);\n"; $dispatch_next = "\nI += $instr_offset;\n" . - "ASSERT(VALID_INSTR(_nextpf));\n" . - "Goto(_nextpf);"; + "ASSERT(VALID_INSTR(next_pf));\n" . + "GotoPF(next_pf);"; } # @@ -1368,6 +1520,11 @@ sub basic_generator { ($size+1, $code, $pack_spec); } +sub arg_offset { + my $offset = shift; + "I[" . ($offset+1) . "]"; +} + sub expand_all { my($code,$bindings_ref) = @_; my %bindings = %{$bindings_ref}; @@ -1382,14 +1539,8 @@ sub expand_all { my $keep = substr($code, 0, $-[0]); my $after = substr($code, $+[0]); - # Keep the special, pre-defined bindings. - my %new_bindings; - foreach my $key (qw(NEXT_INSTRUCTION)) { - $new_bindings{$key} = $bindings{$key}; - } - my $body; - ($body,$code) = expand_macro($macro_name, $after, \%new_bindings); + ($body,$code) = expand_macro($macro_name, $after, \%bindings); $res .= "$keep$body"; } @@ -1436,159 +1587,461 @@ sub expand_macro { $arg =~ s/^\s*//; } - # Now combine bindings from the parameter names and arguments. - my %bindings = %{$bindings_ref}; + # Make sure that the number of arguments are correct. if (@vars != @args) { error("calling $name with ", scalar(@args), " arguments instead of expected ", scalar(@vars), " arguments..."); } + + # Now combine bindings from the parameter names and arguments. + my %bindings = %{$bindings_ref}; + my %new_bindings; + + # Keep the special, pre-defined bindings. + foreach my $key (qw(NEXT_INSTRUCTION IP_ADJUSTMENT)) { + $new_bindings{$key} = $bindings{$key}; + } + for (my $i = 0; $i < @vars; $i++) { - $bindings{$vars[$i]} = $args[$i]; + my $arg = $args[$i]; + $arg = eval { expand_all($arg, \%bindings) }; + unless (defined $arg) { + warn $@; + die "... from the body of $name at $where\n"; + } + $new_bindings{$vars[$i]} = $arg; } - $body = eval { expand_all($body, \%bindings) }; + $body = eval { expand_all($body, \%new_bindings) }; unless (defined $body) { warn $@; die "... from the body of $name at $where\n"; } - ("do {\n$body\n} while (0)",$rest); + + # Handle built-in macros. + if ($name eq 'ARG_POSITION') { + if ($body =~ /^I\[(\d+)\]$/) { + $body = $1; + } else { + $body = 0; + } + } elsif ($name eq 'IS_PACKED') { + $body = ($body =~ /^I\[\d+\]$/) ? 0 : 1; + } + + # Wrap body if needed and return resul.t + $body = "do {\n$body\n} while (0)" + if needs_do_wrapper($body); + ($body,$rest); +} + +# Conservative heuristic to determine whether a do { ... } while(0) +# wrapper is needed. +sub needs_do_wrapper { + local $_ = shift; + + s@^//[|][^\n]*\n@@; + s@^\s*@@s; + s@^/[*].*[*]/\s*@@s; + return 1 if /^(Eterm|Uint|Sint|int|unsigned)/; # Definitely needed. + return 0 if /^do/; + return 0 if /^SET_I/; + return 0 if /^SET_CP/; + return 0 if /^ERTS_NO_FPE_CHECK_INIT/; + return 0 if /^ASSERT/; + return 0 if /^DTRACE/; + return 0 if /^[A-Za-z_]*\s*=/; + return 0 if /^c_p->/; + return 0 if /^[A-Z_]*SWAPOUT/; + return 0 if /^if\s*[(]/; + return 0 if /^goto\b/; + return 0 if /^\d+/; + return 1; # Not sure, say that it is needed. } sub do_pack { - my(@args) = @_; + my($name,$offset,$pack_opts_ref,@args) = @_; + my @pack_opts = @$pack_opts_ref; + my $opt_arg_pos = -1; + + # Look for an optional use operand not as the first argument. + if (@args and $args[0] !~ /[?]$/) { + for (my $pos = 0; $pos < @args; $pos++) { + if ($args[$pos] =~ /[?]$/) { + $opt_arg_pos = $pos; + last; + } + } + } + + @args = map { s/[?]$//; $_ } @args; # Remove any optional use marker. + + # If there is an optional operand, extend the array of pack options. + if ($opt_arg_pos >= 0) { + my @new_pack_opts = grep { $_ & PACK_IN_INSTR_WORD } @pack_opts; + @new_pack_opts = map { + ($_ & ~ PACK_IN_INSTR_WORD) | PACK_OPT_IN_INSTR_WORD; + } @new_pack_opts; + push @pack_opts, @new_pack_opts; + } + + my $ret = ['', ':', @args]; + my $score = 0; + + foreach my $options (@pack_opts) { + my $this_opt_arg_pos = ($options & PACK_OPT_IN_INSTR_WORD) ? $opt_arg_pos : -1; + my($this_score,$this_result) = + do_pack_one($name, $options, $this_opt_arg_pos, $offset, @args); + if ($this_score > $score) { + $ret = $this_result; + $score = $this_score; + } + } + return @$ret; +} + +sub do_pack_one { + my($name,$options,$opt_arg_pos,$offset,@args) = @_; my($packable_args) = 0; - my @is_packable; # Packability (boolean) for each argument. - my $wide_packing = 0; - my(@orig_args) = @args; + my @bits_needed; # Bits needed for each argument. + my $pack_in_iw = $options & PACK_IN_INSTR_WORD; + + # + # Define the minimum number of bits needed for the packable argument types. + # + my %bits_needed = ('x' => 10, + 'y' => 10, + 'Q' => 10, + 'l' => 10, + 'S' => 16, + 'd' => 16, + 't' => 16); + if ($wordsize == 64) { + $bits_needed{'I'} = 32; + if ($options & PACK_JUMP) { + $bits_needed{'f'} = 32; + $bits_needed{'j'} = 32; + } + } # - # Count the number of packable arguments. If we encounter any 's' or 'd' - # arguments, packing is not possible. + # Count the number of packable arguments. # - my $packable_types = "xytQ"; foreach my $arg (@args) { - if ($arg =~ /^[$packable_types]/) { + if (defined $bits_needed{$arg}) { $packable_args++; - push @is_packable, 1; - } elsif ($arg =~ /^I/ and $wordsize == 64 and $packable_args < 2) { - $wide_packing = 1; - push @is_packable, 1; - if (++$packable_args == 2) { - # We can only pack two arguments. Turn off packing - # for the rest of the arguments. - $packable_types = "\xFF"; - } - } elsif ($arg =~ /^[sd]/) { - return ('', '', @args); - } elsif ($arg =~ /^[scq]/ and $packable_args > 0) { - # When packing, this operand will be picked up from the - # code array, put onto the packing stack, and later put - # back into a different location in the code. The problem - # is that if this operand is a literal, the original - # location in the code would have been remembered in a - # literal patch. For packing to work, we would have to - # adjust the position in the literal patch. For the - # moment, adding additional instructions to the packing - # engine to handle this does not seem worth it, so we will - # just turn off packing. - return ('', '', @args); + push @bits_needed, $bits_needed{$arg}; } else { - push @is_packable, 0; + push @bits_needed, 0; } + if ($arg =~ /^[fj]$/) { + # Only pack the first occurrence of 'f' or 'j'. + delete $bits_needed{'f'}; + delete $bits_needed{'j'}; + } } # - # Get out of here if too few or too many arguments. + # Return if there is nothing to pack. # - return ('', '', @args) if $packable_args < 2; + if ($packable_args == 0) { + return (-1); + } elsif ($packable_args == 1 and $options == 0) { + return (-1); + } - my($size) = 0; - my($pack_prefix) = ''; - my($down) = ''; # Pack commands (towards instruction - # beginning). - my($up) = ''; # Pack commands (storing back while - # moving forward). + # + # Determine how many arguments we should pack into each word. + # + my @args_per_word; + my @need_wide_mask; + my $bits; + my $this_wordsize; + my $word = -1; - my $args_per_word = $args_per_word[$packable_args]; - my @shift; - my @mask; - my @instr; + my $next_word = sub { + $word++; + $args_per_word[$word] = 0; + $need_wide_mask[$word] = 0; + $bits = 0; + $this_wordsize = $wordsize; + }; - if ($wide_packing) { - @shift = ('0', 'BEAM_WIDE_SHIFT'); - @mask = ('BEAM_WIDE_MASK', $WHOLE_WORD); - @instr = ('w', 'i'); - } else { - @shift = @{$pack_shift[$args_per_word]}; - @mask = @{$pack_mask[$args_per_word]}; - @instr = @{$pack_instr[$args_per_word]}; + $next_word->(); + $this_wordsize = 32 if $pack_in_iw; + for (my $arg_num = 0; $arg_num < @args; $arg_num++) { + my $needed = $bits_needed[$arg_num]; + + next unless $needed; + next if $arg_num == $opt_arg_pos; + + if ($bits+$needed > $this_wordsize) { # Does not fit. + $next_word->(); + } + if ($args_per_word[$word] == 4) { # Can't handle more than 4 args. + $next_word->(); + } + if ($needed == 32 and $args_per_word[$word] > 1) { + # Must only pack two arguments in this word, and there + # are already at least two arguments here. + $next_word->(); + } + $args_per_word[$word]++; + $bits += $needed; + if ($needed == 32) { + $need_wide_mask[$word]++; + } + if ($need_wide_mask[$word] and $bits > 32) { + # Can only pack two things in a word where one + # item is 32 bits. Force the next item into + # the next word. + $bits = $this_wordsize; + } } # + # Try to balance packing between words. + # + if (@args_per_word == 1 and $args_per_word[0] == 1 and $pack_in_iw) { + # Don't rebalance. + } elsif ($args_per_word[$#args_per_word] == 1) { + if ($args_per_word[$#args_per_word-1] < 3) { + pop @args_per_word; + } else { + $args_per_word[$#args_per_word-1]--; + $args_per_word[$#args_per_word]++; + } + } elsif (@args_per_word == 2 and + $args_per_word[0] == 4 and + $args_per_word[1] == 2) { + $args_per_word[0] = 3; + $args_per_word[1] = 3; + } elsif (@args_per_word == 2 and + $args_per_word[0] == 3 and + $args_per_word[1] == 1) { + $args_per_word[0] = 2; + $args_per_word[1] = 2; + } + + my $size = 0; + my $pack_prefix = ''; + my $down = ''; # Pack commands (towards instruction + # beginning). + my $up = ''; # Pack commands (storing back while + # moving forward). + my $arg_num = 0; # Number of argument. + + # Skip an unpackable argument. Also handle packing of + # an single operand into the instruction word. + my $skip_unpackable = sub { + my($arg) = @_; + + if ($arg_num == $opt_arg_pos) { + my $pack = chr(ord('#') + $arg_num); + $down = PACK_CMD_WIDE . "$pack$down"; + my $unpack = "BeamExtraData(I[0])"; + $args[$arg_num] = "packed:$arg:0:${arg}b($unpack)"; + } elsif ($arg_size{$arg}) { + # Save the argument on the pack engine's stack. + my $push = 'g'; + if ($type_bit{$arg} & $type_bit{'q'}) { + # The operand may be a literal. + $push = 'q'; + } elsif ($type_bit{$arg} & $type_bit{'f'}) { + # The operand may be a failure label. + $push = 'f'; + } + $down = "$push${down}"; + $up = "${up}p"; + } + }; + + # # Now generate the packing instructions. One complication is that # the packing engine works from right-to-left, but we must generate # the instructions from left-to-right because we must calculate # instruction sizes from left-to-right. + for (my $word = 0; $word < @args_per_word; $word++) { + my $ap = 0; # Argument number within word. + my $packed_var = "tmp_packed" . ($word+1); + my $args_per_word = $args_per_word[$word]; + my $pack_word_size = ($pack_in_iw && $word == 0) ? 32 : $wordsize; + + my($shref,$mref,$iref,$unpack_suffix) = + get_pack_parameters($name, $args_per_word, $pack_word_size, + $need_wide_mask[$word]); + my @shift = @$shref; + my @mask = @$mref; + my @instr = @$iref; + + while ($ap < $args_per_word) { + my $reg = $args[$arg_num]; + my $this_size = $arg_size{$reg}; + + if ($bits_needed[$arg_num]) { + $this_size = 0; + + if ($ap == 0) { + my $packed_data; + if ($pack_in_iw and $word == 0) { + $packed_data = "BeamExtraData(I[0])"; + if ($args_per_word == 1) { + $packed_var = $packed_data; + } else { + $pack_prefix .= "Eterm $packed_var = $packed_data;\n"; + } + my $pack = chr(ord('#') + $size); + $down = "$pack$down"; + } else { + $packed_data = arg_offset($size + $offset); + $pack_prefix .= "Eterm $packed_var = $packed_data;\n"; + $down = "P$down"; + $up .= "p"; + $this_size = 1; + } + } + + $down = "$instr[$ap]$down"; + my $unpack = make_unpack($packed_var, $shift[$ap], $mask[$ap]); + my $macro = "$reg$unpack_suffix"; + $args[$arg_num] = "packed:$reg:$this_size:$macro($unpack)"; + + $ap++; + } else { + $skip_unpackable->($reg); + } + $size += $this_size; + $arg_num++; + } + } + # - # XXX Packing 3 't's in one word won't work. Sorry. - - my $did_some_packing = 0; # Nothing packed yet. - my($ap) = 0; # Argument number within word. - my($tmpnum) = 1; # Number of temporary variable. - my($expr) = ''; - for (my $i = 0; $i < @args; $i++) { - my($reg) = $args[$i]; - my($this_size) = $arg_size{$reg}; - if ($is_packable[$i]) { - $this_size = 0; - $did_some_packing = 1; - - if ($ap == 0) { - $pack_prefix .= "Eterm tmp_packed$tmpnum = Arg($size);\n"; - $up .= "p"; - $down = "P$down"; - $this_size = 1; - } + # Skip any unpackable arguments at the end. + # + while ($arg_num < @args) { + my $arg = $args[$arg_num]; + $skip_unpackable->($arg); + $size += $arg_size{$arg}; + $arg_num++; + } - $down = "$instr[$ap]$down"; - my($unpack) = make_unpack($tmpnum, $shift[$ap], $mask[$ap]); - $args[$i] = "pack:$this_size:$reg" . "b($unpack)"; + my $pack_spec = "$down:$up"; + my $score = pack_score($options, @args); - if (++$ap == $args_per_word) { - $ap = 0; - $tmpnum++; - } - } elsif ($arg_size{$reg} && $did_some_packing) { - # - # This is an argument that can't be packed. Normally, we must - # save it on the pack engine's stack, unless: - # - # 1. The argument has zero size (e.g. r(0)). Such arguments - # will not be loaded. They disappear. - # 2. If the argument is on the left of the first packed argument, - # the packing engine will never access it (because the engine - # operates from right-to-left). - # + return ($score, [$pack_prefix,$pack_spec,@args]); +} - $down = "g${down}"; - $up = "${up}p"; - } - $size += $this_size; +sub get_pack_parameters { + my($name,$args_per_word,$pack_word_size,$wide_mask) = @_; + my(@shift,@mask,@instr); + my $unpack_suffix = 'b'; + + if ($wide_mask and $args_per_word > 1) { + @shift = ('0', 'BEAM_WIDE_SHIFT'); + @mask = ('BEAM_WIDE_MASK', $WHOLE_WORD); + @instr = (PACK_CMD_WIDE) x 2; + } elsif ($args_per_word == 1) { + @shift = ('0'); + @mask = ($WHOLE_WORD); + @instr = (PACK_CMD_WIDE); + } elsif ($args_per_word == 2) { + if ($pack_word_size != $wordsize) { + # 64-bit word size, pack 32 bits into instruction word. + @shift = ('0', 'BEAM_TIGHT_SHIFT'); + @mask = ('BEAM_TIGHT_MASK', $WHOLE_WORD); + @instr = (PACK_CMD_TIGHT) x 2; + } else { + # 32/64 bit word size + @shift = ('0', 'BEAM_LOOSE_SHIFT'); + @mask = ('BEAM_LOOSE_MASK', $WHOLE_WORD); + @instr = (PACK_CMD_LOOSE) x 2; + } + } elsif ($args_per_word == 3) { + if ($pack_word_size != $wordsize) { + # 64-bit word size, pack 3 register numbers into instruction word. + @shift = ('0', 'BEAM_TIGHTEST_SHIFT', '(2*BEAM_TIGHTEST_SHIFT)'); + @mask = ('BEAM_TIGHTEST_MASK', 'BEAM_TIGHTEST_MASK', $WHOLE_WORD); + @instr = (PACK_CMD_TIGHTEST) x 3; + $unpack_suffix = ''; + } else { + # 32/64 bit word size. + @shift = ('0', 'BEAM_TIGHT_SHIFT', '(2*BEAM_TIGHT_SHIFT)'); + if ($wordsize == 32) { + @mask = ('BEAM_TIGHT_MASK') x 3; + } elsif ($wordsize == 64) { + @mask = ('BEAM_TIGHT_MASK', 'BEAM_TIGHT_MASK', $WHOLE_WORD); + } + @instr = (PACK_CMD_TIGHT) x 3; + } + } elsif ($args_per_word == 4) { + # 64 bit word size only. + @shift = ('0', + 'BEAM_LOOSE_SHIFT', + '(2*BEAM_LOOSE_SHIFT)', + '(3*BEAM_LOOSE_SHIFT)'); + @mask = ('BEAM_LOOSE_MASK', 'BEAM_LOOSE_MASK', + 'BEAM_LOOSE_MASK', $WHOLE_WORD); + @instr = (PACK_CMD_LOOSE) x 4; + } + + unless (@shift) { + error("$name: internal packing error: args_per_word=$args_per_word, " . + "pack_word_size=$pack_word_size"); } - my $pack_spec = $down . $up; - return ($pack_prefix, $pack_spec, @args); + (\@shift,\@mask,\@instr,$unpack_suffix); +} + +sub pack_score { + my($options,@args) = @_; + my $size = 0; + + # Calculate the number of words. + foreach (@args) { + if (/^packed:[^:]*:(\d+)/) { + $size += $1; + } else { + $size += $arg_size{$_} + } + } + + # Less numbers of words give a higher score; for the same number of + # words, using PACK_JUMP or PACK_IN_INSTR_WORD gives a lower score. + my $score = 1 + 10*($max_spec_operands - $size); + if (($options & PACK_OPT_IN_INSTR_WORD) != 0) { + $score += 4; + } elsif ($options == PACK_IN_INSTR_WORD) { + $score += 0; + } elsif ($options == PACK_JUMP) { + $score += 1; + } elsif ($options == (PACK_JUMP|PACK_IN_INSTR_WORD)) { + $score += 2; + } elsif ($options == 0) { + $score += 3; + } + $score; } sub make_unpack { - my($tmpnum, $shift, $mask) = @_; + my($packed_var, $shift, $mask) = @_; - my($e) = "tmp_packed$tmpnum"; + my $e = $packed_var; $e = "($e>>$shift)" if $shift; $e .= "&$mask" unless $mask eq $WHOLE_WORD; $e; } +sub build_pack_spec { + my $pack_spec = shift; + return '' if $pack_spec eq ''; + my($down,$up) = split /:/, $pack_spec; + while ($down =~ /[gfq]$/ and $up =~ /^p/) { + $down = substr($down, 0, -1); + $up = substr($up, 1); + } + "$down$up"; +} + sub quote { local($_) = @_; return "'$_'" if $_ eq 'try'; @@ -1629,8 +2082,11 @@ sub parse_transformation { # my @to; - if ($to =~ /^(\w+)\((.*?)\)/) { - my($name, $arglist) = ($1, $2); + if ($to =~ /^(\w+)\((.*?)\)(.*)/) { + my($name, $arglist, $garbage) = ($1, $2, $3); + if ($garbage =~ /\S/) { + error("garbage after call to '$name()'"); + } @to = (compile_transform_function($name, split(/\s*,\s*/, $arglist))); } else { @to = split(/\s*\|\s*/, $to); diff --git a/erts/emulator/utils/make_tables b/erts/emulator/utils/make_tables index 47e1528958..094a35ae4b 100755 --- a/erts/emulator/utils/make_tables +++ b/erts/emulator/utils/make_tables @@ -59,7 +59,6 @@ my %dirty_bif_tab; my @bif; my @bif_info; -my $dirty_schedulers = 'no'; my $dirty_schedulers_test = 'no'; my $hipe = 'no'; @@ -73,10 +72,6 @@ while (@ARGV && $ARGV[0] =~ /^-(\w+)/) { $include = shift; die "No directory for -include argument specified" unless defined $include; - } elsif($opt eq '-ds') { - $dirty_schedulers = shift; - die "No -ds argument specified" - unless defined $dirty_schedulers; } elsif($opt eq '-dst') { $dirty_schedulers_test = shift; die "No -dst argument specified" @@ -140,21 +135,19 @@ while (<>) { push(@bif_info, [$type, $sched_type, $alias3, $alias]); } elsif ($type eq 'dirty-cpu' or $type eq 'dirty-io' or $type eq 'dirty-cpu-test' or $type eq 'dirty-io-test') { - if ($dirty_schedulers eq 'yes') { - my($bif,$other) = (@args); - $bif =~ m@^([a-z_.'0-9]+):(.*)/(\d)$@ or error("invalid BIF"); - my($mod,$name,$arity) = ($1,$2,$3); - my $mfa = "$mod:$name/$arity"; - if (($type eq 'dirty-cpu') - or (($dirty_schedulers_test eq 'yes') - and ($type eq 'dirty-cpu-test'))) { - $dirty_bif_tab{$mfa} = 'dirty_cpu'; - } elsif (($type eq 'dirty-io') - or (($dirty_schedulers_test eq 'yes') - and ($type eq 'dirty-io-test'))) { - $dirty_bif_tab{$mfa} = 'dirty_io'; - } - } + my($bif,$other) = (@args); + $bif =~ m@^([a-z_.'0-9]+):(.*)/(\d)$@ or error("invalid BIF"); + my($mod,$name,$arity) = ($1,$2,$3); + my $mfa = "$mod:$name/$arity"; + if (($type eq 'dirty-cpu') + or (($dirty_schedulers_test eq 'yes') + and ($type eq 'dirty-cpu-test'))) { + $dirty_bif_tab{$mfa} = 'dirty_cpu'; + } elsif (($type eq 'dirty-io') + or (($dirty_schedulers_test eq 'yes') + and ($type eq 'dirty-io-test'))) { + $dirty_bif_tab{$mfa} = 'dirty_io'; + } } else { error("invalid line"); } diff --git a/erts/emulator/valgrind/suppress.patched.3.6.0 b/erts/emulator/valgrind/suppress.patched.3.6.0 index fcde4a0123..29f2d3d62d 100644 --- a/erts/emulator/valgrind/suppress.patched.3.6.0 +++ b/erts/emulator/valgrind/suppress.patched.3.6.0 @@ -374,3 +374,10 @@ fun:erts_debug_set_internal_state_2 fun:process_main } +{ +Thread specific dlerror buffer. Either bug in libc or valgrind. +Memcheck:Leak +... +fun:_dlerror_run +... +} diff --git a/erts/emulator/valgrind/suppress.standard b/erts/emulator/valgrind/suppress.standard index bb07c92fc1..99a3ee4048 100644 --- a/erts/emulator/valgrind/suppress.standard +++ b/erts/emulator/valgrind/suppress.standard @@ -342,3 +342,11 @@ fun:erts_debug_set_internal_state_2 fun:process_main } +{ +Thread specific dlerror buffer. Either bug in libc or valgrind. +Memcheck:Leak +... +fun:_dlerror_run +... +} + |