diff options
Diffstat (limited to 'erts/emulator')
72 files changed, 5271 insertions, 985 deletions
diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index a5d8217545..2efbe2d57e 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2011. All Rights Reserved. +# Copyright Ericsson AB 1996-2012. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -23,26 +23,34 @@ include $(ERL_TOP)/make/$(TARGET)/otp.mk ENABLE_ALLOC_TYPE_VARS = @ENABLE_ALLOC_TYPE_VARS@ HIPE_ENABLED=@HIPE_ENABLED@ +DTRACE_ENABLED=@DTRACE_ENABLED@ +DTRACE_ENABLED_2STEP=@DTRACE_ENABLED_2STEP@ +USE_VM_PROBES=@USE_VM_PROBES@ LIBS = @LIBS@ Z_LIB=@Z_LIB@ NO_INLINE_FUNCTIONS=false OPCODE_TABLES = $(ERL_TOP)/lib/compiler/src/genop.tab beam/ops.tab +DEBUG_CFLAGS = @DEBUG_CFLAGS@ +CONFIGURE_CFLAGS = @CFLAGS@ + # # Run this make file with TYPE set to the type of emulator you want. # Different versions of the emulator for different uses. The default # is "debug". For a normal version use "opt". # +DEFS=@DEFS@ THR_DEFS=@EMU_THR_DEFS@ M4FLAGS= CREATE_DIRS= LDFLAGS=@LDFLAGS@ +ARFLAGS=rc ifeq ($(TYPE),debug) PURIFY = TYPEMARKER = .debug -TYPE_FLAGS = @DEBUG_CFLAGS@ -DDEBUG +TYPE_FLAGS = $(DEBUG_CFLAGS) -DDEBUG ENABLE_ALLOC_TYPE_VARS += debug ifeq ($(TARGET),win32) TYPE_FLAGS += -DNO_JUMP_TABLE @@ -53,7 +61,7 @@ else ifeq ($(TYPE),purify) PURIFY = purify $(PURIFY_BUILD_OPTIONS) TYPEMARKER = .purify -TYPE_FLAGS = @DEBUG_CFLAGS@ -DPURIFY -DNO_JUMP_TABLE -DERTS_MSEG_FAKE_SEGMENTS +TYPE_FLAGS = $(DEBUG_CFLAGS) -DPURIFY -DNO_JUMP_TABLE -DERTS_MSEG_FAKE_SEGMENTS ENABLE_ALLOC_TYPE_VARS += purify else @@ -67,14 +75,14 @@ else ifeq ($(TYPE),purecov) PURIFY = purecov --follow-child-processes=yes $(PURECOV_BUILD_OPTIONS) TYPEMARKER = .purecov -TYPE_FLAGS = @DEBUG_CFLAGS@ -DPURECOV -DNO_JUMP_TABLE +TYPE_FLAGS = $(DEBUG_CFLAGS) -DPURECOV -DNO_JUMP_TABLE ENABLE_ALLOC_TYPE_VARS += purecov else ifeq ($(TYPE),gcov) PURIFY = TYPEMARKER = .gcov -TYPE_FLAGS = @DEBUG_CFLAGS@ -DERTS_GCOV -DNO_JUMP_TABLE -fprofile-arcs -ftest-coverage -O0 -DERTS_CAN_INLINE=0 -DERTS_INLINE= +TYPE_FLAGS = $(DEBUG_CFLAGS) -DERTS_GCOV -DNO_JUMP_TABLE -fprofile-arcs -ftest-coverage -O0 -DERTS_CAN_INLINE=0 -DERTS_INLINE= ifneq ($(findstring solaris,$(TARGET)),solaris) LIBS += -lgcov endif @@ -84,7 +92,7 @@ else ifeq ($(TYPE),valgrind) PURIFY = TYPEMARKER = .valgrind -TYPE_FLAGS = @DEBUG_CFLAGS@ -DVALGRIND -DNO_JUMP_TABLE -DERTS_MSEG_FAKE_SEGMENTS +TYPE_FLAGS = $(DEBUG_CFLAGS) -DVALGRIND -DNO_JUMP_TABLE -DERTS_MSEG_FAKE_SEGMENTS ENABLE_ALLOC_TYPE_VARS += valgrind else @@ -147,6 +155,15 @@ endif TF_MARKER=$(TYPEMARKER)$(FLAVOR_MARKER) +ifeq ($(FLAVOR)-@ERTS_BUILD_SMP_EMU@,smp-no) +VOID_EMULATOR = '*** SMP emulator disabled by configure' +else +ifeq ($(TYPE)-@HAVE_VALGRIND@,valgrind-no) +VOID_EMULATOR = '*** valgrind emulator disabled by configure' +else +VOID_EMULATOR = +endif +endif OPSYS=@OPSYS@ sol2CFLAGS= @@ -187,7 +204,7 @@ else EMU_CC = @EMU_CC@ endif WFLAGS = @WFLAGS@ -CFLAGS = @STATIC_CFLAGS@ $(TYPE_FLAGS) $(FLAVOR_FLAGS) @DEFS@ $(WFLAGS) $(THR_DEFS) $(ARCHCFLAGS) +CFLAGS = @STATIC_CFLAGS@ $(TYPE_FLAGS) $(FLAVOR_FLAGS) $(DEFS) $(WFLAGS) $(THR_DEFS) $(ARCHCFLAGS) HCC = @HCC@ LD = @LD@ DEXPORT = @DEXPORT@ @@ -263,30 +280,29 @@ CS_PURIFY = CS_TYPE_FLAGS = $(subst QUANTIFY,FAKE_QUANTIFY, \ $(subst PURIFY,FAKE_PURIFY, $(TYPE_FLAGS))) endif -CS_CFLAGS_ = $(CS_TYPE_FLAGS) @DEFS@ $(WFLAGS) +CS_CFLAGS_ = $(CS_TYPE_FLAGS) $(DEFS) $(WFLAGS) ifeq ($(GCC),yes) CS_CFLAGS = $(subst -O2, $(GEN_OPT_FLGS) $(UNROLL_FLG), $(CS_CFLAGS_)) else CS_CFLAGS = $(CS_CFLAGS_) endif CS_LDFLAGS = $(LDFLAGS) -CS_LIBS = -L../lib/internal/$(TARGET) -lerts_internal @ERTS_INTERNAL_X_LIBS@ +CS_LIBS = -L../lib/internal/$(TARGET) -lerts_internal$(TYPEMARKER) @ERTS_INTERNAL_X_LIBS@ LIBS += @TERMCAP_LIB@ -L../lib/internal/$(TARGET) @ERTS_INTERNAL_X_LIBS@ ifdef Z_LIB # Use shared zlib library LIBS += $(Z_LIB) +DEPLIBS = else +DEPLIBS=$(ZLIB_LIBRARY) ifeq ($(TARGET),win32) -LIBS += -L$(ERL_TOP)/erts/emulator/zlib/obj/$(TARGET)/$(TYPE) -lz -DEPLIBS = $(ERL_TOP)/erts/emulator/zlib/obj/$(TARGET)/$(TYPE)/z.lib +LIBS += -L$(ZLIB_OBJDIR) -lz else # Build on darwin fails if -lz is used -LIBS += $(ERL_TOP)/erts/emulator/zlib/obj/$(TARGET)/$(TYPE)/libz.a -DEPLIBS = $(ERL_TOP)/erts/emulator/zlib/obj/$(TARGET)/$(TYPE)/libz.a +LIBS += $(ZLIB_LIBRARY) endif - endif ifeq ($(TARGET),win32) @@ -313,12 +329,9 @@ LIBSCTP = @LIBSCTP@ ORG_THR_LIBS=@EMU_THR_LIBS@ THR_LIB_NAME=@EMU_THR_LIB_NAME@ -ifneq ($(strip $(THR_LIB_NAME)),) -DEPLIBS += $(ERL_TOP)/erts/lib/internal/$(TARGET)/$(LIB_PREFIX)erts_internal_r$(TYPEMARKER)$(LIB_SUFFIX) \ - $(ERL_TOP)/erts/lib/internal/$(TARGET)/$(LIB_PREFIX)ethread$(TYPEMARKER)$(LIB_SUFFIX) -else -DEPLIBS += $(ERL_TOP)/erts/lib/internal/$(TARGET)/$(LIB_PREFIX)erts_internal$(TYPEMARKER)$(LIB_SUFFIX) -endif +ERTS_LIB_DIR = $(ERL_TOP)/erts/lib_src +ERTS_LIB = $(ERTS_LIB_DIR)/obj/$(TARGET)/$(TYPE)/MADE +DEPLIBS += $(ERTS_LIB) THR_LIBS=$(subst -l$(THR_LIB_NAME),-l$(THR_LIB_NAME)$(TYPEMARKER), \ $(subst -lerts_internal_r,-lerts_internal_r$(TYPEMARKER),$(ORG_THR_LIBS))) @@ -354,8 +367,7 @@ OBJDIR = obj/$(TTF_DIR) CREATE_DIRS += $(OBJDIR) \ pcre/obj/$(TARGET)/$(TYPE) \ - zlib/obj/$(TARGET)/$(TYPE) - + $(ZLIB_OBJDIR) BINDIR = $(ERL_TOP)/bin/$(TARGET) @@ -378,15 +390,12 @@ else UNIX_ONLY_BUILDS = endif -ifeq ($(TYPE)-@HAVE_VALGRIND@,valgrind-no) -all: - @echo '*** valgrind not found by configure' -else -ifeq ($(FLAVOR)-@ERTS_BUILD_SMP_EMU@,smp-no) +.PHONY: all +ifdef VOID_EMULATOR all: - @echo '*** Omitted build of emulator with smp support' + @echo $(VOID_EMULATOR)' - omitted target all' else -all: generate erts_lib zlib $(BINDIR)/$(EMULATOR_EXECUTABLE) $(UNIX_ONLY_BUILDS) +all: $(BINDIR)/$(EMULATOR_EXECUTABLE) $(UNIX_ONLY_BUILDS) ifeq ($(OMIT_OMIT_FP),yes) @echo '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *' @echo '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *' @@ -397,37 +406,25 @@ ifeq ($(OMIT_OMIT_FP),yes) @echo '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *' endif endif -endif - -ifdef Z_LIB -zlib: - @echo 'Skip zlib directory, use shared library' -else -zlib: - @set -e ; cd zlib && $(MAKE) TYPE=$(TYPE) $(TYPE) -endif - +include zlib/zlib.mk include pcre/pcre.mk -erts_lib: - cd $(ERL_TOP)/erts/lib_src && $(MAKE) $(TYPE) +$(ERTS_LIB): + cd $(ERTS_LIB_DIR) && $(MAKE) $(TYPE) +.PHONY: clean clean: -ifeq ($(TARGET),win32) - $(RM) -f $(TARGET)/beams.rc -endif - $(RM) -f $(TARGET)/*.c $(TARGET)/*.h $(TARGET)/depend.mk - $(RM) -f $(TARGET)/*/*/*.c $(TARGET)/*/*/*.h $(TARGET)/*/*/*.S - $(RM) -f $(ERL_TOP)/erts/emulator/obj/$(TARGET)/*/*/*.o - $(RM) -f $(BINDIR)/beam $(BINDIR)/beam.* - $(RM) -rf $(BINDIR)/child_setup $(BINDIR)/child_setup.* - $(RM) -f $(BINDIR)/hipe_mkliterals $(BINDIR)/hipe_mkliterals.* - @set -e ; cd zlib && $(MAKE) clean - rm -f $(OBJS) $(OBJDIR)/libepcre.a - -.PHONY: all zlib clean - + $(RM) -f $(GENERATE) + $(RM) -rf $(TARGET)/*.c $(TARGET)/*.h $(TARGET)/*-GENERATED + $(RM) -rf $(TARGET)/*/* + $(RM) -rf obj/$(TARGET) + $(RM) -rf pcre/obj/$(TARGET) $(PCRE_GENINC) + $(RM) -rf zlib/obj/$(TARGET) + $(RM) -rf bin/$(TARGET) + cd $(ERTS_LIB_DIR) && $(MAKE) clean + +.PHONY: docs docs: # ---------------------------------------------------------------------- @@ -440,13 +437,11 @@ ifeq ($(TARGET),win32) RELEASE_INCLUDES += sys/$(ERLANG_OSTYPE)/erl_win_dyn_driver.h endif -ifeq ($(TYPE)-@HAVE_VALGRIND@,valgrind-no) -release_spec: - @echo '*** valgrind not found by configure' -else -ifeq ($(FLAVOR)-@ERTS_BUILD_SMP_EMU@,smp-no) + +.PHONY: release_spec +ifdef VOID_EMULATOR release_spec: - @echo '*** No emulator with smp support to install' + @echo $(VOID_EMULATOR)' - omitted target release_spec (install)' else release_spec: all $(INSTALL_DIR) $(RELSYSDIR) @@ -463,8 +458,8 @@ ifeq ($(ERLANG_OSTYPE), unix) $(INSTALL_PROGRAM) $(BINDIR)/$(CS_EXECUTABLE) $(RELSYSDIR)/bin endif endif -endif +.PHONY: release_docs_spec release_docs_spec: # ---------------------------------------------------------------------- @@ -473,48 +468,50 @@ release_docs_spec: _create_dirs := $(shell mkdir -p $(CREATE_DIRS)) -.PHONY : generate - -GENERATE= $(TTF_DIR)/beam_opcodes.h \ - $(TARGET)/erl_bif_table.c \ - $(TARGET)/erl_version.h \ - $(TTF_DIR)/driver_tab.c \ - $(TTF_DIR)/erl_alloc_types.h - -ifeq ($(TARGET),win32) -GENERATE += $(TARGET)/beams.rc -else -GENERATE += $(TARGET)/preload.c -endif +GENERATE = +HIPE_ASM = ifeq ($(findstring vxworks,$(TARGET)),vxworks) else ifdef HIPE_ENABLED -GENERATE += $(TTF_DIR)/hipe_x86_asm.h \ +HIPE_ASM += $(TTF_DIR)/hipe_x86_asm.h \ $(TTF_DIR)/hipe_amd64_asm.h \ $(TTF_DIR)/hipe_sparc_asm.h \ $(TTF_DIR)/hipe_ppc_asm.h \ - $(TTF_DIR)/hipe_arm_asm.h \ + $(TTF_DIR)/hipe_arm_asm.h + +GENERATE += $(HIPE_ASM) \ $(TTF_DIR)/hipe_literals.h \ $(BINDIR)/hipe_mkliterals$(TF_MARKER) endif endif -ifeq ($(FLAVOR)-@ERTS_BUILD_SMP_EMU@,smp-no) -GENERATE= -endif - +ifdef DTRACE_ENABLED +# global.h causes problems by including dtrace-wrapper.h which includes +# the autogenerated erlang_dtrace.h ... so make erlang_dtrace.h very early. +generate: $(TARGET)/erlang_dtrace.h $(GENERATE) +else generate: $(GENERATE) +endif ifdef HIPE_ENABLED OPCODE_TABLES += hipe/hipe_ops.tab endif -$(TTF_DIR)/beam_opcodes.h $(TTF_DIR)/beam_opcodes.c: $(OPCODE_TABLES) utils/beam_makeops +$(TTF_DIR)/beam_cold.h \ +$(TTF_DIR)/beam_hot.h \ +$(TTF_DIR)/beam_opcodes.c \ +$(TTF_DIR)/beam_opcodes.h \ +$(TTF_DIR)/beam_pred_funcs.h \ +$(TTF_DIR)/beam_tr_funcs.h \ + : $(TTF_DIR)/OPCODES-GENERATED +$(TTF_DIR)/OPCODES-GENERATED: $(OPCODE_TABLES) utils/beam_makeops LANG=C $(PERL) utils/beam_makeops \ -wordsize @EXTERNAL_WORD_SIZE@ \ -outdir $(TTF_DIR) \ - -emulator $(OPCODE_TABLES) + -DUSE_VM_PROBES=$(if $(USE_VM_PROBES),1,0) \ + -emulator $(OPCODE_TABLES) && echo $? >$(TTF_DIR)/OPCODES-GENERATED +GENERATE += $(TTF_DIR)/OPCODES-GENERATED # bif and atom table ATOMS= beam/atom.names @@ -534,32 +531,44 @@ BIFS += hipe/hipe_perfctr.tab endif endif -TABLES= $(TARGET)/erl_bif_table.c $(TARGET)/erl_bif_table.h \ - $(TARGET)/erl_bif_wrap.c $(TARGET)/erl_bif_list.h \ - $(TARGET)/erl_atom_table.c $(TARGET)/erl_atom_table.h \ - $(TARGET)/erl_pbifs.c - -$(TABLES): $(ATOMS) $(BIFS) utils/make_tables +$(TARGET)/erl_bif_table.c \ +$(TARGET)/erl_bif_table.h \ +$(TARGET)/erl_bif_wrap.c \ +$(TARGET)/erl_bif_list.h \ +$(TARGET)/erl_atom_table.c \ +$(TARGET)/erl_atom_table.h \ +$(TARGET)/erl_pbifs.c \ + : $(TARGET)/TABLES-GENERATED +$(TARGET)/TABLES-GENERATED: $(ATOMS) $(BIFS) utils/make_tables LANG=C $(PERL) utils/make_tables -src $(TARGET) -include $(TARGET)\ - $(ATOMS) $(BIFS) + $(ATOMS) $(BIFS) && echo $? >$(TARGET)/TABLES-GENERATED +GENERATE += $(TARGET)/TABLES-GENERATED $(TTF_DIR)/erl_alloc_types.h: beam/erl_alloc.types utils/make_alloc_types LANG=C $(PERL) utils/make_alloc_types -src $< -dst $@ $(ENABLE_ALLOC_TYPE_VARS) +GENERATE += $(TTF_DIR)/erl_alloc_types.h # version include file $(TARGET)/erl_version.h: ../vsn.mk LANG=C $(PERL) utils/make_version -o $@ $(SYSTEM_VSN) $(VSN)$(SERIALNO) $(TARGET) +GENERATE += $(TARGET)/erl_version.h # driver table $(TTF_DIR)/driver_tab.c: Makefile.in LANG=C $(PERL) utils/make_driver_tab -o $@ $(DRV_OBJS) +GENERATE += $(TTF_DIR)/driver_tab.c + + # Preloaded code. # # This list must be consistent with PRE_LOADED_MODULES in # lib/kernel/src/Makefile. ifeq ($(TARGET),win32) -$(TARGET)/beams.rc: $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ +# On windows the preloaded objects are in a resource object. +PRELOAD_OBJ = $(OBJDIR)/beams.$(RES_EXT) +PRELOAD_SRC = $(TARGET)/beams.rc +$(PRELOAD_SRC): $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ $(ERL_TOP)/erts/preloaded/ebin/init.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_inet.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_file.beam \ @@ -569,7 +578,9 @@ $(TARGET)/beams.rc: $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ $(ERL_TOP)/erts/preloaded/ebin/erlang.beam LANG=C $(PERL) utils/make_preload $(MAKE_PRELOAD_EXTRA) -rc $^ > $@ else -$(TARGET)/preload.c: $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ +PRELOAD_OBJ = $(OBJDIR)/preload.o +PRELOAD_SRC = $(TARGET)/preload.c +$(PRELOAD_SRC): $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ $(ERL_TOP)/erts/preloaded/ebin/init.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_inet.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_file.beam \ @@ -580,6 +591,22 @@ $(TARGET)/preload.c: $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ LANG=C $(PERL) utils/make_preload -old $^ > $@ endif +.PHONY : generate +ifdef VOID_EMULATOR +generate: + @echo $(VOID_EMULATOR)' - omitted target generate' +else +generate: $(TTF_DIR)/GENERATED $(PRELOAD_SRC) + +$(TTF_DIR)/GENERATED: $(GENERATE) + echo $? >$(TTF_DIR)/GENERATED +endif + +$(TARGET)/erlang_dtrace.h: beam/erlang_dtrace.d + dtrace -h -C -Ibeam -s $< -o ./erlang_dtrace.tmp + sed -e '/^#define[ ]*ERLANG_[A-Z0-9_]*(.*)/y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/' ./erlang_dtrace.tmp > $@ + rm ./erlang_dtrace.tmp + # ---------------------------------------------------------------------- # Pattern rules # @@ -623,7 +650,6 @@ $(OBJDIR)/beam_emu.o: beam/beam_emu.c $(EMU_CC) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@ endif - $(OBJDIR)/%.o: beam/%.c $(CC) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@ @@ -656,7 +682,7 @@ endif # CS_SRC = sys/$(ERLANG_OSTYPE)/erl_child_setup.c -$(BINDIR)/$(CS_EXECUTABLE): $(CS_SRC) +$(BINDIR)/$(CS_EXECUTABLE): $(TTF_DIR)/GENERATED $(PRELOAD_SRC) $(CS_SRC) $(ERTS_LIB) $(CS_PURIFY) $(CC) $(CS_LDFLAGS) -o $(BINDIR)/$(CS_EXECUTABLE) \ $(CS_CFLAGS) $(COMMON_INCLUDES) $(CS_SRC) $(CS_LIBS) @@ -685,16 +711,7 @@ $(ERL_TOP)/lib/%.beam: # Object files # -# On windows the preloaded objects are in a resource object. - -ifeq ($(TARGET),win32) -PRELOAD = $(OBJDIR)/beams.$(RES_EXT) -else -PRELOAD = $(OBJDIR)/preload.o -endif - - -INIT_OBJS = $(OBJDIR)/erl_main.o $(PRELOAD) +INIT_OBJS = $(OBJDIR)/erl_main.o $(PRELOAD_OBJ) EMU_OBJS = \ $(OBJDIR)/beam_emu.o $(OBJDIR)/beam_opcodes.o \ @@ -832,7 +849,21 @@ endif BASE_OBJS = $(RUN_OBJS) $(EMU_OBJS) $(OS_OBJS) $(EXTRA_BASE_OBJS) -OBJS = $(BASE_OBJS) $(DRV_OBJS) +before_DTrace_OBJS = $(BASE_OBJS) $(DRV_OBJS) + +DTRACE_OBJS = +ifdef DTRACE_ENABLED_2STEP +DTRACE_OBJS = $(OBJDIR)/erlang_dtrace.o +$(OBJDIR)/erlang_dtrace.o: $(before_DTrace_OBJS) $(TARGET)/erlang_dtrace.h + dtrace -G -C -Ibeam \ + -s beam/erlang_dtrace.d \ + -o $@ $(before_DTrace_OBJS) +endif + +OBJS = $(before_DTrace_OBJS) $(DTRACE_OBJS) + +$(INIT_OBJS): $(TTF_DIR)/GENERATED +$(OBJS): $(TTF_DIR)/GENERATED ######################################## # HiPE section @@ -857,30 +888,49 @@ $(OBJDIR)/%.o: hipe/%.c $(BINDIR)/hipe_mkliterals$(TF_MARKER): $(OBJDIR)/hipe_mkliterals.o $(CC) $(CFLAGS) $(INCLUDES) -o $@ $< -$(OBJDIR)/hipe_mkliterals.o: $(TTF_DIR)/hipe_x86_asm.h $(TTF_DIR)/hipe_ppc_asm.h $(TTF_DIR)/beam_opcodes.h +$(OBJDIR)/hipe_mkliterals.o: $(HIPE_ASM) $(TTF_DIR)/erl_alloc_types.h \ + $(TTF_DIR)/OPCODES-GENERATED $(TARGET)/TABLES-GENERATED $(TTF_DIR)/hipe_literals.h: $(BINDIR)/hipe_mkliterals$(TF_MARKER) $(BINDIR)/hipe_mkliterals$(TF_MARKER) -c > $@ -$(OBJDIR)/hipe_x86_glue.o: hipe/hipe_x86_glue.S $(TTF_DIR)/hipe_x86_asm.h $(TTF_DIR)/hipe_literals.h hipe/hipe_mode_switch.h -$(TTF_DIR)/hipe_x86_bifs.S: hipe/hipe_x86_bifs.m4 hipe/hipe_x86_asm.m4 hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h -$(OBJDIR)/hipe_x86_bifs.o: $(TTF_DIR)/hipe_x86_bifs.S $(TTF_DIR)/hipe_literals.h - -$(OBJDIR)/hipe_amd64_glue.o: hipe/hipe_amd64_glue.S $(TTF_DIR)/hipe_amd64_asm.h $(TTF_DIR)/hipe_literals.h hipe/hipe_mode_switch.h -$(TTF_DIR)/hipe_amd64_bifs.S: hipe/hipe_amd64_bifs.m4 hipe/hipe_amd64_asm.m4 hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h -$(OBJDIR)/hipe_amd64_bifs.o: $(TTF_DIR)/hipe_amd64_bifs.S $(TTF_DIR)/hipe_literals.h - -$(OBJDIR)/hipe_sparc_glue.o: hipe/hipe_sparc_glue.S $(TTF_DIR)/hipe_sparc_asm.h hipe/hipe_mode_switch.h $(TTF_DIR)/hipe_literals.h -$(TTF_DIR)/hipe_sparc_bifs.S: hipe/hipe_sparc_bifs.m4 hipe/hipe_sparc_asm.m4 hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h -$(OBJDIR)/hipe_sparc_bifs.o: $(TTF_DIR)/hipe_sparc_bifs.S $(TTF_DIR)/hipe_literals.h - -$(OBJDIR)/hipe_ppc_glue.o: hipe/hipe_ppc_glue.S $(TTF_DIR)/hipe_ppc_asm.h hipe/hipe_mode_switch.h $(TTF_DIR)/hipe_literals.h -$(TTF_DIR)/hipe_ppc_bifs.S: hipe/hipe_ppc_bifs.m4 hipe/hipe_ppc_asm.m4 hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h -$(OBJDIR)/hipe_ppc_bifs.o: $(TTF_DIR)/hipe_ppc_bifs.S $(TTF_DIR)/hipe_literals.h - -$(OBJDIR)/hipe_arm_glue.o: hipe/hipe_arm_glue.S $(TTF_DIR)/hipe_arm_asm.h hipe/hipe_mode_switch.h $(TTF_DIR)/hipe_literals.h -$(TTF_DIR)/hipe_arm_bifs.S: hipe/hipe_arm_bifs.m4 hipe/hipe_arm_asm.m4 hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h -$(OBJDIR)/hipe_arm_bifs.o: $(TTF_DIR)/hipe_arm_bifs.S $(TTF_DIR)/hipe_literals.h +$(OBJDIR)/hipe_x86_glue.o: hipe/hipe_x86_glue.S \ + $(TTF_DIR)/hipe_x86_asm.h $(TTF_DIR)/hipe_literals.h \ + hipe/hipe_mode_switch.h +$(TTF_DIR)/hipe_x86_bifs.S: hipe/hipe_x86_bifs.m4 hipe/hipe_x86_asm.m4 \ + hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h +$(OBJDIR)/hipe_x86_bifs.o: $(TTF_DIR)/hipe_x86_bifs.S \ + $(TTF_DIR)/hipe_literals.h + +$(OBJDIR)/hipe_amd64_glue.o: hipe/hipe_amd64_glue.S \ + $(TTF_DIR)/hipe_amd64_asm.h $(TTF_DIR)/hipe_literals.h \ + hipe/hipe_mode_switch.h +$(TTF_DIR)/hipe_amd64_bifs.S: hipe/hipe_amd64_bifs.m4 hipe/hipe_amd64_asm.m4 \ + hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h +$(OBJDIR)/hipe_amd64_bifs.o: $(TTF_DIR)/hipe_amd64_bifs.S \ + $(TTF_DIR)/hipe_literals.h + +$(OBJDIR)/hipe_sparc_glue.o: hipe/hipe_sparc_glue.S \ + $(TTF_DIR)/hipe_sparc_asm.h hipe/hipe_mode_switch.h \ + $(TTF_DIR)/hipe_literals.h +$(TTF_DIR)/hipe_sparc_bifs.S: hipe/hipe_sparc_bifs.m4 hipe/hipe_sparc_asm.m4 \ + hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h +$(OBJDIR)/hipe_sparc_bifs.o: $(TTF_DIR)/hipe_sparc_bifs.S \ + $(TTF_DIR)/hipe_literals.h + +$(OBJDIR)/hipe_ppc_glue.o: hipe/hipe_ppc_glue.S $(TTF_DIR)/hipe_ppc_asm.h \ + hipe/hipe_mode_switch.h $(TTF_DIR)/hipe_literals.h +$(TTF_DIR)/hipe_ppc_bifs.S: hipe/hipe_ppc_bifs.m4 hipe/hipe_ppc_asm.m4 \ + hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h +$(OBJDIR)/hipe_ppc_bifs.o: $(TTF_DIR)/hipe_ppc_bifs.S \ + $(TTF_DIR)/hipe_literals.h + +$(OBJDIR)/hipe_arm_glue.o: hipe/hipe_arm_glue.S $(TTF_DIR)/hipe_arm_asm.h \ + hipe/hipe_mode_switch.h $(TTF_DIR)/hipe_literals.h +$(TTF_DIR)/hipe_arm_bifs.S: hipe/hipe_arm_bifs.m4 hipe/hipe_arm_asm.m4 \ + hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h +$(OBJDIR)/hipe_arm_bifs.o: $(TTF_DIR)/hipe_arm_bifs.S \ + $(TTF_DIR)/hipe_literals.h # end of HiPE section ######################################## @@ -927,13 +977,6 @@ $(BINDIR)/$(EMULATOR_EXECUTABLE): $(INIT_OBJS) $(OBJS) $(DEPLIBS) endif -# -# Create directories -# - -$(CREATE_DIRS): - $(MKDIR) -p $@ - # ---------------------------------------------------------------------- # Dependencies # @@ -945,6 +988,7 @@ $(TARGET)/Makefile: Makefile.in #SED_REPL_WIN_DRIVE=s|\([ ]\)\([A-Za-z]\):|\1/cygdrive/\2|g;s|^\([A-Za-z]\):|/cygdrive/\1|g SED_REPL_O=s|^\([^:]*:\)|$$(OBJDIR)/\1|g +SED_REPL_O_ZLIB=s|^\([^:]*:\)|$$(ZLIB_OBJDIR)/\1|g SED_REPL_TTF_DIR=s|$(TTF_DIR)/|$$(TTF_DIR)/|g SED_REPL_ERL_TOP=s|\([ ]\)$(ERL_TOP)/|\1$$(ERL_TOP)/|g;s|^$(ERL_TOP)/|$$(ERL_TOP)/|g SED_REPL_POLL=s|$$(OBJDIR)/erl_poll.o|$$(OBJDIR)/erl_poll.kp.o $$(OBJDIR)/erl_poll.nkp.o|g @@ -964,6 +1008,7 @@ SED_SUFFIX= endif SED_DEPEND=sed '$(SED_PREFIX)$(SED_REPL_O);$(SED_REPL_TTF_DIR);$(SED_REPL_ERL_TOP)$(SED_SUFFIX)' +SED_DEPEND_ZLIB=sed '$(SED_PREFIX)$(SED_REPL_O_ZLIB)' ifdef HIPE_ENABLED HIPE_SRC=$(wildcard hipe/*.c) @@ -1008,22 +1053,36 @@ DEP_FLAGS=-MM $(MG_FLAG) $(CFLAGS) $(INCLUDES) -Idrivers/common -Idrivers/$(ERLA SYS_SRC=$(ALL_SYS_SRC) endif +.PHONY: depend +ifdef VOID_EMULATOR depend: + @echo $(VOID_EMULATOR)' - omitted target depend' +else +depend: $(TTF_DIR)/depend.mk +$(TTF_DIR)/depend.mk: $(TTF_DIR)/GENERATED $(PRELOAD_SRC) $(DEP_CC) $(DEP_FLAGS) $(BEAM_SRC) \ - | $(SED_DEPEND) > $(TARGET)/depend.mk + | $(SED_DEPEND) > $(TTF_DIR)/depend.mk $(DEP_CC) $(DEP_FLAGS) -DLIBSCTP=$(LIBSCTP) $(DRV_COMMON_SRC) \ - | $(SED_DEPEND) >> $(TARGET)/depend.mk + | $(SED_DEPEND) >> $(TTF_DIR)/depend.mk $(DEP_CC) $(DEP_FLAGS) -I../etc/$(ERLANG_OSTYPE) $(DRV_OSTYPE_SRC) \ - | $(SED_DEPEND) >> $(TARGET)/depend.mk + | $(SED_DEPEND) >> $(TTF_DIR)/depend.mk $(DEP_CC) $(DEP_FLAGS) $(SYS_SRC) \ - | $(SED_DEPEND) >> $(TARGET)/depend.mk + | $(SED_DEPEND) >> $(TTF_DIR)/depend.mk $(DEP_CC) $(DEP_FLAGS) $(TARGET_SRC) \ - | $(SED_DEPEND) >> $(TARGET)/depend.mk + | $(SED_DEPEND) >> $(TTF_DIR)/depend.mk + $(DEP_CC) $(DEP_FLAGS) $(ZLIB_SRC) \ + | $(SED_DEPEND_ZLIB) >> $(TTF_DIR)/depend.mk ifdef HIPE_ENABLED $(DEP_CC) $(DEP_FLAGS) $(HIPE_SRC) \ - | $(SED_DEPEND) >> $(TARGET)/depend.mk + | $(SED_DEPEND) >> $(TTF_DIR)/depend.mk +endif + cd $(ERTS_LIB_DIR) && $(MAKE) depend endif --include $(TARGET)/depend.mk - - +ifneq ($(MAKECMDGOALS),clean) +ifneq ($(MAKECMDGOALS),generate) +ifndef VOID_EMULATOR +-include $(TTF_DIR)/depend.mk +endif +endif +endif diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 71454b3e57..9ce1068b23 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -95,6 +95,7 @@ atom atom atom atom_used atom attributes atom await_proc_exit +atom await_sched_wall_time_modifications atom awaiting_load atom awaiting_unload atom backtrace backtrace_depth @@ -239,6 +240,7 @@ atom generational atom get_seq_token atom get_tcw atom getenv +atom gather_sched_wall_time_result atom getting_linked atom getting_unlinked atom global @@ -246,6 +248,7 @@ atom global_heaps_size atom Gt='>' atom grun atom group_leader +atom have_dt_utag atom heap_block_size atom heap_size atom heap_sizes @@ -554,6 +557,7 @@ atom waiting atom wall_clock atom warning atom warning_msg +atom scheduler_wall_time atom wordsize atom write_concurrency atom xor diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 692fa61fe8..dd13cd179a 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -1329,10 +1329,14 @@ static BpData *get_break(Process *p, BeamInstr *pc, BeamInstr break_op) { } static BpData *is_break(BeamInstr *pc, BeamInstr break_op) { - BpData **rs = (BpData **) pc[-4]; + BpData **rs; BpData *bd = NULL, *ebd = NULL; ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + if (erts_is_native_break(pc)) { + return NULL; + } + rs = (BpData **) pc[-4]; if (! rs) { return NULL; } diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index c65b2be106..8b4f067b98 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -41,6 +41,7 @@ #include "hipe_mode_switch.h" #include "hipe_bif1.h" #endif +#include "dtrace-wrapper.h" /* #define HARDDEBUG 1 */ @@ -1050,6 +1051,101 @@ init_emulator(void) # define REG_tmp_arg2 #endif +#ifdef USE_VM_PROBES +# define USE_VM_CALL_PROBES +#endif + +#ifdef USE_VM_CALL_PROBES + +#define DTRACE_LOCAL_CALL(p, m, f, a) \ + if (DTRACE_ENABLED(local_function_entry)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + int depth = STACK_START(p) - STACK_TOP(p); \ + dtrace_fun_decode(p, m, f, a, \ + process_name, mfa); \ + DTRACE3(local_function_entry, process_name, mfa, depth); \ + } + +#define DTRACE_GLOBAL_CALL(p, m, f, a) \ + if (DTRACE_ENABLED(global_function_entry)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + int depth = STACK_START(p) - STACK_TOP(p); \ + dtrace_fun_decode(p, m, f, a, \ + process_name, mfa); \ + DTRACE3(global_function_entry, process_name, mfa, depth); \ + } + +#define DTRACE_RETURN(p, m, f, a) \ + if (DTRACE_ENABLED(function_return)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + int depth = STACK_START(p) - STACK_TOP(p); \ + dtrace_fun_decode(p, m, f, a, \ + process_name, mfa); \ + DTRACE3(function_return, process_name, mfa, depth); \ + } + +#define DTRACE_BIF_ENTRY(p, m, f, a) \ + if (DTRACE_ENABLED(bif_entry)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, m, f, a, \ + process_name, mfa); \ + DTRACE2(bif_entry, process_name, mfa); \ + } + +#define DTRACE_BIF_RETURN(p, m, f, a) \ + if (DTRACE_ENABLED(bif_return)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, m, f, a, \ + process_name, mfa); \ + DTRACE2(bif_return, process_name, mfa); \ + } + +#define DTRACE_NIF_ENTRY(p, m, f, a) \ + if (DTRACE_ENABLED(nif_entry)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, m, f, a, \ + process_name, mfa); \ + DTRACE2(nif_entry, process_name, mfa); \ + } + +#define DTRACE_NIF_RETURN(p, m, f, a) \ + if (DTRACE_ENABLED(nif_return)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, m, f, a, \ + process_name, mfa); \ + DTRACE2(nif_return, process_name, mfa); \ + } + +#else /* USE_VM_PROBES */ + +#define DTRACE_LOCAL_CALL(p, m, f, a) do {} while (0) +#define DTRACE_GLOBAL_CALL(p, m, f, a) do {} while (0) +#define DTRACE_RETURN(p, m, f, a) do {} while (0) +#define DTRACE_BIF_ENTRY(p, m, f, a) do {} while (0) +#define DTRACE_BIF_RETURN(p, m, f, a) do {} while (0) +#define DTRACE_NIF_ENTRY(p, m, f, a) do {} while (0) +#define DTRACE_NIF_RETURN(p, m, f, a) do {} while (0) + +#endif /* USE_VM_PROBES */ + +#ifdef USE_VM_PROBES +void +dtrace_drvport_str(ErlDrvPort drvport, char *port_buf) +{ + Port *port = erts_drvport2port(drvport); + + erts_snprintf(port_buf, DTRACE_TERM_BUF_SIZE, "#Port<%lu.%lu>", + port_channel_no(port->id), + port_number(port->id)); +} +#endif /* * process_main() is called twice: * The first call performs some initialisation, including exporting @@ -1221,6 +1317,30 @@ void process_main(void) #endif SWAPIN; ASSERT(VALID_INSTR(next)); + +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(process_scheduled)) { + DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(fun_buf, DTRACE_TERM_BUF_SIZE); + dtrace_proc_str(c_p, process_buf); + + if (ERTS_PROC_IS_EXITING(c_p)) { + strcpy(fun_buf, "<exiting>"); + } else { + BeamInstr *fptr = find_function_from_pc(c_p->i); + if (fptr) { + dtrace_fun_decode(c_p, (Eterm)fptr[0], + (Eterm)fptr[1], (Uint)fptr[2], + NULL, fun_buf); + } else { + erts_snprintf(fun_buf, sizeof(fun_buf), + "<unknown/%p>", next); + } + } + + DTRACE2(process_scheduled, process_buf, fun_buf); + } +#endif Goto(next); } @@ -1397,6 +1517,7 @@ void process_main(void) /* FALL THROUGH */ OpCase(i_call_only_f): { SET_I((BeamInstr *) Arg(0)); + DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]); Dispatch(); } @@ -1408,6 +1529,7 @@ void process_main(void) RESTORE_CP(E); E = ADD_BYTE_OFFSET(E, Arg(1)); SET_I((BeamInstr *) Arg(0)); + DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]); Dispatch(); } @@ -1419,6 +1541,7 @@ void process_main(void) OpCase(i_call_f): { SET_CP(c_p, I+2); SET_I((BeamInstr *) Arg(0)); + DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]); Dispatch(); } @@ -1435,6 +1558,12 @@ void process_main(void) * is not loaded, it points to code which will invoke the error handler * (see lb_call_error_handler below). */ +#ifdef USE_VM_CALL_PROBES + if (DTRACE_ENABLED(global_function_entry)) { + BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->address); + DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); + } +#endif Dispatchx(); OpCase(i_move_call_ext_cre): { @@ -1444,6 +1573,12 @@ void process_main(void) /* FALL THROUGH */ OpCase(i_call_ext_e): SET_CP(c_p, I+2); +#ifdef USE_VM_CALL_PROBES + if (DTRACE_ENABLED(global_function_entry)) { + BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->address); + DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); + } +#endif Dispatchx(); OpCase(i_move_call_ext_only_ecr): { @@ -1451,6 +1586,12 @@ void process_main(void) } /* FALL THROUGH */ OpCase(i_call_ext_only_e): +#ifdef USE_VM_CALL_PROBES + if (DTRACE_ENABLED(global_function_entry)) { + BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->address); + DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); + } +#endif Dispatchx(); OpCase(init_y): { @@ -1486,7 +1627,16 @@ void process_main(void) OpCase(return): { +#ifdef USE_VM_CALL_PROBES + BeamInstr* fptr; +#endif SET_I(c_p->cp); + +#ifdef USE_VM_CALL_PROBES + if (DTRACE_ENABLED(function_return) && (fptr = find_function_from_pc(c_p->cp))) { + DTRACE_RETURN(c_p, (Eterm)fptr[0], (Eterm)fptr[1], (Uint)fptr[2]); + } +#endif /* * We must clear the CP to make sure that a stale value do not * create a false module dependcy preventing code upgrading. @@ -1755,6 +1905,7 @@ void process_main(void) * remove it... */ ASSERT(!msgp->data.attached); + /* TODO: Add DTrace probe for this bad message situation? */ UNLINK_MESSAGE(c_p, msgp); free_message(msgp); goto loop_rec__; @@ -1780,24 +1931,88 @@ void process_main(void) save_calls(c_p, &exp_receive); } if (ERL_MESSAGE_TOKEN(msgp) == NIL) { - SEQ_TRACE_TOKEN(c_p) = NIL; +#ifdef USE_VM_PROBES + if (DT_UTAG(c_p) != NIL) { + if (DT_UTAG_FLAGS(c_p) & DT_UTAG_PERMANENT) { + SEQ_TRACE_TOKEN(c_p) = am_have_dt_utag; +#ifdef DTRACE_TAG_HARDDEBUG + if (DT_UTAG_FLAGS(c_p) & DT_UTAG_SPREADING) + erts_fprintf(stderr, + "Dtrace -> (%T) stop spreading " + "tag %T with message %T\r\n", + c_p->id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp)); +#endif + } else { +#ifdef DTRACE_TAG_HARDDEBUG + erts_fprintf(stderr, + "Dtrace -> (%T) kill tag %T with " + "message %T\r\n", + c_p->id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp)); +#endif + DT_UTAG(c_p) = NIL; + SEQ_TRACE_TOKEN(c_p) = NIL; + } + } else { +#endif + SEQ_TRACE_TOKEN(c_p) = NIL; +#ifdef USE_VM_PROBES + } + DT_UTAG_FLAGS(c_p) &= ~DT_UTAG_SPREADING; +#endif } else if (ERL_MESSAGE_TOKEN(msgp) != am_undefined) { Eterm msg; SEQ_TRACE_TOKEN(c_p) = ERL_MESSAGE_TOKEN(msgp); - ASSERT(is_tuple(SEQ_TRACE_TOKEN(c_p))); - ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5); - ASSERT(is_small(SEQ_TRACE_TOKEN_SERIAL(c_p))); - ASSERT(is_small(SEQ_TRACE_TOKEN_LASTCNT(c_p))); - ASSERT(is_small(SEQ_TRACE_TOKEN_FLAGS(c_p))); - ASSERT(is_pid(SEQ_TRACE_TOKEN_SENDER(c_p))); - c_p->seq_trace_lastcnt = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p)); - if (c_p->seq_trace_clock < unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p))) { - c_p->seq_trace_clock = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p)); +#ifdef USE_VM_PROBES + if (ERL_MESSAGE_TOKEN(msgp) == am_have_dt_utag) { + if (DT_UTAG(c_p) == NIL) { + DT_UTAG(c_p) = ERL_MESSAGE_DT_UTAG(msgp); + } + DT_UTAG_FLAGS(c_p) |= DT_UTAG_SPREADING; +#ifdef DTRACE_TAG_HARDDEBUG + erts_fprintf(stderr, + "Dtrace -> (%T) receive tag (%T) " + "with message %T\r\n", + c_p->id, DT_UTAG(c_p), ERL_MESSAGE_TERM(msgp)); +#endif + } else { +#endif + ASSERT(is_tuple(SEQ_TRACE_TOKEN(c_p))); + ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5); + ASSERT(is_small(SEQ_TRACE_TOKEN_SERIAL(c_p))); + ASSERT(is_small(SEQ_TRACE_TOKEN_LASTCNT(c_p))); + ASSERT(is_small(SEQ_TRACE_TOKEN_FLAGS(c_p))); + ASSERT(is_pid(SEQ_TRACE_TOKEN_SENDER(c_p))); + c_p->seq_trace_lastcnt = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p)); + if (c_p->seq_trace_clock < unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p))) { + c_p->seq_trace_clock = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p)); + } + msg = ERL_MESSAGE_TERM(msgp); + seq_trace_output(SEQ_TRACE_TOKEN(c_p), msg, SEQ_TRACE_RECEIVE, + c_p->id, c_p); +#ifdef USE_VM_PROBES } - msg = ERL_MESSAGE_TERM(msgp); - seq_trace_output(SEQ_TRACE_TOKEN(c_p), msg, SEQ_TRACE_RECEIVE, - c_p->id, c_p); +#endif + } +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(message_receive)) { + Eterm token2 = NIL; + DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); + Sint tok_label = 0; + Sint tok_lastcnt = 0; + Sint tok_serial = 0; + + dtrace_proc_str(c_p, receiver_name); + token2 = SEQ_TRACE_TOKEN(c_p); + if (token2 != NIL && token2 != am_have_dt_utag) { + tok_label = signed_val(SEQ_TRACE_T_LABEL(token2)); + tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token2)); + tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token2)); + } + DTRACE6(message_receive, + receiver_name, size_object(ERL_MESSAGE_TERM(msgp)), + c_p->msg.len - 1, tok_label, tok_lastcnt, tok_serial); } +#endif UNLINK_MESSAGE(c_p, msgp); JOIN_MESSAGE(c_p); CANCEL_TIMER(c_p); @@ -3157,6 +3372,7 @@ void process_main(void) */ BifFunction vbf; + DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); c_p->current = I-3; /* current and vbf set to please handle_error */ SWAPOUT; c_p->fcalls = FCALLS - 1; @@ -3178,6 +3394,8 @@ void process_main(void) ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result)); PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + + DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); goto apply_bif_or_nif_epilogue; OpCase(apply_bif): @@ -3197,6 +3415,8 @@ void process_main(void) c_p->arity = 0; /* To allow garbage collection on ourselves * (check_process_code/2). */ + DTRACE_BIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + SWAPOUT; c_p->fcalls = FCALLS - 1; vbf = (BifFunction) Arg(0); @@ -3216,6 +3436,8 @@ void process_main(void) PROCESS_MAIN_CHK_LOCKS(c_p); } + DTRACE_BIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + apply_bif_or_nif_epilogue: ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); ERTS_HOLE_CHECK(c_p); @@ -5899,6 +6121,12 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) save_calls(p, ep); } +#ifdef USE_VM_CALL_PROBES + if (DTRACE_ENABLED(global_function_entry)) { + BeamInstr *fptr = (BeamInstr *) ep->address; + DTRACE_GLOBAL_CALL(p, (Eterm)fptr[-3], (Eterm)fptr[-2], (Uint)fptr[-1]); + } +#endif return ep->address; } @@ -5948,6 +6176,12 @@ fixed_apply(Process* p, Eterm* reg, Uint arity) save_calls(p, ep); } +#ifdef USE_VM_CALL_PROBES + if (DTRACE_ENABLED(global_function_entry)) { + BeamInstr *fptr = (BeamInstr *) ep->address; + DTRACE_GLOBAL_CALL(p, (Eterm)fptr[-3], (Eterm)fptr[-2], (Uint)fptr[-1]); + } +#endif return ep->address; } @@ -5997,6 +6231,15 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re c_p->max_arg_reg = sizeof(c_p->def_arg_reg)/sizeof(c_p->def_arg_reg[0]); } +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(process_hibernate)) { + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); + dtrace_fun_decode(c_p, module, function, arity, + process_name, mfa); + DTRACE2(process_hibernate, process_name, mfa); + } +#endif /* * Arrange for the process to be resumed at the given MFA with * the stack cleared. @@ -6072,6 +6315,9 @@ call_fun(Process* p, /* Current process. */ actual_arity = (int) code_ptr[-1]; if (actual_arity == arity+num_free) { + DTRACE_LOCAL_CALL(p, (Eterm)code_ptr[-3], + (Eterm)code_ptr[-2], + code_ptr[-1]); if (num_free == 0) { return code_ptr; } else { @@ -6089,7 +6335,7 @@ call_fun(Process* p, /* Current process. */ } else { /* * Something wrong here. First build a list of the arguments. - */ + */ if (is_non_value(args)) { Uint sz = 2 * arity; @@ -6164,6 +6410,7 @@ call_fun(Process* p, /* Current process. */ actual_arity = (int) ep->code[2]; if (arity == actual_arity) { + DTRACE_GLOBAL_CALL(p, ep->code[0], ep->code[1], (Uint)ep->code[2]); return ep->address; } else { /* @@ -6239,6 +6486,7 @@ call_fun(Process* p, /* Current process. */ reg[1] = function; reg[2] = args; } + DTRACE_GLOBAL_CALL(p, module, function, arity); return ep->address; } else { badfun: diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 55f4798892..39d4582435 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -43,6 +43,9 @@ static Export* set_cpu_topology_trap = NULL; static Export* await_proc_exit_trap = NULL; Export* erts_format_cpu_topology_trap = NULL; +static Export *await_sched_wall_time_mod_trap; +static erts_smp_atomic32_t sched_wall_time; + #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) /* @@ -560,7 +563,11 @@ erts_queue_monitor_message(Process *p, ref_copy = copy_struct(ref, ref_size, &hp, ohp); tup = TUPLE5(hp, am_DOWN, ref_copy, type, item_copy, reason_copy); - erts_queue_message(p, p_locksp, bp, tup, NIL); + erts_queue_message(p, p_locksp, bp, tup, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); } static BIF_RETTYPE @@ -1941,7 +1948,11 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) { if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) save_calls(p, &exp_send); - if (SEQ_TRACE_TOKEN(p) != NIL) { + if (SEQ_TRACE_TOKEN(p) != NIL +#ifdef USE_VM_PROBES + && SEQ_TRACE_TOKEN(p) != am_have_dt_utag +#endif + ) { seq_trace_update_send(p); seq_trace_output(SEQ_TRACE_TOKEN(p), msg, SEQ_TRACE_SEND, portid, p); @@ -3662,43 +3673,122 @@ BIF_RETTYPE display_nl_0(BIF_ALIST_0) /* ARGSUSED */ BIF_RETTYPE halt_0(BIF_ALIST_0) { - VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt/0\n")); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit(0, ""); - return NIL; /* Pedantic (lint does not know about erl_exit) */ + VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt()\n")); + erl_halt(0); + ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined); } /**********************************************************************/ -#define MSG_SIZE 200 +#define HALT_MSG_SIZE 200 +static char halt_msg[HALT_MSG_SIZE]; /* stop the system with exit code */ /* ARGSUSED */ BIF_RETTYPE halt_1(BIF_ALIST_1) { Sint code; - static char msg[MSG_SIZE]; - int i; if (is_small(BIF_ARG_1) && (code = signed_val(BIF_ARG_1)) >= 0) { - VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%d)\n", code)); + VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1)); + erl_halt((int)(- code)); + ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined); + } + else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) { + VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1)); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit(-code, ""); - } else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) { - if ((i = intlist_to_buf(BIF_ARG_1, msg, MSG_SIZE-1)) < 0) { + erl_exit(ERTS_ABORT_EXIT, ""); + } + else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) { + int i; + + if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE-1)) < 0) { goto error; } - msg[i] = '\0'; - VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%s)\n", msg)); + halt_msg[i] = '\0'; + VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1)); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit(ERTS_DUMP_EXIT, "%s\n", msg); - } else { - error: + erl_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg); + } + else + goto error; + return NIL; /* Pedantic (lint does not know about erl_exit) */ + error: BIF_ERROR(BIF_P, BADARG); +} + +/**********************************************************************/ + +/* stop the system with exit code and flags */ +/* ARGSUSED */ +BIF_RETTYPE halt_2(BIF_ALIST_2) +{ + Sint code; + Eterm optlist = BIF_ARG_2; + int flush = 0; + + for (optlist = BIF_ARG_2; + is_list(optlist); + optlist = CDR(list_val(optlist))) { + Eterm *tp, opt = CAR(list_val(optlist)); + if (is_not_tuple(opt)) + goto error; + tp = tuple_val(opt); + if (tp[0] != make_arityval(2)) + goto error; + if (tp[1] == am_flush) { + if (tp[2] == am_true) + flush = 1; + else if (tp[2] == am_false) + flush = 0; + else + goto error; + } + else + goto error; + } + if (is_not_nil(optlist)) + goto error; + + if (is_small(BIF_ARG_1) && (code = signed_val(BIF_ARG_1)) >= 0) { + VERBOSE(DEBUG_SYSTEM, + ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); + if (flush) { + erl_halt((int)(- code)); + ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined); + } + else { + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erl_exit((int)(- code), ""); + } + } + else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) { + VERBOSE(DEBUG_SYSTEM, + ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erl_exit(ERTS_ABORT_EXIT, ""); } + else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) { + int i; + + if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE-1)) < 0) { + goto error; + } + halt_msg[i] = '\0'; + VERBOSE(DEBUG_SYSTEM, + ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erl_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg); + } + else + goto error; return NIL; /* Pedantic (lint does not know about erl_exit) */ + error: + BIF_ERROR(BIF_P, BADARG); } +/**********************************************************************/ + BIF_RETTYPE function_exported_3(BIF_ALIST_3) { if (is_not_atom(BIF_ARG_1) || @@ -4144,13 +4234,21 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) for (i = 0; i < erts_max_processes; i++) { if (process_tab[i] != (Process*) 0) { Process* p = process_tab[i]; +#ifdef USE_VM_PROBES + p->seq_trace_token = (p->dt_utag != NIL) ? am_have_dt_utag : NIL; +#else p->seq_trace_token = NIL; +#endif p->seq_trace_clock = 0; p->seq_trace_lastcnt = 0; ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); mp = p->msg.first; while(mp != NULL) { +#ifdef USE_VM_PROBES + ERL_MESSAGE_TOKEN(mp) = (ERL_MESSAGE_DT_UTAG(mp) != NIL) ? am_have_dt_utag : NIL; +#else ERL_MESSAGE_TOKEN(mp) = NIL; +#endif mp = mp->next; } } @@ -4160,6 +4258,18 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(am_true); + } else if (BIF_ARG_1 == am_scheduler_wall_time) { + if (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false) { + erts_aint32_t new = BIF_ARG_2 == am_true ? 1 : 0; + erts_aint32_t old = erts_smp_atomic32_xchg_nob(&sched_wall_time, + new); + Eterm ref = erts_sched_wall_time_request(BIF_P, 1, new); + ASSERT(is_value(ref)); + BIF_TRAP2(await_sched_wall_time_mod_trap, + BIF_P, + ref, + old ? am_true : am_false); + } } else if (ERTS_IS_ATOM_STR("scheduling_statistics", BIF_ARG_1)) { int what; if (ERTS_IS_ATOM_STR("disable", BIF_ARG_2)) @@ -4457,6 +4567,9 @@ void erts_init_bif(void) am_format_cpu_topology, 1); await_proc_exit_trap = erts_export_put(am_erlang,am_await_proc_exit,3); + await_sched_wall_time_mod_trap + = erts_export_put(am_erlang, am_await_sched_wall_time_modifications, 2); + erts_smp_atomic32_init_nob(&sched_wall_time, 0); } #ifdef HARDDEBUG @@ -4533,3 +4646,193 @@ BIF_RETTYPE get_module_info_2(BIF_ALIST_2) } BIF_RET(ret); } + +BIF_RETTYPE dt_put_tag_1(BIF_ALIST_1) +{ +#ifdef USE_VM_PROBES + Eterm otag; + if (BIF_ARG_1 == am_undefined) { + otag = (DT_UTAG(BIF_P) == NIL) ? am_undefined : DT_UTAG(BIF_P); + DT_UTAG(BIF_P) = NIL; + DT_UTAG_FLAGS(BIF_P) = 0; + if (SEQ_TRACE_TOKEN(BIF_P) == am_have_dt_utag) { + SEQ_TRACE_TOKEN(BIF_P) = NIL; + } + BIF_RET(otag); + } + if (!is_binary(BIF_ARG_1)) { + BIF_ERROR(BIF_P,BADARG); + } + otag = (DT_UTAG(BIF_P) == NIL) ? am_undefined : DT_UTAG(BIF_P); + DT_UTAG(BIF_P) = BIF_ARG_1; + DT_UTAG_FLAGS(BIF_P) |= DT_UTAG_PERMANENT; + if (SEQ_TRACE_TOKEN(BIF_P) == NIL) { + SEQ_TRACE_TOKEN(BIF_P) = am_have_dt_utag; + } + BIF_RET(otag); +#else + BIF_RET(am_undefined); +#endif +} + +BIF_RETTYPE dt_get_tag_0(BIF_ALIST_0) +{ +#ifdef USE_VM_PROBES + BIF_RET((DT_UTAG(BIF_P) == NIL || !(DT_UTAG_FLAGS(BIF_P) & DT_UTAG_PERMANENT)) ? am_undefined : DT_UTAG(BIF_P)); +#else + BIF_RET(am_undefined); +#endif +} +BIF_RETTYPE dt_get_tag_data_0(BIF_ALIST_0) +{ +#ifdef USE_VM_PROBES + BIF_RET((DT_UTAG(BIF_P) == NIL) ? am_undefined : DT_UTAG(BIF_P)); +#else + BIF_RET(am_undefined); +#endif +} +BIF_RETTYPE dt_prepend_vm_tag_data_1(BIF_ALIST_1) +{ +#ifdef USE_VM_PROBES + Eterm b; + Eterm *hp; + hp = HAlloc(BIF_P,2); + if (is_binary((DT_UTAG(BIF_P)))) { + Uint sz = binary_size(DT_UTAG(BIF_P)); + int i; + unsigned char *p,*q; + byte *temp_alloc = NULL; + b = new_binary(BIF_P,NULL,sz+1); + q = binary_bytes(b); + p = erts_get_aligned_binary_bytes(DT_UTAG(BIF_P),&temp_alloc); + for(i=0;i<sz;++i) { + q[i] = p[i]; + } + erts_free_aligned_binary_bytes(temp_alloc); + q[sz] = '\0'; + } else { + b = new_binary(BIF_P,(byte *)"\0",1); + } + BIF_RET(CONS(hp,b,BIF_ARG_1)); +#else + BIF_RET(BIF_ARG_1); +#endif +} +BIF_RETTYPE dt_append_vm_tag_data_1(BIF_ALIST_1) +{ +#ifdef USE_VM_PROBES + Eterm b; + Eterm *hp; + hp = HAlloc(BIF_P,2); + if (is_binary((DT_UTAG(BIF_P)))) { + Uint sz = binary_size(DT_UTAG(BIF_P)); + int i; + unsigned char *p,*q; + byte *temp_alloc = NULL; + b = new_binary(BIF_P,NULL,sz+1); + q = binary_bytes(b); + p = erts_get_aligned_binary_bytes(DT_UTAG(BIF_P),&temp_alloc); + for(i=0;i<sz;++i) { + q[i] = p[i]; + } + erts_free_aligned_binary_bytes(temp_alloc); + q[sz] = '\0'; + } else { + b = new_binary(BIF_P,(byte *)"\0",1); + } + BIF_RET(CONS(hp,BIF_ARG_1,b)); +#else + BIF_RET(BIF_ARG_1); +#endif +} +BIF_RETTYPE dt_spread_tag_1(BIF_ALIST_1) +{ +#ifdef USE_VM_PROBES + Eterm ret; + Eterm *hp; +#endif + if (BIF_ARG_1 != am_true && BIF_ARG_1 != am_false) { + BIF_ERROR(BIF_P,BADARG); + } +#ifdef USE_VM_PROBES + hp = HAlloc(BIF_P,3); + ret = TUPLE2(hp,make_small(DT_UTAG_FLAGS(BIF_P)),DT_UTAG(BIF_P)); + if (DT_UTAG(BIF_P) != NIL) { + if (BIF_ARG_1 == am_true) { + DT_UTAG_FLAGS(BIF_P) |= DT_UTAG_SPREADING; +#ifdef DTRACE_TAG_HARDDEBUG + erts_fprintf(stderr, + "Dtrace -> (%T) start spreading tag %T\r\n", + BIF_P->id,DT_UTAG(BIF_P)); +#endif + } else { + DT_UTAG_FLAGS(BIF_P) &= ~DT_UTAG_SPREADING; +#ifdef DTRACE_TAG_HARDDEBUG + erts_fprintf(stderr, + "Dtrace -> (%T) stop spreading tag %T\r\n", + BIF_P->id,DT_UTAG(BIF_P)); +#endif + } + } + BIF_RET(ret); +#else + BIF_RET(am_true); +#endif +} +BIF_RETTYPE dt_restore_tag_1(BIF_ALIST_1) +{ +#ifdef USE_VM_PROBES + Eterm *tpl; + Uint x; + if (is_not_tuple(BIF_ARG_1)) { + BIF_ERROR(BIF_P,BADARG); + } + tpl = tuple_val(BIF_ARG_1); + if(arityval(*tpl) != 2 || is_not_small(tpl[1]) || (is_not_binary(tpl[2]) && tpl[2] != NIL)) { + BIF_ERROR(BIF_P,BADARG); + } + if (tpl[2] == NIL) { + if (DT_UTAG(BIF_P) != NIL) { +#ifdef DTRACE_TAG_HARDDEBUG + erts_fprintf(stderr, + "Dtrace -> (%T) restore Killing tag!\r\n", + BIF_P->id); +#endif + } + DT_UTAG(BIF_P) = NIL; + if (SEQ_TRACE_TOKEN(BIF_P) == am_have_dt_utag) { + SEQ_TRACE_TOKEN(BIF_P) = NIL; + } + DT_UTAG_FLAGS(BIF_P) = 0; + } else { + x = unsigned_val(tpl[1]) & (DT_UTAG_SPREADING | DT_UTAG_PERMANENT); +#ifdef DTRACE_TAG_HARDDEBUG + + if (!(x & DT_UTAG_SPREADING) && (DT_UTAG_FLAGS(BIF_P) & + DT_UTAG_SPREADING)) { + erts_fprintf(stderr, + "Dtrace -> (%T) restore stop spreading " + "tag %T\r\n", + BIF_P->id, tpl[2]); + } else if ((x & DT_UTAG_SPREADING) && + !(DT_UTAG_FLAGS(BIF_P) & DT_UTAG_SPREADING)) { + erts_fprintf(stderr, + "Dtrace -> (%T) restore start spreading " + "tag %T\r\n",BIF_P->id,tpl[2]); + } +#endif + DT_UTAG_FLAGS(BIF_P) = x; + DT_UTAG(BIF_P) = tpl[2]; + if (SEQ_TRACE_TOKEN(BIF_P) == NIL) { + SEQ_TRACE_TOKEN(BIF_P) = am_have_dt_utag; + } + } +#else + if (BIF_ARG_1 != am_true) { + BIF_ERROR(BIF_P,BADARG); + } +#endif + BIF_RET(am_true); +} + + diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 8cc568b16c..8a85e102d1 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2011. All Rights Reserved. +# Copyright Ericsson AB 1996-2012. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -115,6 +115,8 @@ bif erlang:halt/0 bif 'erl.lang.system':halt/0 ebif_halt_0 bif erlang:halt/1 bif 'erl.lang.system':halt/1 ebif_halt_1 +bif erlang:halt/2 +bif 'erl.lang.system':halt/2 ebif_halt_2 bif erlang:phash/2 bif erlang:phash2/1 bif erlang:phash2/2 @@ -812,6 +814,23 @@ bif erlang:check_old_code/1 # bif erlang:universaltime_to_posixtime/1 bif erlang:posixtime_to_universaltime/1 + +# +# New in R15B01 +# + +# The dtrace BIF's are always present, but give dummy results if dynamic trace is not enabled in the build +bif erlang:dt_put_tag/1 +bif erlang:dt_get_tag/0 +bif erlang:dt_get_tag_data/0 +bif erlang:dt_spread_tag/1 +bif erlang:dt_restore_tag/1 + +# These are dummies even with enabled dynamic trace unless vm probes are enabled. +# They are also internal, for dtrace tags sent to the VM's own drivers (efile) +bif erlang:dt_prepend_vm_tag_data/1 +bif erlang:dt_append_vm_tag_data/1 + # # Obsolete # diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index 976f05c990..25ac790d81 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -1844,6 +1844,7 @@ dsize_t big_bytes(Eterm x) /* ** Load a bignum from bytes ** xsz is the number of bytes in xp +** *r is untouched if number fits in small */ Eterm bytes_to_big(byte *xp, dsize_t xsz, int xsgn, Eterm *r) { @@ -1852,7 +1853,7 @@ Eterm bytes_to_big(byte *xp, dsize_t xsz, int xsgn, Eterm *r) ErtsDigit d; int i; - while(xsz >= sizeof(ErtsDigit)) { + while(xsz > sizeof(ErtsDigit)) { d = 0; for(i = sizeof(ErtsDigit); --i >= 0;) d = (d << 8) | xp[i]; @@ -1867,11 +1868,20 @@ Eterm bytes_to_big(byte *xp, dsize_t xsz, int xsgn, Eterm *r) d = 0; for(i = xsz; --i >= 0;) d = (d << 8) | xp[i]; + if (++rsz == 1 && IS_USMALL(xsgn,d)) { + if (xsgn) d = -d; + return make_small(d); + } *rwp = d; rwp++; - rsz++; } - return big_norm(r, rsz, (short) xsgn); + if (xsgn) { + *r = make_neg_bignum_header(rsz); + } + else { + *r = make_pos_bignum_header(rsz); + } + return make_big(r); } /* diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 0d3b6a4dba..6f5020dc14 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -646,6 +646,9 @@ bin_check(void) void erl_crash_dump_v(char *file, int line, char* fmt, va_list args) { +#ifdef ERTS_SMP + ErtsThrPrgrData tpd_buf; /* in case we aren't a managed thread... */ +#endif int fd; time_t now; size_t dumpnamebufsize = MAXPATHLEN; @@ -663,7 +666,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) * We do not release system again. We expect an exit() or abort() after * dump has been written. */ - erts_thr_progress_fatal_error_block(60000); + erts_thr_progress_fatal_error_block(60000, &tpd_buf); /* Either worked or not... */ /* Allow us to pass certain places without locking... */ diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 1d968fb147..2c355fadfa 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -30,6 +30,7 @@ #include "big.h" #include "erl_binary.h" #include "erl_bits.h" +#include "dtrace-wrapper.h" #ifdef HYBRID MA_STACK_DECLARE(src); @@ -59,6 +60,14 @@ copy_object(Eterm obj, Process* to) Eterm* hp = HAlloc(to, size); Eterm res; +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(copy_object)) { + DTRACE_CHARBUF(proc_name, 64); + + erts_snprintf(proc_name, sizeof(proc_name), "%T", to->id); + DTRACE2(copy_object, proc_name, size); + } +#endif res = copy_struct(obj, size, &hp, &to->off_heap); #ifdef DEBUG if (eq(obj, res) == 0) { @@ -213,6 +222,8 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) if (IS_CONST(obj)) return obj; + DTRACE1(copy_struct, (int32_t)sz); + hp = htop = *hpp; hbot = htop + sz; hstart = (char *)htop; diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index bee61e7273..802feaeb1c 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -42,6 +42,7 @@ #include "external.h" #include "erl_binary.h" #include "erl_thr_progress.h" +#include "dtrace-wrapper.h" /* Turn this on to get printouts of all distribution messages * which go on the line @@ -381,7 +382,11 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp) Eterm tup; Eterm *hp = erts_alloc_message_heap(3,&bp,&ohp,rp,&rp_locks); tup = TUPLE2(hp, am_nodedown, name); - erts_queue_message(rp, &rp_locks, bp, tup, NIL); + erts_queue_message(rp, &rp_locks, bp, tup, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); } erts_smp_proc_unlock(rp, rp_locks); } @@ -740,19 +745,50 @@ erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message) Eterm token = NIL; Process *sender = dsdp->proc; int res; +#ifdef USE_VM_PROBES + Sint tok_label = 0; + Sint tok_lastcnt = 0; + Sint tok_serial = 0; + Uint msize = 0; + DTRACE_CHARBUF(node_name, 64); + DTRACE_CHARBUF(sender_name, 64); + DTRACE_CHARBUF(receiver_name, 64); +#endif UseTmpHeapNoproc(5); - if (SEQ_TRACE_TOKEN(sender) != NIL) { + if (SEQ_TRACE_TOKEN(sender) != NIL +#ifdef USE_VM_PROBES + && SEQ_TRACE_TOKEN(sender) != am_have_dt_utag +#endif + ) { seq_trace_update_send(sender); token = SEQ_TRACE_TOKEN(sender); seq_trace_output(token, message, SEQ_TRACE_SEND, remote, sender); } +#ifdef USE_VM_PROBES + *node_name = *sender_name = *receiver_name = '\0'; + if (DTRACE_ENABLED(message_send) || DTRACE_ENABLED(message_send_remote)) { + erts_snprintf(node_name, sizeof(node_name), "%T", dsdp->dep->sysname); + erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->id); + erts_snprintf(receiver_name, sizeof(receiver_name), "%T", remote); + msize = size_object(message); + if (token != NIL && token != am_have_dt_utag) { + tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); + tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); + tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); + } + } +#endif if (token != NIL) ctl = TUPLE4(&ctl_heap[0], make_small(DOP_SEND_TT), am_Cookie, remote, token); else ctl = TUPLE3(&ctl_heap[0], make_small(DOP_SEND), am_Cookie, remote); + DTRACE6(message_send, sender_name, receiver_name, + msize, tok_label, tok_lastcnt, tok_serial); + DTRACE7(message_send_remote, sender_name, node_name, receiver_name, + msize, tok_label, tok_lastcnt, tok_serial); res = dsig_send(dsdp, ctl, message, 0); UnUseTmpHeapNoproc(5); return res; @@ -766,13 +802,41 @@ erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message) Eterm token = NIL; Process *sender = dsdp->proc; int res; +#ifdef USE_VM_PROBES + Sint tok_label = 0; + Sint tok_lastcnt = 0; + Sint tok_serial = 0; + Uint32 msize = 0; + DTRACE_CHARBUF(node_name, 64); + DTRACE_CHARBUF(sender_name, 64); + DTRACE_CHARBUF(receiver_name, 128); +#endif UseTmpHeapNoproc(6); - if (SEQ_TRACE_TOKEN(sender) != NIL) { + if (SEQ_TRACE_TOKEN(sender) != NIL +#ifdef USE_VM_PROBES + && SEQ_TRACE_TOKEN(sender) != am_have_dt_utag +#endif + ) { seq_trace_update_send(sender); token = SEQ_TRACE_TOKEN(sender); seq_trace_output(token, message, SEQ_TRACE_SEND, remote_name, sender); } +#ifdef USE_VM_PROBES + *node_name = *sender_name = *receiver_name = '\0'; + if (DTRACE_ENABLED(message_send) || DTRACE_ENABLED(message_send_remote)) { + erts_snprintf(node_name, sizeof(node_name), "%T", dsdp->dep->sysname); + erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->id); + erts_snprintf(receiver_name, sizeof(receiver_name), + "{%T,%s}", remote_name, node_name); + msize = size_object(message); + if (token != NIL && token != am_have_dt_utag) { + tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); + tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); + tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); + } + } +#endif if (token != NIL) ctl = TUPLE5(&ctl_heap[0], make_small(DOP_REG_SEND_TT), @@ -780,6 +844,10 @@ erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message) else ctl = TUPLE4(&ctl_heap[0], make_small(DOP_REG_SEND), sender->id, am_Cookie, remote_name); + DTRACE6(message_send, sender_name, receiver_name, + msize, tok_label, tok_lastcnt, tok_serial); + DTRACE7(message_send_remote, sender_name, node_name, receiver_name, + msize, tok_label, tok_lastcnt, tok_serial); res = dsig_send(dsdp, ctl, message, 0); UnUseTmpHeapNoproc(6); return res; @@ -793,9 +861,23 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, Eterm ctl; DeclareTmpHeapNoproc(ctl_heap,6); int res; +#ifdef USE_VM_PROBES + Process *sender = dsdp->proc; + Sint tok_label = 0; + Sint tok_lastcnt = 0; + Sint tok_serial = 0; + DTRACE_CHARBUF(node_name, 64); + DTRACE_CHARBUF(sender_name, 64); + DTRACE_CHARBUF(remote_name, 128); + DTRACE_CHARBUF(reason_str, 128); +#endif UseTmpHeapNoproc(6); - if (token != NIL) { + if (token != NIL +#ifdef USE_VM_PROBES + && token != am_have_dt_utag +#endif + ) { seq_trace_update_send(dsdp->proc); seq_trace_output_exit(token, reason, SEQ_TRACE_SEND, remote, local); ctl = TUPLE5(&ctl_heap[0], @@ -803,6 +885,23 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, } else { ctl = TUPLE4(&ctl_heap[0], make_small(DOP_EXIT), local, remote, reason); } +#ifdef USE_VM_PROBES + *node_name = *sender_name = *remote_name = '\0'; + if (DTRACE_ENABLED(process_exit_signal_remote)) { + erts_snprintf(node_name, sizeof(node_name), "%T", dsdp->dep->sysname); + erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->id); + erts_snprintf(remote_name, sizeof(remote_name), + "{%T,%s}", remote, node_name); + erts_snprintf(reason_str, sizeof(reason), "%T", reason); + if (token != NIL && token != am_have_dt_utag) { + tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); + tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); + tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); + } + } +#endif + DTRACE7(process_exit_signal_remote, sender_name, node_name, + remote_name, reason_str, tok_label, tok_lastcnt, tok_serial); /* forced, i.e ignore busy */ res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1); UnUseTmpHeapNoproc(6); @@ -1619,6 +1718,18 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) if (!(dep->qflgs & ERTS_DE_QFLG_BUSY)) { if (suspended) resume = 1; /* was busy when we started, but isn't now */ +#ifdef USE_VM_PROBES + if (resume && DTRACE_ENABLED(dist_port_not_busy)) { + DTRACE_CHARBUF(port_str, 64); + DTRACE_CHARBUF(remote_str, 64); + + erts_snprintf(port_str, sizeof(port_str), "%T", cid); + erts_snprintf(remote_str, sizeof(remote_str), + "%T", dep->sysname); + DTRACE3(dist_port_not_busy, erts_this_node_sysname, + port_str, remote_str); + } +#endif } else { /* Enqueue suspended process on dist entry */ @@ -1668,6 +1779,19 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) } if (suspended) { +#ifdef USE_VM_PROBES + if (!resume && DTRACE_ENABLED(dist_port_busy)) { + DTRACE_CHARBUF(port_str, 64); + DTRACE_CHARBUF(remote_str, 64); + DTRACE_CHARBUF(pid_str, 16); + + erts_snprintf(port_str, sizeof(port_str), "%T", cid); + erts_snprintf(remote_str, sizeof(remote_str), "%T", dep->sysname); + erts_snprintf(pid_str, sizeof(pid_str), "%T", c_p->id); + DTRACE4(dist_port_busy, erts_this_node_sysname, + port_str, remote_str, pid_str); + } +#endif if (!resume && erts_system_monitor_flags.busy_dist_port) monitor_generic(c_p, am_busy_dist_port, cid); return ERTS_DSIG_SEND_YIELD; @@ -1691,6 +1815,18 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf) "(%beu bytes) passed.\n", size); +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(dist_output)) { + DTRACE_CHARBUF(port_str, 64); + DTRACE_CHARBUF(remote_str, 64); + + erts_snprintf(port_str, sizeof(port_str), "%T", prt->id); + erts_snprintf(remote_str, sizeof(remote_str), + "%T", prt->dist_entry->sysname); + DTRACE4(dist_output, erts_this_node_sysname, port_str, + remote_str, size); + } +#endif prt->caller = NIL; fpe_was_unmasked = erts_block_fpe(); (*prt->drv_ptr->output)((ErlDrvData) prt->drv_data, @@ -1733,6 +1869,18 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) ASSERT(prt->drv_ptr->outputv); +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(dist_outputv)) { + DTRACE_CHARBUF(port_str, 64); + DTRACE_CHARBUF(remote_str, 64); + + erts_snprintf(port_str, sizeof(port_str), "%T", prt->id); + erts_snprintf(remote_str, sizeof(remote_str), + "%T", prt->dist_entry->sysname); + DTRACE4(dist_outputv, erts_this_node_sysname, port_str, + remote_str, size); + } +#endif prt->caller = NIL; fpe_was_unmasked = erts_block_fpe(); (*prt->drv_ptr->outputv)((ErlDrvData) prt->drv_data, &eiov); @@ -2052,6 +2200,18 @@ erts_dist_command(Port *prt, int reds_limit) void erts_dist_port_not_busy(Port *prt) { +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(dist_port_not_busy)) { + DTRACE_CHARBUF(port_str, 64); + DTRACE_CHARBUF(remote_str, 64); + + erts_snprintf(port_str, sizeof(port_str), "%T", prt->id); + erts_snprintf(remote_str, sizeof(remote_str), + "%T", prt->dist_entry->sysname); + DTRACE3(dist_port_not_busy, erts_this_node_sysname, + port_str, remote_str); + } +#endif erts_schedule_dist_command(prt, NULL); } @@ -2972,7 +3132,11 @@ send_nodes_mon_msg(Process *rp, } ASSERT(hend == hp); - erts_queue_message(rp, rp_locksp, bp, msg, NIL); + erts_queue_message(rp, rp_locksp, bp, msg, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); } static void @@ -2985,6 +3149,21 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas ASSERT(is_immed(what)); ASSERT(is_immed(node)); ASSERT(is_immed(type)); +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(dist_monitor)) { + DTRACE_CHARBUF(what_str, 12); + DTRACE_CHARBUF(node_str, 64); + DTRACE_CHARBUF(type_str, 12); + DTRACE_CHARBUF(reason_str, 64); + + erts_snprintf(what_str, sizeof(what_str), "%T", what); + erts_snprintf(node_str, sizeof(node_str), "%T", node); + erts_snprintf(type_str, sizeof(type_str), "%T", type); + erts_snprintf(reason_str, sizeof(reason_str), "%T", reason); + DTRACE5(dist_monitor, erts_this_node_sysname, + what_str, node_str, type_str, reason_str); + } +#endif ERTS_SMP_LC_ASSERT(!c_p || (erts_proc_lc_my_proc_locks(c_p) diff --git a/erts/emulator/beam/dtrace-wrapper.h b/erts/emulator/beam/dtrace-wrapper.h new file mode 100644 index 0000000000..9d1e55fc43 --- /dev/null +++ b/erts/emulator/beam/dtrace-wrapper.h @@ -0,0 +1,109 @@ +/* + * %CopyrightBegin% + * + * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011. + * All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +#ifndef __DTRACE_WRAPPER_H +#define __DTRACE_WRAPPER_H + +#define DTRACE_TERM_BUF_SIZE 256 + +/* + * Some varieties of SystemTap macros do not like statically-sized + * char[N] buffers. (For example, CentOS 6's macros.) + * So, we'll play a game to humor them. + * + * The code necessary to play nice with CentOS 6's SystemTap looks + * stupid to a C programmer's eyes, so we hide the ugliness with this + * macro, which expands: + * + * DTRACE_CHARBUF(proc_name, 64); + * + * to become: + * + * char proc_name_BUFFER[64], *proc_name = proc_name_BUFFER; + */ + +#define DTRACE_CHARBUF(name, size) \ + char name##_BUFFER[size], *name = name##_BUFFER + +#if defined(USE_DYNAMIC_TRACE) && defined(USE_VM_PROBES) + +#include "erlang_dtrace.h" + +#define DTRACE_ENABLED(name) \ + erlang_##name##_enabled() +#define DTRACE0(name) \ + erlang_##name() +#define DTRACE1(name, a0) \ + erlang_##name(a0) +#define DTRACE2(name, a0, a1) \ + erlang_##name((a0), (a1)) +#define DTRACE3(name, a0, a1, a2) \ + erlang_##name((a0), (a1), (a2)) +#define DTRACE4(name, a0, a1, a2, a3) \ + erlang_##name((a0), (a1), (a2), (a3)) +#define DTRACE5(name, a0, a1, a2, a3, a4) \ + erlang_##name((a0), (a1), (a2), (a3), (a4)) +#define DTRACE6(name, a0, a1, a2, a3, a4, a5) \ + erlang_##name((a0), (a1), (a2), (a3), (a4), (a5)) +#define DTRACE7(name, a0, a1, a2, a3, a4, a5, a6) \ + erlang_##name((a0), (a1), (a2), (a3), (a4), (a5), (a6)) +#define DTRACE10(name, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) \ + erlang_##name((a0), (a1), (a2), (a3), (a4), (a5), (a6), (a7), (a8), (a9)) +#define DTRACE11(name, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10) \ + erlang_##name((a0), (a1), (a2), (a3), (a4), (a5), (a6), (a7), (a8), (a9), (a10)) + +#if defined(_SDT_PROBE) && !defined(STAP_PROBE11) +/* SLF: This is Ubuntu 11-style SystemTap hackery */ +/* work arround for missing STAP macro */ +#define STAP_PROBE11(provider,name,arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10,arg11) \ + _SDT_PROBE(provider, name, 11, \ + (arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10,arg11)) +#define _SDT_ASM_OPERANDS_11(arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10,arg11) \ + _SDT_ASM_OPERANDS_10(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9,arg10), \ + _SDT_ARG(11, arg11) +#endif + +#ifdef STAP_PROBE_ADDR +/* SLF: This is CentOS 5-style SystemTap hackery */ +/* SystemTap compat mode cannot support 11 args. We'll ignore the 11th */ +#define STAP_PROBE11(provider,probe,parm1,parm2,parm3,parm4,parm5,parm6,parm7,parm8,parm9,parm10,parm11) \ + STAP_PROBE10(provider,probe,(parm1),(parm2),(parm3),(parm4),(parm5),(parm6),(parm7),(parm8),(parm9),(parm10)) +#endif /* STAP_PROBE_ADDR */ + +#else /* USE_DYNAMIC_TRACE && USE_VM_PROBES */ + +/* Render all macros to do nothing */ +#define DTRACE_ENABLED(name) 0 +#define DTRACE0(name) do {} while (0) +#define DTRACE1(name, a0) do {} while (0) +#define DTRACE2(name, a0, a1) do {} while (0) +#define DTRACE3(name, a0, a1, a2) do {} while (0) +#define DTRACE4(name, a0, a1, a2, a3) do {} while (0) +#define DTRACE5(name, a0, a1, a2, a3, a4) do {} while (0) +#define DTRACE6(name, a0, a1, a2, a3, a4, a5) do {} while (0) +#define DTRACE7(name, a0, a1, a2, a3, a4, a5, a6) do {} while (0) +#define DTRACE10(name, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) \ + do {} while (0) +#define DTRACE11(name, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10) \ + do {} while (0) + +#endif /* USE_DYNAMIC_TRACE && USE_VM_PROBES */ + +#endif /* __DTRACE_WRAPPER_H */ diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 4d02a67d54..8130d5c576 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2011. All Rights Reserved. + * Copyright Ericsson AB 2002-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -1577,9 +1577,11 @@ erts_alloc_register_scheduler(void *vesdp) } } +#ifdef ERTS_SMP void erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp, int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, int *more_work) { ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp; @@ -1599,10 +1601,12 @@ erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp, erts_alcu_check_delayed_dealloc(allctr, 1, need_thr_progress, + thr_prgr_p, more_work); } } } +#endif erts_aint32_t erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs) @@ -2997,7 +3001,11 @@ reply_alloc_info(void *vair) HRelease(rp, hp_end, hp); } - erts_queue_message(rp, &rp_locks, bp, msg, NIL); + erts_queue_message(rp, &rp_locks, bp, msg, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); if (air->req_sched == sched_id) rp_locks &= ~ERTS_PROC_LOCK_MAIN; diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index 991061c48e..e475f9d8a2 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2011. All Rights Reserved. + * Copyright Ericsson AB 2002-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -21,6 +21,10 @@ #define ERL_ALLOC_H__ #include "erl_alloc_types.h" +#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY +#define ERL_THR_PROGRESS_TSD_TYPE_ONLY +#include "erl_thr_progress.h" +#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY #include "erl_alloc_util.h" #ifdef USE_THREADS #include "erl_threads.h" @@ -132,9 +136,12 @@ typedef struct { extern ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1]; void erts_alloc_register_scheduler(void *vesdp); +#ifdef ERTS_SMP void erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp, int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, int *more_work); +#endif erts_aint32_t erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs); __decl_noreturn void erts_alloc_enomem(ErtsAlcType_t,Uint) diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 962db8b831..90a6c0cbee 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -367,6 +367,7 @@ type EXPORT LONG_LIVED_LOW CODE export_entry type MONITOR_SH STANDARD_LOW PROCESSES monitor_sh type NLINK_SH STANDARD_LOW PROCESSES nlink_sh type AINFO_REQ STANDARD_LOW SYSTEM alloc_info_request +type SCHED_WTIME_REQ STANDARD_LOW SYSTEM sched_wall_time_request +else # "fullword" @@ -383,6 +384,7 @@ type EXPORT LONG_LIVED CODE export_entry type MONITOR_SH FIXED_SIZE PROCESSES monitor_sh type NLINK_SH FIXED_SIZE PROCESSES nlink_sh type AINFO_REQ SHORT_LIVED SYSTEM alloc_info_request +type SCHED_WTIME_REQ SHORT_LIVED SYSTEM sched_wall_time_request +endif diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index c32938bdff..e0d525bdde 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2011. All Rights Reserved. + * Copyright Ericsson AB 2002-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -949,7 +949,6 @@ ddq_check_incoming(ErtsAllctrDDQueue_t *ddq) ERTS_THR_MEMORY_BARRIER; else { ddq->head.next.unref_end = (ErtsAllctrDDBlock_t *) ilast; - ERTS_THR_MEMORY_BARRIER; ddq->head.next.thr_progress = erts_thr_progress_later(); erts_atomic32_set_relb(&ddq->tail.data.um_refc_ix, um_refc_ix); @@ -961,12 +960,24 @@ ddq_check_incoming(ErtsAllctrDDQueue_t *ddq) return 1; } +static ERTS_INLINE void +store_earliest_thr_prgr(ErtsThrPrgrVal *prev_val, ErtsAllctrDDQueue_t *ddq) +{ + if (!ddq->head.next.thr_progress_reached + && (*prev_val == ERTS_THR_PRGR_INVALID + || erts_thr_progress_cmp(ddq->head.next.thr_progress, + *prev_val) < 0)) { + *prev_val = ddq->head.next.thr_progress; + } +} + static ERTS_INLINE int handle_delayed_dealloc(Allctr_t *allctr, int allctr_locked, int use_limit, int ops_limit, int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, int *need_more_work) { int need_thr_prgr = 0; @@ -1008,8 +1019,12 @@ handle_delayed_dealloc(Allctr_t *allctr, if (have_checked_incoming) break; need_thr_prgr = ddq_check_incoming(ddq); - if (need_thr_progress) + if (need_thr_progress) { *need_thr_progress |= need_thr_prgr; + if (need_thr_prgr) + store_earliest_thr_prgr(thr_prgr_p, ddq); + + } have_checked_incoming = 1; goto dequeue; } @@ -1067,6 +1082,8 @@ handle_delayed_dealloc(Allctr_t *allctr, if (need_thr_progress && !(need_thr_prgr | need_mr_wrk)) { need_thr_prgr = ddq_check_incoming(ddq); *need_thr_progress |= need_thr_prgr; + if (need_thr_prgr) + store_earliest_thr_prgr(thr_prgr_p, ddq); } if (allctr->thread_safe && !allctr_locked) @@ -1086,25 +1103,27 @@ enqueue_dealloc_other_instance(ErtsAlcType_t type, Allctr_t *allctr, void *ptr) #endif +#ifdef ERTS_SMP void erts_alcu_check_delayed_dealloc(Allctr_t *allctr, int limit, int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, int *more_work) { -#ifdef ERTS_SMP handle_delayed_dealloc(allctr, 0, limit, ERTS_ALCU_DD_OPS_LIM_HIGH, need_thr_progress, + thr_prgr_p, more_work); -#endif } +#endif #define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \ handle_delayed_dealloc((Allctr), (Locked), 1, \ - ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL) + ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL, NULL) /* Multi block carrier alloc/realloc/free ... */ @@ -3956,7 +3975,8 @@ realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size, if (used_allctr->thread_safe && (!force_move || used_allctr != pref_allctr)) erts_mtx_lock(&used_allctr->mutex); - ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&used_allctr->mutex)); + ERTS_SMP_LC_ASSERT(!used_allctr->thread_safe || + erts_lc_mtx_is_locked(&used_allctr->mutex)); cpy_size = BLK_SZ(blk); if (used_allctr->thread_safe && (!force_move || used_allctr != pref_allctr)) diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index fc1eddb116..cedf4ccf85 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2011. All Rights Reserved. + * Copyright Ericsson AB 2002-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -170,7 +170,9 @@ Eterm erts_alcu_info(Allctr_t *, int, int *, void *, Uint **, Uint *); void erts_alcu_init(AlcUInit_t *); void erts_alcu_current_size(Allctr_t *, AllctrSize_t *, ErtsAlcUFixInfo_t *, int); -void erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, int *); +#ifdef ERTS_SMP +void erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, ErtsThrPrgrVal *, int *); +#endif erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); #endif diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index 8bca9ae582..f0e98b33a5 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -26,6 +26,7 @@ #include "erl_threads.h" #include "erl_thr_queue.h" #include "erl_async.h" +#include "dtrace-wrapper.h" #define ERTS_MAX_ASYNC_READY_CALLS_IN_SEQ 20 @@ -121,6 +122,14 @@ typedef struct { #endif } ErtsAsyncData; +/* + * Some compilers, e.g. GCC 4.2.1 and -O3, will optimize away DTrace + * calls if they're the last thing in the function. :-( + * Many thanks to Trond Norbye, via: + * https://github.com/memcached/memcached/commit/6298b3978687530bc9d219b6ac707a1b681b2a46 + */ +static unsigned gcc_optimizer_hack = 0; + int erts_async_max_threads; /* Initialized by erl_init.c */ int erts_async_thread_suggested_stack_size; /* Initialized by erl_init.c */ @@ -244,6 +253,8 @@ erts_get_async_ready_queue(Uint sched_id) static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q) { + int len; + if (is_internal_port(a->port)) { #if ERTS_USE_ASYNC_READY_Q ErtsAsyncReadyQ *arq = async_ready_q(a->sched_id); @@ -259,6 +270,17 @@ static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q) #endif erts_thr_q_enqueue(&q->thr_q, a); +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(aio_pool_add)) { + DTRACE_CHARBUF(port_str, 16); + + erts_snprintf(port_str, sizeof(port_str), "%T", a->port); + /* DTRACE TODO: Get the queue length from erts_thr_q_enqueue() ? */ + len = -1; + DTRACE2(aio_pool_add, port_str, len); + } +#endif + gcc_optimizer_hack++; } static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, @@ -269,6 +291,7 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, int saved_fin_deq = 0; ErtsThrQFinDeQ_t fin_deq; #endif + int len; while (1) { ErtsAsync *a = (ErtsAsync *) erts_thr_q_dequeue(q); @@ -280,7 +303,16 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, if (saved_fin_deq) erts_thr_q_append_finalize_dequeue_data(&a->q.fin_deq, &fin_deq); #endif +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(aio_pool_get)) { + DTRACE_CHARBUF(port_str, 16); + erts_snprintf(port_str, sizeof(port_str), "%T", a->port); + /* DTRACE TODO: Get the length from erts_thr_q_dequeue() ? */ + len = -1; + DTRACE2(aio_pool_get, port_str, len); + } +#endif return a; } diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index 37d540b41b..bcfdacb91c 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -45,6 +45,7 @@ #include "big.h" #include "dist.h" #include "erl_version.h" +#include "dtrace-wrapper.h" #ifdef ERTS_SMP #define DDLL_SMP 1 @@ -1647,6 +1648,7 @@ static int do_unload_driver_entry(DE_Handle *dh, Eterm *save_name) diver_list lock here!*/ if (q->finish) { int fpe_was_unmasked = erts_block_fpe(); + DTRACE1(driver_finish, q->name); (*(q->finish))(); erts_unblock_fpe(fpe_was_unmasked); } @@ -1760,7 +1762,11 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type, hp += REF_THING_SIZE; mess = TUPLE5(hp,type,r,am_driver,driver_name,tag); } - erts_queue_message(proc, &rp_locks, bp, mess, am_undefined); + erts_queue_message(proc, &rp_locks, bp, mess, am_undefined +#ifdef USE_VM_PROBES + , NIL +#endif + ); erts_smp_proc_unlock(proc, rp_locks); ERTS_SMP_CHK_NO_PROC_LOCKS; } diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 5a806777fe..041eac240d 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -57,6 +57,8 @@ static Export* alloc_info_trap = NULL; static Export* alloc_sizes_trap = NULL; +static Export *gather_sched_wall_time_res_trap; + #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) /* Keep erts_system_version as a global variable for easy access from a core */ @@ -113,6 +115,12 @@ static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE #ifdef VALGRIND " [valgrind-compiled]" #endif +#ifdef USE_DTRACE + " [dtrace]" +#endif +#ifdef USE_SYSTEMTAP + " [systemtap]" +#endif "\n"); #define ASIZE(a) (sizeof(a)/sizeof(a[0])) @@ -2718,6 +2726,24 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) #endif BIF_RET(am_true); } + else if (ERTS_IS_ATOM_STR("dynamic_trace", BIF_ARG_1)) { +#if defined(USE_DTRACE) + DECL_AM(dtrace); + BIF_RET(AM_dtrace); +#elif defined(USE_SYSTEMTAP) + DECL_AM(systemtap); + BIF_RET(AM_systemtap); +#else + BIF_RET(am_none); +#endif + } + else if (ERTS_IS_ATOM_STR("dynamic_trace_probes", BIF_ARG_1)) { +#if defined(USE_VM_PROBES) + BIF_RET(am_true); +#else + BIF_RET(am_false); +#endif + } #ifdef ERTS_SMP else if (ERTS_IS_ATOM_STR("thread_progress", BIF_ARG_1)) { erts_thr_progress_dbg_print_state(); @@ -3180,7 +3206,12 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) Eterm res; Eterm* hp; - if (BIF_ARG_1 == am_context_switches) { + if (BIF_ARG_1 == am_scheduler_wall_time) { + res = erts_sched_wall_time_request(BIF_P, 0, 0); + if (is_non_value(res)) + BIF_RET(am_undefined); + BIF_TRAP1(gather_sched_wall_time_res_trap, BIF_P, res); + } else if (BIF_ARG_1 == am_context_switches) { Eterm cs = erts_make_integer(erts_get_total_context_switches(), BIF_P); hp = HAlloc(BIF_P, 3); res = TUPLE2(hp, cs, SMALL_ZERO); @@ -4160,6 +4191,8 @@ erts_bif_info_init(void) alloc_info_trap = erts_export_put(am_erlang, am_alloc_info, 1); alloc_sizes_trap = erts_export_put(am_erlang, am_alloc_sizes, 1); + gather_sched_wall_time_res_trap + = erts_export_put(am_erlang, am_gather_sched_wall_time_result, 1); process_info_init(); os_info_init(); } diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index cd423eb200..3056319809 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -40,6 +40,7 @@ #include "external.h" #include "packet_parser.h" #include "erl_bits.h" +#include "dtrace-wrapper.h" static int open_port(Process* p, Eterm name, Eterm settings, int *err_nump); static byte* convert_environment(Process* p, Eterm env); @@ -343,6 +344,16 @@ port_call(Process* c_p, Eterm arg1, Eterm arg2, Eterm arg3) __FILE__, __LINE__, endp - (bytes + size)); } erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(driver_call)) { + DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); + + dtrace_pid_str(p->connected, process_str); + dtrace_port_str(p, port_str); + DTRACE5(driver_call, process_str, port_str, p->name, op, real_size); + } +#endif prc = (char *) port_resp; fpe_was_unmasked = erts_block_fpe(); ret = drv->call((ErlDrvData)p->drv_data, @@ -539,6 +550,18 @@ BIF_RETTYPE port_connect_2(BIF_ALIST_2) prt->connected = pid; /* internal pid */ erts_smp_port_unlock(prt); +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(port_connect)) { + DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(newprocess_str, DTRACE_TERM_BUF_SIZE); + + dtrace_pid_str(prt->connected, process_str); + erts_snprintf(port_str, sizeof(port_str), "%T", prt->id); + dtrace_proc_str(rp, newprocess_str); + DTRACE4(port_connect, process_str, port_str, prt->name, newprocess_str); + } +#endif BIF_RET(am_true); } @@ -904,7 +927,16 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); port_num = erts_open_driver(driver, p->id, name_buf, &opts, err_nump); +#ifdef USE_VM_PROBES + if (port_num >= 0 && DTRACE_ENABLED(port_open)) { + DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); + dtrace_proc_str(p, process_str); + erts_snprintf(port_str, sizeof(port_str), "%T", erts_port[port_num].id); + DTRACE3(port_open, process_str, name_buf, port_str); + } +#endif erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); if (port_num < 0) { diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c index a922a33da3..0002f8374f 100644 --- a/erts/emulator/beam/erl_bif_timer.c +++ b/erts/emulator/beam/erl_bif_timer.c @@ -373,7 +373,11 @@ bif_timer_timeout(ErtsBifTimer* btm) message = TUPLE3(hp, am_timeout, ref, message); } - erts_queue_message(rp, &rp_locks, bp, message, NIL); + erts_queue_message(rp, &rp_locks, bp, message, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); erts_smp_proc_unlock(rp, rp_locks); erts_smp_proc_dec_refc(rp); } diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index b0a58c80ea..1ef4b07c24 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -1744,9 +1744,17 @@ Eterm erts_seq_trace(Process *p, Eterm arg1, Eterm arg2, return THE_NON_VALUE; } if (build_result) { +#ifdef USE_VM_PROBES + old_value = (SEQ_TRACE_TOKEN(p) == am_have_dt_utag) ? NIL : SEQ_TRACE_TOKEN(p); +#else old_value = SEQ_TRACE_TOKEN(p); +#endif } +#ifdef USE_VM_PROBES + SEQ_TRACE_TOKEN(p) = (DT_UTAG(p) != NIL) ? am_have_dt_utag : NIL; +#else SEQ_TRACE_TOKEN(p) = NIL; +#endif return old_value; } else { @@ -1759,7 +1767,11 @@ new_seq_trace_token(Process* p) { Eterm* hp; - if (SEQ_TRACE_TOKEN(p) == NIL) { + if (SEQ_TRACE_TOKEN(p) == NIL +#ifdef USE_VM_PROBES + || SEQ_TRACE_TOKEN(p) == am_have_dt_utag +#endif + ) { hp = HAlloc(p, 6); SEQ_TRACE_TOKEN(p) = TUPLE5(hp, make_small(0), /* Flags */ make_small(0), /* Label */ @@ -1779,7 +1791,11 @@ BIF_RETTYPE erl_seq_trace_info(Process *p, Eterm item) BIF_ERROR(p, BADARG); } - if (SEQ_TRACE_TOKEN(p) == NIL) { + if (SEQ_TRACE_TOKEN(p) == NIL +#ifdef USE_VM_PROBES + || SEQ_TRACE_TOKEN(p) == am_have_dt_utag +#endif + ) { if ((item == am_send) || (item == am_receive) || (item == am_print) || (item == am_timestamp)) { hp = HAlloc(p,3); @@ -1836,8 +1852,13 @@ BIF_RETTYPE seq_trace_info_1(BIF_ALIST_1) */ BIF_RETTYPE seq_trace_print_1(BIF_ALIST_1) { - if (SEQ_TRACE_TOKEN(BIF_P) == NIL) + if (SEQ_TRACE_TOKEN(BIF_P) == NIL +#ifdef USE_VM_PROBES + || SEQ_TRACE_TOKEN(BIF_P) == am_have_dt_utag +#endif + ) { BIF_RET(am_false); + } seq_trace_update_send(BIF_P); seq_trace_output(SEQ_TRACE_TOKEN(BIF_P), BIF_ARG_1, SEQ_TRACE_PRINT, NIL, BIF_P); @@ -1854,8 +1875,13 @@ BIF_RETTYPE seq_trace_print_1(BIF_ALIST_1) */ BIF_RETTYPE seq_trace_print_2(BIF_ALIST_2) { - if (SEQ_TRACE_TOKEN(BIF_P) == NIL) + if (SEQ_TRACE_TOKEN(BIF_P) == NIL +#ifdef USE_VM_PROBES + || SEQ_TRACE_TOKEN(BIF_P) == am_have_dt_utag +#endif + ) { BIF_RET(am_false); + } if (!(is_atom(BIF_ARG_1) || is_small(BIF_ARG_1))) { BIF_ERROR(BIF_P, BADARG); } diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index eb89baf1c9..51bdf53823 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -251,7 +251,7 @@ free_dbtable(DbTable* tb) ASSERT(is_immed(tb->common.heir_data)); erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable)); ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable)); - ERTS_THR_MEMORY_BARRIER; + ERTS_SMP_MEMORY_BARRIER; } #ifdef ERTS_SMP @@ -3816,7 +3816,7 @@ void db_info(int to, void *to_arg, int show) /* Called by break handler */ Uint erts_get_ets_misc_mem_size(void) { - ERTS_THR_MEMORY_BARRIER; + ERTS_SMP_MEMORY_BARRIER; /* Memory not allocated in ets_alloc */ return (Uint) erts_smp_atomic_read_nob(&erts_ets_misc_mem_size); } diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 038a667b06..c726be5fb4 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2011. All Rights Reserved. + * Copyright Ericsson AB 1998-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -105,11 +105,7 @@ #define NSEG_2 256 /* Size of second segment table */ #define NSEG_INC 128 /* Number of segments to grow after that */ -#ifdef ETHR_ORDERED_READ_DEPEND -#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_nob(&(tb)->segtab)) -#else -#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_rb(&(tb)->segtab)) -#endif +#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_ddrb(&(tb)->segtab)) #define NACTIVE(tb) ((int)erts_smp_atomic_read_nob(&(tb)->nactive)) #define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems)) diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 4821a7d9fb..be345e7c9b 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -2203,7 +2203,11 @@ restart: *esp++ = am_true; break; case matchIsSeqTrace: - if (SEQ_TRACE_TOKEN(c_p) != NIL) + if (SEQ_TRACE_TOKEN(c_p) != NIL +#ifdef USE_VM_PROBES + && SEQ_TRACE_TOKEN(c_p) != am_have_dt_utag +#endif + ) *esp++ = am_true; else *esp++ = am_false; @@ -2227,7 +2231,11 @@ restart: --esp; break; case matchGetSeqToken: - if (SEQ_TRACE_TOKEN(c_p) == NIL) + if (SEQ_TRACE_TOKEN(c_p) == NIL +#ifdef USE_VM_PROBES + || SEQ_TRACE_TOKEN(c_p) == am_have_dt_utag +#endif + ) *esp++ = NIL; else { Eterm sender = SEQ_TRACE_TOKEN_SENDER(c_p); diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 7510f6b724..1ae9a211d7 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -649,6 +649,8 @@ EXTERN int erl_drv_getenv(char *key, char *value, size_t *value_size); #endif +/* also in global.h, but driver's can't include global.h */ +void dtrace_drvport_str(ErlDrvPort port, char *port_buf); diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index eb2b945877..82f2dc6091 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -35,6 +35,7 @@ #include "hipe_stack.h" #include "hipe_mode_switch.h" #endif +#include "dtrace-wrapper.h" #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20 @@ -349,7 +350,9 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) Uint reclaimed_now = 0; int done = 0; Uint ms1, s1, us1; - +#ifdef USE_VM_PROBES + DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); +#endif if (IS_TRACED_FL(p, F_TRACE_GC)) { trace_gc(p, am_gc_start); } @@ -357,10 +360,11 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); p->gcstatus = p->status; p->status = P_GARBING; + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + if (erts_system_monitor_long_gc != 0) { get_now(&ms1, &s1, &us1); } - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); ERTS_CHK_OFFHEAP(p); @@ -368,15 +372,27 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) if (GEN_GCS(p) >= MAX_GEN_GCS(p)) { FLAGS(p) |= F_NEED_FULLSWEEP; } - +#ifdef USE_VM_PROBES + *pidbuf = '\0'; + if (DTRACE_ENABLED(gc_major_start) + || DTRACE_ENABLED(gc_major_end) + || DTRACE_ENABLED(gc_minor_start) + || DTRACE_ENABLED(gc_minor_end)) { + dtrace_proc_str(p, pidbuf); + } +#endif /* * Test which type of GC to do. */ while (!done) { if ((FLAGS(p) & F_NEED_FULLSWEEP) != 0) { + DTRACE2(gc_major_start, pidbuf, need); done = major_collection(p, need, objv, nobj, &reclaimed_now); + DTRACE2(gc_major_end, pidbuf, reclaimed_now); } else { + DTRACE2(gc_minor_start, pidbuf, need); done = minor_collection(p, need, objv, nobj, &reclaimed_now); + DTRACE2(gc_minor_end, pidbuf, reclaimed_now); } } @@ -910,7 +926,18 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) * XXX: WARNING: If HiPE starts storing other non-Erlang values on the * nstack, such as floats, then this will have to be changed. */ -#define offset_nstack(p,offs,area,area_size) offset_heap_ptr(hipe_nstack_start((p)),hipe_nstack_used((p)),(offs),(area),(area_size)) +static ERTS_INLINE void offset_nstack(Process* p, Sint offs, + char* area, Uint area_size) +{ + if (p->hipe.nstack) { + ASSERT(p->hipe.nsp && p->hipe.nstend); + offset_heap_ptr(hipe_nstack_start(p), hipe_nstack_used(p), + offs, area, area_size); + } + else { + ASSERT(!p->hipe.nsp && !p->hipe.nstend); + } +} #else /* !HIPE */ @@ -1106,6 +1133,15 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) sys_memcpy(n_heap + new_sz - n, p->stop, n * sizeof(Eterm)); p->stop = n_heap + new_sz - n; +#ifdef USE_VM_PROBES + if (HEAP_SIZE(p) != new_sz && DTRACE_ENABLED(process_heap_grow)) { + DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); + + dtrace_proc_str(p, pidbuf); + DTRACE3(process_heap_grow, pidbuf, HEAP_SIZE(p), new_sz); + } +#endif + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, (void*)HEAP_START(p), HEAP_SIZE(p) * sizeof(Eterm)); @@ -1327,6 +1363,15 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) sys_memcpy(n_heap + new_sz - n, p->stop, n * sizeof(Eterm)); p->stop = n_heap + new_sz - n; +#ifdef USE_VM_PROBES + if (HEAP_SIZE(p) != new_sz && DTRACE_ENABLED(process_heap_grow)) { + DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); + + dtrace_proc_str(p, pidbuf); + DTRACE3(process_heap_grow, pidbuf, HEAP_SIZE(p), new_sz); + } +#endif + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, (void *) HEAP_START(p), (HEAP_END(p) - HEAP_START(p)) * sizeof(Eterm)); @@ -1895,7 +1940,13 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) roots[n].sz = 1; n++; } - +#ifdef USE_VM_PROBES + if (is_not_immed(p->dt_utag)) { + roots[n].v = &p->dt_utag; + roots[n].sz = 1; + n++; + } +#endif ASSERT(is_nil(p->tracer_proc) || is_internal_pid(p->tracer_proc) || is_internal_port(p->tracer_proc)); @@ -1997,6 +2048,16 @@ grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj) HEAP_TOP(p) = new_heap + heap_size; HEAP_START(p) = new_heap; } + +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(process_heap_grow)) { + DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); + + dtrace_proc_str(p, pidbuf); + DTRACE3(process_heap_grow, pidbuf, HEAP_SIZE(p), new_sz); + } +#endif + HEAP_SIZE(p) = new_sz; } @@ -2006,7 +2067,6 @@ shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj) Eterm* new_heap; Uint heap_size = HEAP_TOP(p) - HEAP_START(p); Sint offs; - Uint stack_size = p->hend - p->stop; ASSERT(new_sz < p->heap_sz); @@ -2035,6 +2095,16 @@ shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj) HEAP_TOP(p) = new_heap + heap_size; HEAP_START(p) = new_heap; } + +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(process_heap_shrink)) { + DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); + + dtrace_proc_str(p, pidbuf); + DTRACE3(process_heap_shrink, pidbuf, HEAP_SIZE(p), new_sz); + } +#endif + HEAP_SIZE(p) = new_sz; } @@ -2417,6 +2487,13 @@ offset_mqueue(Process *p, Sint offs, char* area, Uint area_size) if (is_boxed(mesg) && in_area(ptr_val(mesg), area, area_size)) { ERL_MESSAGE_TOKEN(mp) = offset_ptr(mesg, offs); } +#ifdef USE_VM_PROBES + mesg = ERL_MESSAGE_DT_UTAG(mp); + if (is_boxed(mesg) && in_area(ptr_val(mesg), area, area_size)) { + ERL_MESSAGE_DT_UTAG(mp) = offset_ptr(mesg, offs); + } +#endif + ASSERT((is_nil(ERL_MESSAGE_TOKEN(mp)) || is_tuple(ERL_MESSAGE_TOKEN(mp)) || is_atom(ERL_MESSAGE_TOKEN(mp)))); @@ -2436,6 +2513,9 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size, offset_heap_ptr(&p->fvalue, 1, offs, area, area_size); offset_heap_ptr(&p->ftrace, 1, offs, area, area_size); offset_heap_ptr(&p->seq_trace_token, 1, offs, area, area_size); +#ifdef USE_VM_PROBES + offset_heap_ptr(&p->dt_utag, 1, offs, area, area_size); +#endif offset_heap_ptr(&p->group_leader, 1, offs, area, area_size); offset_mqueue(p, offs, area, area_size); offset_heap_ptr(p->stop, (STACK_START(p) - p->stop), offs, area, area_size); diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 717315d8bd..ca4385dd3a 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2011. All Rights Reserved. + * Copyright Ericsson AB 1997-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -1510,7 +1510,7 @@ __decl_noreturn void erts_thr_fatal_error(int err, char *what) #endif static void -system_cleanup(int exit_code) +system_cleanup(int flush_async) { /* * Make sure only one thread exits the runtime system. @@ -1542,7 +1542,7 @@ system_cleanup(int exit_code) * (in threaded non smp case). */ - if (exit_code != 0 + if (!flush_async || !erts_initialized #if defined(USE_THREADS) && !defined(ERTS_SMP) || !erts_equal_tids(main_thread, erts_thr_self()) @@ -1585,21 +1585,12 @@ system_cleanup(int exit_code) erts_exit_flush_async(); } -/* - * Common exit function, all exits from the system go through here. - * n <= 0 -> normal exit with status n; - * n = 127 -> Erlang crash dump produced, exit with status 1; - * other positive n -> Erlang crash dump and core dump produced. - */ - -__decl_noreturn void erl_exit0(char *file, int line, int n, char *fmt,...) +static __decl_noreturn void __noreturn +erl_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) { unsigned int an; - va_list args; - va_start(args, fmt); - - system_cleanup(n); + system_cleanup(flush_async); save_statistics(); @@ -1609,66 +1600,42 @@ __decl_noreturn void erl_exit0(char *file, int line, int n, char *fmt,...) erts_mtrace_exit((Uint32) an); /* Produce an Erlang core dump if error */ - if (n > 0 && erts_initialized && - (erts_no_crash_dump == 0 || n == ERTS_DUMP_EXIT)) { - erl_crash_dump_v(file, line, fmt, args); + if (((n > 0 && erts_no_crash_dump == 0) || n == ERTS_DUMP_EXIT) + && erts_initialized) { + erl_crash_dump_v((char*) NULL, 0, fmt, args1); } - /* need to reinitialize va_args thing */ - va_end(args); - va_start(args, fmt); - if (fmt != NULL && *fmt != '\0') - erl_error(fmt, args); /* Print error message. */ - va_end(args); + erl_error(fmt, args2); /* Print error message. */ sys_tty_reset(n); if (n == ERTS_INTR_EXIT) exit(0); - else if (n == 127) + else if (n == ERTS_DUMP_EXIT) ERTS_EXIT_AFTER_DUMP(1); else if (n > 0 || n == ERTS_ABORT_EXIT) abort(); exit(an); } -__decl_noreturn void erl_exit(int n, char *fmt,...) +/* Exit without flushing async threads */ +__decl_noreturn void __noreturn erl_exit(int n, char *fmt, ...) { - unsigned int an; - va_list args; - - va_start(args, fmt); - - system_cleanup(n); - - save_statistics(); - - an = abs(n); - - if (erts_mtrace_enabled) - erts_mtrace_exit((Uint32) an); - - /* Produce an Erlang core dump if error */ - if (n > 0 && erts_initialized && - (erts_no_crash_dump == 0 || n == ERTS_DUMP_EXIT)) { - erl_crash_dump_v((char*) NULL, 0, fmt, args); - } - - /* need to reinitialize va_args thing */ - va_end(args); - va_start(args, fmt); - - if (fmt != NULL && *fmt != '\0') - erl_error(fmt, args); /* Print error message. */ - va_end(args); - sys_tty_reset(n); - - if (n == ERTS_INTR_EXIT) - exit(0); - else if (n == ERTS_DUMP_EXIT) - ERTS_EXIT_AFTER_DUMP(1); - else if (n > 0 || n == ERTS_ABORT_EXIT) - abort(); - exit(an); + va_list args1, args2; + va_start(args1, fmt); + va_start(args2, fmt); + erl_exit_vv(n, 0, fmt, args1, args2); + va_end(args2); + va_end(args1); } +/* Exit after flushing async threads */ +__decl_noreturn void __noreturn erl_exit_flush_async(int n, char *fmt, ...) +{ + va_list args1, args2; + va_start(args1, fmt); + va_start(args2, fmt); + erl_exit_vv(n, 1, fmt, args1, args2); + va_end(args2); + va_end(args1); +} diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 09e85893c3..5eb2a69242 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -183,6 +183,9 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "save_ops_lock", NULL }, #endif #endif +#ifdef USE_VM_PROBES + { "efile_drv dtrace mutex", NULL }, +#endif { "mtrace_buf", NULL }, { "erts_alloc_hard_debug", NULL } }; diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index ab1ab7b1ea..4cdf2d7d09 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -31,6 +31,7 @@ #include "erl_process.h" #include "erl_nmgc.h" #include "erl_binary.h" +#include "dtrace-wrapper.h" ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message, ErlMessage, @@ -335,6 +336,11 @@ erts_queue_dist_message(Process *rcvr, Eterm token) { ErlMessage* mp; +#ifdef USE_VM_PROBES + Sint tok_label = 0; + Sint tok_lastcnt = 0; + Sint tok_serial = 0; +#endif #ifdef ERTS_SMP ErtsProcLocks need_locks; #endif @@ -376,15 +382,61 @@ erts_queue_dist_message(Process *rcvr, message_free(mp); msg = erts_msg_distext2heap(rcvr, rcvr_locks, &mbuf, &token, dist_ext); if (is_value(msg)) - erts_queue_message(rcvr, rcvr_locks, mbuf, msg, token); +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(message_queued)) { + DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); + + dtrace_proc_str(rcvr, receiver_name); + if (token != NIL && token != am_have_dt_utag) { + tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); + tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); + tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); + } + DTRACE6(message_queued, + receiver_name, size_object(msg), rcvr->msg.len, + tok_label, tok_lastcnt, tok_serial); + } +#endif + erts_queue_message(rcvr, rcvr_locks, mbuf, msg, token +#ifdef USE_VM_PROBES + , NIL +#endif + ); } else { /* Enqueue message on external format */ ERL_MESSAGE_TERM(mp) = THE_NON_VALUE; - ERL_MESSAGE_TOKEN(mp) = token; +#ifdef USE_VM_PROBES + ERL_MESSAGE_DT_UTAG(mp) = NIL; + if (token == am_have_dt_utag) { + ERL_MESSAGE_TOKEN(mp) = NIL; + } else { +#endif + ERL_MESSAGE_TOKEN(mp) = token; +#ifdef USE_VM_PROBES + } +#endif mp->next = NULL; +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(message_queued)) { + DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); + + dtrace_proc_str(rcvr, receiver_name); + if (token != NIL && token != am_have_dt_utag) { + tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); + tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); + tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); + } + /* + * TODO: We don't know the real size of the external message here. + * -1 will appear to a D script as 4294967295. + */ + DTRACE6(message_queued, receiver_name, -1, rcvr->msg.len + 1, + tok_label, tok_lastcnt, tok_serial); + } +#endif mp->data.dist_ext = dist_ext; LINK_MESSAGE(rcvr, mp); @@ -398,7 +450,11 @@ erts_queue_message(Process* receiver, ErtsProcLocks *receiver_locks, ErlHeapFragment* bp, Eterm message, - Eterm seq_trace_token) + Eterm seq_trace_token +#ifdef USE_VM_PROBES + , Eterm dt_utag +#endif +) { ErlMessage* mp; #ifdef ERTS_SMP @@ -439,6 +495,9 @@ erts_queue_message(Process* receiver, ERL_MESSAGE_TERM(mp) = message; ERL_MESSAGE_TOKEN(mp) = seq_trace_token; +#ifdef USE_VM_PROBES + ERL_MESSAGE_DT_UTAG(mp) = dt_utag; +#endif mp->next = NULL; mp->data.heap_frag = bp; @@ -462,12 +521,30 @@ erts_queue_message(Process* receiver, LINK_MESSAGE(receiver, mp); #endif +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(message_queued)) { + DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); + Sint tok_label = 0; + Sint tok_lastcnt = 0; + Sint tok_serial = 0; + + dtrace_proc_str(receiver, receiver_name); + if (seq_trace_token != NIL && is_tuple(seq_trace_token)) { + tok_label = signed_val(SEQ_TRACE_T_LABEL(seq_trace_token)); + tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token)); + tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token)); + } + DTRACE6(message_queued, + receiver_name, size_object(message), receiver->msg.len, + tok_label, tok_lastcnt, tok_serial); + } +#endif notify_new_message(receiver); if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) { trace_receive(receiver, message); } - + #ifndef ERTS_SMP ERTS_HOLE_CHECK(receiver); #endif @@ -497,6 +574,9 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) Sint offs; Uint sz; ErlHeapFragment *bp; +#ifdef USE_VM_PROBES + Eterm utag; +#endif #ifdef HARD_DEBUG ProcBin *dbg_mso_start = off_heap->mso; @@ -506,32 +586,56 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) ErlHeapFragment *dbg_bp; Uint *dbg_hp, *dbg_thp_start; Uint dbg_term_sz, dbg_token_sz; +#ifdef USE_VM_PROBES + Eterm dbg_utag; + Uint dbg_utag_sz; +#endif #endif bp = msg->data.heap_frag; term = ERL_MESSAGE_TERM(msg); token = ERL_MESSAGE_TOKEN(msg); +#ifdef USE_VM_PROBES + utag = ERL_MESSAGE_DT_UTAG(msg); +#endif if (!bp) { +#ifdef USE_VM_PROBES + ASSERT(is_immed(term) && is_immed(token) && is_immed(utag)); +#else ASSERT(is_immed(term) && is_immed(token)); +#endif return; } #ifdef HARD_DEBUG dbg_term_sz = size_object(term); dbg_token_sz = size_object(token); + dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz); +#ifdef USE_VM_PROBES + dbg_utag_sz = size_object(utag); + dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz + dbg_utag_sz ); +#endif /*ASSERT(dbg_term_sz + dbg_token_sz == erts_msg_used_frag_sz(msg)); Copied size may be smaller due to removed SubBins's or garbage. Copied size may be larger due to duplicated shared terms. */ - dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz); dbg_hp = dbg_bp->mem; dbg_term = copy_struct(term, dbg_term_sz, &dbg_hp, &dbg_bp->off_heap); dbg_token = copy_struct(token, dbg_token_sz, &dbg_hp, &dbg_bp->off_heap); - dbg_thp_start = *hpp; +#ifdef USE_VM_PROBES + dbg_utag = copy_struct(utag, dbg_utag_sz, &dbg_hp, &dbg_bp->off_heap); +#endif + dbg_thp_start = *hpp; #endif if (bp->next != NULL) { - move_multi_frags(hpp, off_heap, bp, msg->m, 2); + move_multi_frags(hpp, off_heap, bp, msg->m, +#ifdef USE_VM_PROBES + 3 +#else + 2 +#endif + ); goto copy_done; } @@ -633,6 +737,16 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) ASSERT(hp > ptr_val(ERL_MESSAGE_TERM(msg))); #endif } +#ifdef USE_VM_PROBES + if (is_not_immed(utag)) { + ASSERT(in_heapfrag(ptr_val(utag), bp)); + ERL_MESSAGE_DT_UTAG(msg) = offset_ptr(utag, offs); +#ifdef HARD_DEBUG + ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_DT_UTAG(msg))); + ASSERT(hp > ptr_val(ERL_MESSAGE_DT_UTAG(msg))); +#endif + } +#endif copy_done: @@ -699,6 +813,9 @@ copy_done: #ifdef HARD_DEBUG ASSERT(eq(ERL_MESSAGE_TERM(msg), dbg_term)); ASSERT(eq(ERL_MESSAGE_TOKEN(msg), dbg_token)); +#ifdef USE_VM_PROBES + ASSERT(eq(ERL_MESSAGE_DT_UTAG(msg), dbg_utag)); +#endif free_message_buffer(dbg_bp); #endif @@ -774,39 +891,101 @@ erts_send_message(Process* sender, Uint msize; ErlHeapFragment* bp = NULL; Eterm token = NIL; - +#ifdef USE_VM_PROBES + DTRACE_CHARBUF(sender_name, 64); + DTRACE_CHARBUF(receiver_name, 64); + Sint tok_label = 0; + Sint tok_lastcnt = 0; + Sint tok_serial = 0; +#endif BM_STOP_TIMER(system); BM_MESSAGE(message,sender,receiver); BM_START_TIMER(send); + #ifdef USE_VM_PROBES + *sender_name = *receiver_name = '\0'; + if (DTRACE_ENABLED(message_send)) { + erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->id); + erts_snprintf(receiver_name, sizeof(receiver_name), "%T", receiver->id); + } +#endif if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) { Eterm* hp; + Eterm stoken = SEQ_TRACE_TOKEN(sender); + Uint seq_trace_size = 0; +#ifdef USE_VM_PROBES + Uint dt_utag_size = 0; + Eterm utag = NIL; +#endif - BM_SWAP_TIMER(send,size); + BM_SWAP_TIMER(send,size); msize = size_object(message); - BM_SWAP_TIMER(size,send); + BM_SWAP_TIMER(size,send); + +#ifdef USE_VM_PROBES + if (stoken != am_have_dt_utag) { +#endif + + seq_trace_update_send(sender); + seq_trace_output(stoken, message, SEQ_TRACE_SEND, + receiver->id, sender); + seq_trace_size = 6; /* TUPLE5 */ +#ifdef USE_VM_PROBES + } + if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) { + dt_utag_size = size_object(DT_UTAG(sender)); + } else if (stoken == am_have_dt_utag ) { + stoken = NIL; + } +#endif - seq_trace_update_send(sender); - seq_trace_output(SEQ_TRACE_TOKEN(sender), message, SEQ_TRACE_SEND, - receiver->id, sender); - bp = new_message_buffer(msize + 6 /* TUPLE5 */); + bp = new_message_buffer(msize + seq_trace_size +#ifdef USE_VM_PROBES + + dt_utag_size +#endif + ); hp = bp->mem; BM_SWAP_TIMER(send,copy); - token = copy_struct(SEQ_TRACE_TOKEN(sender), - 6 /* TUPLE5 */, + token = copy_struct(stoken, + seq_trace_size, &hp, &bp->off_heap); message = copy_struct(message, msize, &hp, &bp->off_heap); +#ifdef USE_VM_PROBES + if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) { + utag = copy_struct(DT_UTAG(sender), dt_utag_size, &hp, &bp->off_heap); +#ifdef DTRACE_TAG_HARDDEBUG + erts_fprintf(stderr, + "Dtrace -> (%T) Spreading tag (%T) with " + "message %T!\r\n",sender->id, utag, message); +#endif + } +#endif BM_MESSAGE_COPIED(msize); BM_SWAP_TIMER(copy,send); +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(message_send)) { + if (stoken != NIL && stoken != am_have_dt_utag) { + tok_label = signed_val(SEQ_TRACE_T_LABEL(stoken)); + tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(stoken)); + tok_serial = signed_val(SEQ_TRACE_T_SERIAL(stoken)); + } + DTRACE6(message_send, sender_name, receiver_name, + msize, tok_label, tok_lastcnt, tok_serial); + } +#endif erts_queue_message(receiver, receiver_locks, bp, message, - token); + token +#ifdef USE_VM_PROBES + , utag +#endif + ); BM_SWAP_TIMER(send,system); #ifdef HYBRID } else { @@ -835,8 +1014,13 @@ erts_send_message(Process* sender, #endif LAZY_COPY(sender,message); BM_SWAP_TIMER(copy,send); + DTRACE6(message_send, sender_name, receiver_name, + size_object(message)msize, tok_label, tok_lastcnt, tok_serial); ERL_MESSAGE_TERM(mp) = message; ERL_MESSAGE_TOKEN(mp) = NIL; +#ifdef USE_VM_PROBES + ERL_MESSAGE_DT_UTAG(mp) = NIL; +#endif mp->next = NULL; LINK_MESSAGE(receiver, mp); ACTIVATE(receiver); @@ -874,9 +1058,14 @@ erts_send_message(Process* sender, { ErlMessage* mp = message_alloc(); + DTRACE6(message_send, sender_name, receiver_name, + size_object(message), tok_label, tok_lastcnt, tok_serial); mp->data.attached = NULL; ERL_MESSAGE_TERM(mp) = message; ERL_MESSAGE_TOKEN(mp) = NIL; +#ifdef USE_VM_PROBES + ERL_MESSAGE_DT_UTAG(mp) = NIL; +#endif mp->next = NULL; /* * We move 'in queue' to 'private queue' and place @@ -908,7 +1097,13 @@ erts_send_message(Process* sender, message = copy_struct(message, msize, &hp, ohp); BM_MESSAGE_COPIED(msz); BM_SWAP_TIMER(copy,send); - erts_queue_message(receiver, receiver_locks, bp, message, token); + DTRACE6(message_send, sender_name, receiver_name, + msize, tok_label, tok_lastcnt, tok_serial); + erts_queue_message(receiver, receiver_locks, bp, message, token +#ifdef USE_VM_PROBES + , NIL +#endif + ); BM_SWAP_TIMER(send,system); #else ErlMessage* mp = message_alloc(); @@ -928,8 +1123,13 @@ erts_send_message(Process* sender, message = copy_struct(message, msize, &hp, &receiver->off_heap); BM_MESSAGE_COPIED(msize); BM_SWAP_TIMER(copy,send); + DTRACE6(message_send, sender_name, receiver_name, + (uint32_t)msize, tok_label, tok_lastcnt, tok_serial); ERL_MESSAGE_TERM(mp) = message; ERL_MESSAGE_TOKEN(mp) = NIL; +#ifdef USE_VM_PROBES + ERL_MESSAGE_DT_UTAG(mp) = NIL; +#endif mp->next = NULL; mp->data.attached = NULL; LINK_MESSAGE(receiver, mp); @@ -968,7 +1168,11 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, Eterm temptoken; ErlHeapFragment* bp = NULL; - if (token != NIL) { + if (token != NIL +#ifdef USE_VM_PROBES + && token != am_have_dt_utag +#endif + ) { ASSERT(is_tuple(token)); sz_reason = size_object(reason); @@ -983,7 +1187,11 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, /* the trace token must in this case be updated by the caller */ seq_trace_output(token, save, SEQ_TRACE_SEND, to->id, NULL); temptoken = copy_struct(token, sz_token, &hp, &bp->off_heap); - erts_queue_message(to, to_locksp, bp, save, temptoken); + erts_queue_message(to, to_locksp, bp, save, temptoken +#ifdef USE_VM_PROBES + , NIL +#endif + ); } else { ErlOffHeap *ohp; sz_reason = size_object(reason); @@ -1000,7 +1208,11 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, ? from : copy_struct(from, sz_from, &hp, ohp)); save = TUPLE3(hp, am_EXIT, from_copy, mess); - erts_queue_message(to, to_locksp, bp, save, NIL); + erts_queue_message(to, to_locksp, bp, save, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); } } diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 5aca0db6fe..7678c7c753 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -70,11 +70,18 @@ typedef struct erl_mesg { ErlHeapFragment *heap_frag; void *attached; } data; +#ifdef USE_VM_PROBES + Eterm m[3]; /* m[0] = message, m[1] = seq trace token, m[3] = dynamic trace user tag */ +#else Eterm m[2]; /* m[0] = message, m[1] = seq trace token */ +#endif } ErlMessage; #define ERL_MESSAGE_TERM(mp) ((mp)->m[0]) #define ERL_MESSAGE_TOKEN(mp) ((mp)->m[1]) +#ifdef USE_VM_PROBES +#define ERL_MESSAGE_DT_UTAG(mp) ((mp)->m[2]) +#endif /* Size of default message buffer (erl_message.c) */ #define ERL_MESSAGE_BUF_SZ 500 @@ -221,7 +228,11 @@ ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint, Eterm *, Uint); void free_message_buffer(ErlHeapFragment *); void erts_queue_dist_message(Process*, ErtsProcLocks*, ErtsDistExternal *, Eterm); -void erts_queue_message(Process*, ErtsProcLocks*, ErlHeapFragment*, Eterm, Eterm); +void erts_queue_message(Process*, ErtsProcLocks*, ErlHeapFragment*, Eterm, Eterm +#ifdef USE_VM_PROBES + , Eterm dt_utag +#endif +); void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm); void erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned); void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp); diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 58a09986d2..40f2fde578 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -66,6 +66,9 @@ static void add_readonly_check(ErlNifEnv*, unsigned char* ptr, unsigned sz); static int is_offheap(const ErlOffHeap* off_heap); #endif +#ifdef USE_VM_PROBES +void dtrace_nifenv_str(ErlNifEnv *, char *); +#endif #define MIN_HEAP_FRAG_SZ 200 static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp); @@ -350,7 +353,11 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, if (flush_me) { flush_env(env); /* Needed for ERTS_HOLE_CHECK */ } - erts_queue_message(rp, &rp_locks, frags, msg, am_undefined); + erts_queue_message(rp, &rp_locks, frags, msg, am_undefined +#ifdef USE_VM_PROBES + , NIL +#endif + ); if (rp_locks) { ERTS_SMP_LC_ASSERT(rp_locks == (rp_had_locks | (ERTS_PROC_LOCK_MSGQ | ERTS_PROC_LOCK_STATUS))); @@ -1779,6 +1786,13 @@ void erl_nif_init() resource_type_list.name = THE_NON_VALUE; } +#ifdef USE_VM_PROBES +void dtrace_nifenv_str(ErlNifEnv *env, char *process_buf) +{ + dtrace_pid_str(env->proc->id, process_buf); +} +#endif + #ifdef READONLY_CHECK /* Use checksums to assert that NIFs do not write into inspected binaries */ diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 908ba755ed..1481f66b55 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -27,6 +27,7 @@ #include "big.h" #include "error.h" #include "erl_thr_progress.h" +#include "dtrace-wrapper.h" Hash erts_dist_table; Hash erts_node_table; @@ -42,6 +43,8 @@ Sint erts_no_of_not_connected_dist_entries; DistEntry *erts_this_dist_entry; ErlNode *erts_this_node; +char erts_this_node_sysname_BUFFER[256], + *erts_this_node_sysname = "uninitialized yet"; static Uint node_entries; static Uint dist_entries; @@ -702,6 +705,9 @@ erts_set_this_node(Eterm sysname, Uint creation) (void) hash_erase(&erts_node_table, (void *) erts_this_node); erts_this_node->sysname = sysname; erts_this_node->creation = creation; + erts_this_node_sysname = erts_this_node_sysname_BUFFER; + erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname), + "%T", sysname); (void) hash_put(&erts_node_table, (void *) erts_this_node); erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); @@ -789,6 +795,9 @@ void erts_init_node_tables(void) erts_this_node->sysname = am_Noname; erts_this_node->creation = 0; erts_this_node->dist_entry = erts_this_dist_entry; + erts_this_node_sysname = erts_this_node_sysname_BUFFER; + erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname), + "%T", erts_this_node->sysname); (void) hash_put(&erts_node_table, (void *) erts_this_node); diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index b0a63ae035..5cfd0ac641 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -169,6 +169,7 @@ extern Sint erts_no_of_not_connected_dist_entries; extern DistEntry *erts_this_dist_entry; extern ErlNode *erts_this_node; +extern char *erts_this_node_sysname; /* must match erl_node_tables.c */ DistEntry *erts_channel_no_to_dist_entry(Uint); DistEntry *erts_sysname_to_connected_dist_entry(Eterm); diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index a2b08fcf56..a8cb4563d6 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -32,6 +32,7 @@ #include "global.h" #include "erl_port_task.h" #include "dist.h" +#include "dtrace-wrapper.h" #if defined(DEBUG) && 0 #define HARD_DEBUG @@ -61,6 +62,20 @@ do { \ (P)->sched.next = NULL; \ } while (0) +#ifdef USE_VM_PROBES +#define DTRACE_DRIVER(PROBE_NAME, PP) \ + if (DTRACE_ENABLED(driver_ready_input)) { \ + DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); \ + \ + dtrace_pid_str(PP->connected, process_str); \ + dtrace_port_str(PP, port_str); \ + DTRACE3(PROBE_NAME, process_str, port_str, PP->name); \ + } +#else +#define DTRACE_DRIVER(PROBE_NAME, PP) do {} while(0) +#endif + erts_smp_atomic_t erts_port_task_outstanding_io_tasks; struct ErtsPortTaskQueue_ { @@ -823,12 +838,15 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) goto tasks_done; case ERTS_PORT_TASK_TIMEOUT: reds += ERTS_PORT_REDS_TIMEOUT; - if (!(pp->status & ERTS_PORT_SFLGS_DEAD)) + if (!(pp->status & ERTS_PORT_SFLGS_DEAD)) { + DTRACE_DRIVER(driver_timeout, pp); (*pp->drv_ptr->timeout)((ErlDrvData) pp->drv_data); + } break; case ERTS_PORT_TASK_INPUT: reds += ERTS_PORT_REDS_INPUT; ASSERT((pp->status & ERTS_PORT_SFLGS_DEAD) == 0); + DTRACE_DRIVER(driver_ready_input, pp); /* NOTE some windows drivers use ->ready_input for input and output */ (*pp->drv_ptr->ready_input)((ErlDrvData) pp->drv_data, ptp->event); io_tasks_executed++; @@ -836,12 +854,14 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) case ERTS_PORT_TASK_OUTPUT: reds += ERTS_PORT_REDS_OUTPUT; ASSERT((pp->status & ERTS_PORT_SFLGS_DEAD) == 0); + DTRACE_DRIVER(driver_ready_output, pp); (*pp->drv_ptr->ready_output)((ErlDrvData) pp->drv_data, ptp->event); io_tasks_executed++; break; case ERTS_PORT_TASK_EVENT: reds += ERTS_PORT_REDS_EVENT; ASSERT((pp->status & ERTS_PORT_SFLGS_DEAD) == 0); + DTRACE_DRIVER(driver_event, pp); (*pp->drv_ptr->event)((ErlDrvData) pp->drv_data, ptp->event, ptp->event_data); io_tasks_executed++; break; diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 5469a59d8c..95d408f79d 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -42,6 +42,7 @@ #include "erl_thr_progress.h" #include "erl_thr_queue.h" #include "erl_async.h" +#include "dtrace-wrapper.h" #define ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED (2000*CONTEXT_REDS) #define ERTS_RUNQ_CALL_CHECK_BALANCE_REDS \ @@ -104,6 +105,9 @@ do { \ #define ERTS_EMPTY_RUNQ(RQ) \ ((RQ)->len == 0 && (RQ)->misc.start == NULL) +#define ERTS_EMPTY_RUNQ_PORTS(RQ) \ + ((RQ)->ports.info.len == 0 && (RQ)->misc.start == NULL) + extern BeamInstr beam_apply[]; extern BeamInstr beam_exit[]; extern BeamInstr beam_continue_exit[]; @@ -228,7 +232,6 @@ static Uint last_reductions; static Uint last_exact_reductions; Uint erts_default_process_flags; Eterm erts_system_monitor; -Eterm erts_system_monitor_msg_queue_len; Eterm erts_system_monitor_long_gc; Eterm erts_system_monitor_large_heap; struct erts_system_monitor_flags_t erts_system_monitor_flags; @@ -348,40 +351,28 @@ dbg_chk_aux_work_val(erts_aint32_t value) { erts_aint32_t valid = 0; -#ifdef ERTS_SSI_AUX_WORK_SET_TMO valid |= ERTS_SSI_AUX_WORK_SET_TMO; -#endif -#ifdef ERTS_SSI_AUX_WORK_CHECK_CHILDREN - valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN; -#endif -#ifdef ERTS_SSI_AUX_WORK_MISC valid |= ERTS_SSI_AUX_WORK_MISC; -#endif -#ifdef ERTS_SSI_AUX_WORK_MISC_THR_PRGR - valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR; -#endif -#ifdef ERTS_SSI_AUX_WORK_ASYNC_READY - valid |= ERTS_SSI_AUX_WORK_ASYNC_READY; -#endif -#ifdef ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN - valid |= ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; -#endif - -#ifdef ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM; -#endif -#ifdef ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC; +#if ERTS_USE_ASYNC_READY_Q + valid |= ERTS_SSI_AUX_WORK_ASYNC_READY; + valid |= ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; #endif -#ifdef ERTS_SSI_AUX_WORK_DD +#ifdef ERTS_SMP + valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR; valid |= ERTS_SSI_AUX_WORK_DD; -#endif -#ifdef ERTS_SSI_AUX_WORK_DD valid |= ERTS_SSI_AUX_WORK_DD_THR_PRGR; #endif -#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK +#if HAVE_ERTS_MSEG valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK; #endif +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN + valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN; +#endif +#ifdef ERTS_SSI_AUX_WORK_REAP_PORTS + valid |= ERTS_SSI_AUX_WORK_REAP_PORTS; +#endif if (~valid & value) erl_exit(ERTS_ABORT_EXIT, @@ -537,6 +528,213 @@ erts_late_init_process(void) } +static void +init_sched_wall_time(ErtsSchedWallTime *swtp) +{ + swtp->enabled = 0; + swtp->start = 0; + swtp->working.total = 0; + swtp->working.start = 0; + swtp->working.currently = 0; +} + +static ERTS_INLINE Uint64 +sched_wall_time_ts(void) +{ +#ifdef HAVE_GETHRTIME + return (Uint64) sys_gethrtime(); +#else + Uint64 res; + SysTimeval tv; + sys_gettimeofday(&tv); + res = (Uint64) tv.tv_sec*1000000; + res += (Uint64) tv.tv_usec; + return res; +#endif +} + +static ERTS_INLINE void +sched_wall_time_change(ErtsSchedulerData *esdp, int working) +{ + if (esdp->sched_wall_time.enabled) { + Uint64 ts = sched_wall_time_ts(); + if (working) { +#ifdef DEBUG + ASSERT(!esdp->sched_wall_time.working.currently); + esdp->sched_wall_time.working.currently = 1; +#endif + ts -= esdp->sched_wall_time.start; + esdp->sched_wall_time.working.start = ts; + } + else { +#ifdef DEBUG + ASSERT(esdp->sched_wall_time.working.currently); + esdp->sched_wall_time.working.currently = 0; +#endif + ts -= esdp->sched_wall_time.start; + ts -= esdp->sched_wall_time.working.start; + esdp->sched_wall_time.working.total += ts; + } + } +} + +typedef struct { + int set; + int enable; + Process *proc; + Eterm ref; + Eterm ref_heap[REF_THING_SIZE]; + Uint req_sched; + erts_smp_atomic32_t refc; +} ErtsSchedWallTimeReq; + +#if !HALFWORD_HEAP +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(swtreq, + ErtsSchedWallTimeReq, + 5, + ERTS_ALC_T_SCHED_WTIME_REQ) +#else +static ERTS_INLINE ErtsSchedWallTimeReq * +swtreq_alloc(void) +{ + return erts_alloc(ERTS_ALC_T_SCHED_WTIME_REQ, + sizeof(ErtsSchedWallTimeReq)); +} + +static ERTS_INLINE void +swtreq_free(ErtsSchedWallTimeReq *ptr) +{ + erts_free(ERTS_ALC_T_SCHED_WTIME_REQ, ptr); +} +#endif + +static void +reply_sched_wall_time(void *vswtrp) +{ + Uint64 working = 0, total = 0; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsSchedWallTimeReq *swtrp = (ErtsSchedWallTimeReq *) vswtrp; + ErtsProcLocks rp_locks = (swtrp->req_sched == esdp->no + ? ERTS_PROC_LOCK_MAIN + : 0); + Process *rp = swtrp->proc; + Eterm ref_copy = NIL, msg; + Eterm *hp = NULL; + Eterm **hpp; + Uint sz, *szp; + ErlOffHeap *ohp = NULL; + ErlHeapFragment *bp = NULL; + + ASSERT(esdp); + + if (swtrp->set) { + if (!swtrp->enable && esdp->sched_wall_time.enabled) + esdp->sched_wall_time.enabled = 0; + else if (swtrp->enable && !esdp->sched_wall_time.enabled) { + Uint64 ts = sched_wall_time_ts(); + esdp->sched_wall_time.enabled = 1; + esdp->sched_wall_time.start = ts; + esdp->sched_wall_time.working.total = 0; + esdp->sched_wall_time.working.start = 0; + esdp->sched_wall_time.working.currently = 1; + } + } + + if (esdp->sched_wall_time.enabled) { + Uint64 ts = sched_wall_time_ts(); + ASSERT(esdp->sched_wall_time.working.currently); + ts -= esdp->sched_wall_time.start; + total = ts; + ts -= esdp->sched_wall_time.working.start; + working = esdp->sched_wall_time.working.total + ts; + } + + sz = 0; + hpp = NULL; + szp = &sz; + + while (1) { + if (hpp) + ref_copy = STORE_NC(hpp, ohp, swtrp->ref); + else + *szp += REF_THING_SIZE; + + if (swtrp->set) + msg = ref_copy; + else { + msg = (!esdp->sched_wall_time.enabled + ? am_notsup + : erts_bld_tuple(hpp, szp, 3, + make_small(esdp->no), + erts_bld_uint64(hpp, szp, working), + erts_bld_uint64(hpp, szp, total))); + + msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg); + } + if (hpp) + break; + + hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks); + szp = NULL; + hpp = &hp; + } + + erts_queue_message(rp, &rp_locks, bp, msg, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); + + if (swtrp->req_sched == esdp->no) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + + erts_smp_proc_dec_refc(rp); + + if (erts_smp_atomic32_dec_read_nob(&swtrp->refc) == 0) + swtreq_free(vswtrp); +} + +Eterm +erts_sched_wall_time_request(Process *c_p, int set, int enable) +{ + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + Eterm ref; + ErtsSchedWallTimeReq *swtrp; + Eterm *hp; + + if (!set && !esdp->sched_wall_time.enabled) + return THE_NON_VALUE; + + swtrp = swtreq_alloc(); + ref = erts_make_ref(c_p); + hp = &swtrp->ref_heap[0]; + + swtrp->set = set; + swtrp->enable = enable; + swtrp->proc = c_p; + swtrp->ref = STORE_NC(&hp, NULL, ref); + swtrp->req_sched = esdp->no; + erts_smp_atomic32_init_nob(&swtrp->refc, + (erts_aint32_t) erts_no_schedulers); + + erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers); + +#ifdef ERTS_SMP + if (erts_no_schedulers > 1) + erts_schedule_multi_misc_aux_work(1, + erts_no_schedulers, + reply_sched_wall_time, + (void *) swtrp); +#endif + + reply_sched_wall_time((void *) swtrp); + + return ref; +} + static ERTS_INLINE ErtsProcList * proclist_create(Process *p) { @@ -674,8 +872,6 @@ set_aux_work_flags_wakeup_nob(ErtsSchedulerSleepInfo *ssi, } } -#if 0 /* Currently not used */ - static ERTS_INLINE void set_aux_work_flags_wakeup_relb(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs) @@ -695,8 +891,6 @@ set_aux_work_flags_wakeup_relb(ErtsSchedulerSleepInfo *ssi, } } -#endif - static ERTS_INLINE erts_aint32_t set_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs) { @@ -709,6 +903,27 @@ unset_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs) return erts_atomic32_read_band_nob(&ssi->aux_work, ~flgs); } +#ifdef ERTS_SMP + +static ERTS_INLINE void +thr_prgr_current_reset(ErtsAuxWorkData *awdp) +{ + awdp->current_thr_prgr = ERTS_THR_PRGR_INVALID; +} + +static ERTS_INLINE ErtsThrPrgrVal +thr_prgr_current(ErtsAuxWorkData *awdp) +{ + ErtsThrPrgrVal current = awdp->current_thr_prgr; + if (current == ERTS_THR_PRGR_INVALID) { + current = erts_thr_progress_current(); + awdp->current_thr_prgr = current; + } + return current; +} + +#endif + typedef struct erts_misc_aux_work_t_ erts_misc_aux_work_t; struct erts_misc_aux_work_t_ { void (*func)(void *); @@ -781,7 +996,7 @@ misc_aux_work_clean(ErtsThrQ_t *q, return aux_work; } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_misc_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { @@ -801,12 +1016,13 @@ handle_misc_aux_work(ErtsAuxWorkData *awdp, #ifdef ERTS_SMP -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { - if (!erts_thr_progress_has_reached(awdp->misc.thr_prgr)) - return aux_work; + if (!erts_thr_progress_has_reached_this(thr_prgr_current(awdp), + awdp->misc.thr_prgr)) + return aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR; unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR); @@ -879,7 +1095,7 @@ erts_notify_check_async_ready_queue(void *vno) ERTS_SSI_AUX_WORK_ASYNC_READY); } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_async_ready(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { @@ -901,7 +1117,7 @@ handle_async_ready(ErtsAuxWorkData *awdp, | ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN); } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_async_ready_clean(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { @@ -909,7 +1125,8 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp, #ifdef ERTS_SMP if (awdp->async_ready.need_thr_prgr - && !erts_thr_progress_has_reached(awdp->async_ready.thr_prgr)) { + && !erts_thr_progress_has_reached_this(thr_prgr_current(awdp), + awdp->async_ready.thr_prgr)) { return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; } @@ -937,7 +1154,7 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp, #endif -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { ErtsSchedulerSleepInfo *ssi = awdp->ssi; @@ -965,16 +1182,18 @@ erts_alloc_notify_delayed_dealloc(int ix) ERTS_SSI_AUX_WORK_DD); } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { ErtsSchedulerSleepInfo *ssi = awdp->ssi; int need_thr_progress = 0; + ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; int more_work = 0; unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD); erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp, &need_thr_progress, + &wakeup, &more_work); if (more_work) { if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD) @@ -986,9 +1205,12 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) } if (need_thr_progress) { + if (wakeup == ERTS_THR_PRGR_INVALID) + wakeup = erts_thr_progress_later_than(thr_prgr_current(awdp)); + awdp->dd.thr_prgr = wakeup; set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); - awdp->dd.thr_prgr = erts_thr_progress_later(); - erts_thr_progress_wakeup(awdp->esdp, awdp->dd.thr_prgr); + awdp->dd.thr_prgr = wakeup; + erts_thr_progress_wakeup(awdp->esdp, wakeup); } else if (awdp->dd.completed_callback) { awdp->dd.completed_callback(awdp->dd.completed_arg); @@ -998,14 +1220,16 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) return aux_work & ~ERTS_SSI_AUX_WORK_DD; } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { ErtsSchedulerSleepInfo *ssi; int need_thr_progress; int more_work; + ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; + ErtsThrPrgrVal current = thr_prgr_current(awdp); - if (!erts_thr_progress_has_reached(awdp->dd.thr_prgr)) + if (!erts_thr_progress_has_reached_this(current, awdp->dd.thr_prgr)) return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR; ssi = awdp->ssi; @@ -1014,6 +1238,7 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp, &need_thr_progress, + &wakeup, &more_work); if (more_work) { set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD); @@ -1023,8 +1248,10 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) } if (need_thr_progress) { - awdp->dd.thr_prgr = erts_thr_progress_later(); - erts_thr_progress_wakeup(awdp->esdp, awdp->dd.thr_prgr); + if (wakeup == ERTS_THR_PRGR_INVALID) + wakeup = erts_thr_progress_later_than(current); + awdp->dd.thr_prgr = wakeup; + erts_thr_progress_wakeup(awdp->esdp, wakeup); } else { unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); @@ -1121,7 +1348,7 @@ erts_smp_notify_check_children_needed(void) ERTS_SSI_AUX_WORK_CHECK_CHILDREN); } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_CHECK_CHILDREN); @@ -1131,9 +1358,68 @@ handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) #endif -#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK +static void +notify_reap_ports_relb(void) +{ + int i; + for (i = 0; i < erts_no_schedulers; i++) { + set_aux_work_flags_wakeup_relb(ERTS_SCHED_SLEEP_INFO_IX(i), + ERTS_SSI_AUX_WORK_REAP_PORTS); + } +} + +erts_smp_atomic32_t erts_halt_progress; +int erts_halt_code; -static erts_aint32_t +static ERTS_INLINE erts_aint32_t +handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_REAP_PORTS); + awdp->esdp->run_queue->halt_in_progress = 1; + if (erts_smp_atomic32_dec_read_acqb(&erts_halt_progress) == 0) { + int i; + erts_smp_atomic32_set_nob(&erts_halt_progress, 1); + for (i = 0; i < erts_max_ports; i++) { + Port *prt = &erts_port[i]; + erts_smp_port_state_lock(prt); + if ((prt->status & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP + | ERTS_PORT_SFLG_HALT))) { + erts_smp_port_state_unlock(prt); + continue; + } + /* We need to set the halt flag - get the port lock */ +#ifdef ERTS_SMP + erts_smp_atomic_inc_nob(&prt->refc); +#endif + erts_smp_port_state_unlock(prt); +#ifdef ERTS_SMP + erts_smp_mtx_lock(prt->lock); +#endif + if ((prt->status & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP + | ERTS_PORT_SFLG_HALT))) { + erts_port_release(prt); + continue; + } + erts_port_status_bor_set(prt, ERTS_PORT_SFLG_HALT); + erts_smp_atomic32_inc_nob(&erts_halt_progress); + if (prt->status & (ERTS_PORT_SFLG_EXITING + | ERTS_PORT_SFLG_CLOSING)) { + erts_port_release(prt); + continue; + } + erts_do_exit_port(prt, prt->id, am_killed); + erts_port_release(prt); + } + if (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0) { + erl_exit_flush_async(erts_halt_code, ""); + } + } + return aux_work & ~ERTS_SSI_AUX_WORK_REAP_PORTS; +} + +#if HAVE_ERTS_MSEG + +static ERTS_INLINE erts_aint32_t handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK); @@ -1143,7 +1429,7 @@ handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) #endif -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_SET_TMO); @@ -1151,69 +1437,95 @@ handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) return aux_work & ~ERTS_SSI_AUX_WORK_SET_TMO; } -static ERTS_INLINE erts_aint32_t -handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +static erts_aint32_t +handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work) { +#undef HANDLE_AUX_WORK +#define HANDLE_AUX_WORK(FLG, HNDLR) \ + ignore |= FLG; \ + if (aux_work & FLG) { \ + aux_work = HNDLR(awdp, aux_work); \ + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \ + if (!(aux_work & ~ignore)) { \ + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \ + return aux_work; \ + } \ + } + + erts_aint32_t aux_work = orig_aux_work; + erts_aint32_t ignore = 0; + +#ifdef ERTS_SMP + thr_prgr_current_reset(awdp); +#endif + + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + ASSERT(aux_work); + /* * Handlers are *only* allowed to modify flags in return value * and ssi flags that are explicity handled by the handler. * Handlers are, e.g., not allowed to read the ssi flag field and * then unconditionally return that value. + * + * Flag field returned should only contain flags for work that + * can continue immediately. + */ + + /* + * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative + * eachother. Most frequent first. */ - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - if (aux_work & ERTS_SSI_AUX_WORK_SET_TMO) { - aux_work = handle_setup_aux_work_timer(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } #ifdef ERTS_SMP - if (aux_work & ERTS_SSI_AUX_WORK_MISC_THR_PRGR) { - aux_work = handle_misc_aux_work_thr_prgr(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD, + handle_delayed_dealloc); + /* DD must be before DD_THR_PRGR */ + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD_THR_PRGR, + handle_delayed_dealloc_thr_prgr); #endif - if (aux_work & ERTS_SSI_AUX_WORK_MISC) { - aux_work = handle_misc_aux_work(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + + HANDLE_AUX_WORK((ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), + handle_fix_alloc); + #if ERTS_USE_ASYNC_READY_Q - if (aux_work & ERTS_SSI_AUX_WORK_ASYNC_READY) { - aux_work = handle_async_ready(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } - if (aux_work & ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN) { - aux_work = handle_async_ready_clean(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } -#endif -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) { - aux_work = handle_check_children(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY, + handle_async_ready); + /* ASYNC_READY must be before ASYNC_READY_CLEAN */ + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN, + handle_async_ready_clean); #endif - if (aux_work & (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM - | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC)) { - aux_work = handle_fix_alloc(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + #ifdef ERTS_SMP - if (aux_work & ERTS_SSI_AUX_WORK_DD) { - aux_work = handle_delayed_dealloc(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } - if (aux_work & ERTS_SSI_AUX_WORK_DD_THR_PRGR) { - aux_work = handle_delayed_dealloc_thr_prgr(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC_THR_PRGR, + handle_misc_aux_work_thr_prgr); #endif -#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK - if (aux_work & ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK) { - aux_work = handle_mseg_cache_check(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + /* MISC_THR_PRGR must be before MISC */ + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC, + handle_misc_aux_work); + +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CHECK_CHILDREN, + handle_check_children); +#endif + + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_SET_TMO, + handle_setup_aux_work_timer); + +#if HAVE_ERTS_MSEG + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK, + handle_mseg_cache_check); #endif + + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_REAP_PORTS, + handle_reap_ports); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + return aux_work; + +#undef HANDLE_AUX_WORK + } typedef struct { @@ -1707,6 +2019,7 @@ aux_thread(void *unused) static void scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) { + int working = 1; ErtsSchedulerSleepInfo *ssi = esdp->ssi; int spincount; erts_aint32_t aux_work = 0; @@ -1737,12 +2050,17 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) tse_wait: + if (thr_prgr_active != working) + sched_wall_time_change(esdp, thr_prgr_active); + while (1) { aux_work = erts_atomic32_read_acqb(&ssi->aux_work); if (aux_work) { - if (!thr_prgr_active) + if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } aux_work = handle_aux_work(&esdp->aux_work_data, aux_work); if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); @@ -1751,8 +2069,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) if (aux_work) flgs = erts_smp_atomic32_read_acqb(&ssi->flags); else { - if (thr_prgr_active) + if (thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 0); + sched_wall_time_change(esdp, 0); + } erts_thr_progress_prepare_wait(esdp); flgs = sched_spin_wait(ssi, spincount); @@ -1789,8 +2109,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) if (flgs & ~ERTS_SSI_FLG_SUSPENDED) erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); - if (!thr_prgr_active) + if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } erts_smp_runq_lock(rq); sched_active(esdp->no, rq); @@ -1806,14 +2128,21 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) sched_waiting_sys(esdp->no, rq); + erts_smp_runq_unlock(rq); + ASSERT(working); + sched_wall_time_change(esdp, working = 0); + spincount = ERTS_SCHED_SYS_SLEEP_SPINCOUNT; while (spincount-- > 0) { sys_poll_aux_work: + if (working) + sched_wall_time_change(esdp, working = 0); + ASSERT(!erts_port_task_have_outstanding_io_tasks()); erl_sys_schedule(1); /* Might give us something to do */ @@ -1828,6 +2157,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) aux_work = erts_atomic32_read_acqb(&ssi->aux_work); if (aux_work) { + if (!working) + sched_wall_time_change(esdp, working = 1); #ifdef ERTS_SMP if (!thr_prgr_active) erts_thr_progress_active(esdp, thr_prgr_active = 1); @@ -1920,6 +2251,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_smp_runq_unlock(rq); + if (working) + sched_wall_time_change(esdp, working = 0); + #ifdef ERTS_SMP if (thr_prgr_active) erts_thr_progress_active(esdp, thr_prgr_active = 0); @@ -1955,6 +2289,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) if (flgs & ~ERTS_SSI_FLG_SUSPENDED) erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); #endif + if (!working) + sched_wall_time_change(esdp, working = 1); sched_active_sys(esdp->no, rq); } @@ -2449,6 +2785,9 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq) ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, *rq_lockedp); ERTS_SMP_LC_CHK_RUNQ_LOCK(vrq, vrq_locked); + if (rq->halt_in_progress) + goto try_steal_port; + /* * Check for a runnable process to steal... */ @@ -2535,6 +2874,8 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq) vrq_locked = 1; } + try_steal_port: + ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, *rq_lockedp); ERTS_SMP_LC_CHK_RUNQ_LOCK(vrq, vrq_locked); @@ -2650,7 +2991,8 @@ try_steal_task(ErtsRunQueue *rq) erts_smp_runq_lock(rq); if (!res) - res = !ERTS_EMPTY_RUNQ(rq); + res = rq->halt_in_progress ? + !ERTS_EMPTY_RUNQ_PORTS(rq) : !ERTS_EMPTY_RUNQ(rq); return res; } @@ -3316,6 +3658,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) rq->len = 0; rq->wakeup_other = 0; rq->wakeup_other_reds = 0; + rq->halt_in_progress = 0; rq->procs.len = 0; rq->procs.pending_exiters = NULL; @@ -3430,9 +3773,14 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) esdp->run_queue->scheduler = esdp; init_aux_work_data(&esdp->aux_work_data, esdp); + init_sched_wall_time(&esdp->sched_wall_time); } init_misc_aux_work(); +#if !HALFWORD_HEAP + init_swtreq_alloc(); +#endif + #ifdef ERTS_SMP @@ -3505,6 +3853,9 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL); #endif #endif + + erts_smp_atomic32_init_relb(&erts_halt_progress, -1); + erts_halt_code = 0; } ErtsRunQueue * @@ -3755,6 +4106,8 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_inactive); + sched_wall_time_change(esdp, 0); + erts_smp_mtx_lock(&schdlr_sspnd.mtx); flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); @@ -3813,16 +4166,20 @@ suspend_scheduler(ErtsSchedulerData *esdp) aux_work = erts_atomic32_read_acqb(&ssi->aux_work); if (aux_work) { - if (!thr_prgr_active) + if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } aux_work = handle_aux_work(&esdp->aux_work_data, aux_work); if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); } if (!aux_work) { - if (thr_prgr_active) + if (thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 0); + sched_wall_time_change(esdp, 0); + } erts_thr_progress_prepare_wait(esdp); flgs = sched_spin_suspended(ssi, ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); @@ -3877,8 +4234,10 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_active); - if (!thr_prgr_active) + if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); @@ -5860,6 +6219,15 @@ Process *schedule(Process *p, int calls) int actual_reds; int reds; +#ifdef USE_VM_PROBES + if (p != NULL && DTRACE_ENABLED(process_unscheduled)) { + DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE); + + dtrace_proc_str(p, process_buf); + DTRACE1(process_unscheduled, process_buf); + } +#endif + if (ERTS_USE_MODIFIED_TIMING()) { context_reds = ERTS_MODIFIED_TIMING_CONTEXT_REDS; input_reductions = ERTS_MODIFIED_TIMING_INPUT_REDS; @@ -6063,7 +6431,9 @@ Process *schedule(Process *p, int calls) ASSERT(rq->len == rq->procs.len + rq->ports.info.len); - if (rq->len == 0 && !rq->misc.start) { + if ((rq->len == 0 && !rq->misc.start) + || (rq->halt_in_progress + && rq->ports.info.len == 0 && !rq->misc.start)) { #ifdef ERTS_SMP @@ -6161,7 +6531,8 @@ Process *schedule(Process *p, int calls) if (rq->ports.info.len) { int have_outstanding_io; have_outstanding_io = erts_port_task_execute(rq, &esdp->current_port); - if (have_outstanding_io && fcalls > 2*input_reductions) { + if ((have_outstanding_io && fcalls > 2*input_reductions) + || rq->halt_in_progress) { /* * If we have performed more than 2*INPUT_REDUCTIONS since * last call to erl_sys_schedule() and we still haven't @@ -6895,6 +7266,10 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->seq_trace_lastcnt = 0; p->seq_trace_clock = 0; SEQ_TRACE_TOKEN(p) = NIL; +#ifdef USE_VM_PROBES + DT_UTAG(p) = NIL; + DT_UTAG_FLAGS(p) = 0; +#endif p->parent = parent->id == ERTS_INVALID_PID ? NIL : parent->id; #ifdef HYBRID @@ -7027,6 +7402,16 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). VERBOSE(DEBUG_PROCESSES, ("Created a new process: %T\n",p->id)); +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(process_spawn)) { + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); + + dtrace_fun_decode(p, mod, func, arity, process_name, mfa); + DTRACE2(process_spawn, process_name, mfa); + } +#endif + error: erts_smp_proc_unlock(parent, ERTS_PROC_LOCKS_ALL_MINOR); @@ -7478,7 +7863,11 @@ static ERTS_INLINE void send_exit_message(Process *to, ErtsProcLocks *to_locksp, Eterm exit_term, Uint term_size, Eterm token) { - if (token == NIL) { + if (token == NIL +#ifdef USE_VM_PROBES + || token == am_have_dt_utag +#endif + ) { Eterm* hp; Eterm mess; ErlHeapFragment* bp; @@ -7486,7 +7875,11 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp, hp = erts_alloc_message_heap(term_size, &bp, &ohp, to, to_locksp); mess = copy_struct(exit_term, term_size, &hp, ohp); - erts_queue_message(to, to_locksp, bp, mess, NIL); + erts_queue_message(to, to_locksp, bp, mess, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); } else { ErlHeapFragment* bp; Eterm* hp; @@ -7502,7 +7895,11 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp, /* the trace token must in this case be updated by the caller */ seq_trace_output(token, mess, SEQ_TRACE_SEND, to->id, NULL); temp_token = copy_struct(token, sz_token, &hp, &bp->off_heap); - erts_queue_message(to, to_locksp, bp, mess, temp_token); + erts_queue_message(to, to_locksp, bp, mess, temp_token +#ifdef USE_VM_PROBES + , NIL +#endif + ); } } @@ -7595,9 +7992,26 @@ send_exit_signal(Process *c_p, /* current process if and only ASSERT(reason != THE_NON_VALUE); +#ifdef USE_VM_PROBES + if(DTRACE_ENABLED(process_exit_signal) && is_pid(from)) { + DTRACE_CHARBUF(sender_str, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(receiver_str, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(reason_buf, DTRACE_TERM_BUF_SIZE); + + dtrace_pid_str(from, sender_str); + dtrace_proc_str(rp, receiver_str); + erts_snprintf(reason_buf, sizeof(reason_buf) - 1, "%T", reason); + DTRACE3(process_exit_signal, sender_str, receiver_str, reason_buf); + } +#endif + if (ERTS_PROC_IS_TRAPPING_EXITS(rp) && (reason != am_kill || (flags & ERTS_XSIG_FLG_IGN_KILL))) { - if (is_not_nil(token) && token_update) + if (is_not_nil(token) +#ifdef USE_VM_PROBES + && token != am_have_dt_utag +#endif + && token_update) seq_trace_update_send(token_update); if (is_value(exit_tuple)) send_exit_message(rp, rp_locks, exit_tuple, exit_tuple_sz, token); @@ -8021,7 +8435,18 @@ erts_do_exit_process(Process* p, Eterm reason) p->arity = 0; /* No live registers */ p->fvalue = reason; - + +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(process_exit)) { + DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(reason_buf, DTRACE_TERM_BUF_SIZE); + + dtrace_proc_str(p, process_buf); + erts_snprintf(reason_buf, DTRACE_TERM_BUF_SIZE - 1, "%T", reason); + DTRACE2(process_exit, process_buf, reason_buf); + } +#endif + #ifdef ERTS_SMP ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); /* By locking all locks (main lock is already locked) when going @@ -9587,3 +10012,30 @@ debug_processes_assert_error(char* expr, char* file, int line) /* *\ * End of the processes/0 BIF implementation. * \* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * A nice system halt closing all open port goes as follows: + * 1) This function schedules the aux work ERTS_SSI_AUX_WORK_REAP_PORTS + * on all schedulers, then schedules itself out. + * 2) All shedulers detect this and set the flag halt_in_progress + * on their run queue. The last scheduler sets all non-closed ports + * ERTS_PORT_SFLG_HALT. Global atomic erts_halt_progress is used + * as refcount to determine which is last. + * 3) While the run ques has flag halt_in_progress no processes + * will be scheduled, only ports. + * 4) When the last port closes that scheduler calls erlang:halt/1. + * The same global atomic is used as refcount. + * + * A BIF that calls this should make sure to schedule out to never come back: + * erl_halt((int)(- code)); + * ERTS_BIF_YIELD1(bif_export[BIF_erlang_halt_1], BIF_P, NIL); + */ +void erl_halt(int code) +{ + if (-1 == erts_smp_atomic32_cmpxchg_acqb(&erts_halt_progress, + erts_no_schedulers, + -1)) { + erts_halt_code = code; + notify_reap_ports_relb(); + } +} diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index a51b380bb0..cff0783bc4 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -248,25 +248,23 @@ typedef enum { | ERTS_SSI_FLG_WAITING \ | ERTS_SSI_FLG_SUSPENDED) -#define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 0) -#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 1) -#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 2) -#ifdef ERTS_SMP -#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR (((erts_aint32_t) 1) << 3) -#endif -#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM (((erts_aint32_t) 1) << 4) -#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC (((erts_aint32_t) 1) << 5) -#define ERTS_SSI_AUX_WORK_ASYNC_READY (((erts_aint32_t) 1) << 6) -#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN (((erts_aint32_t) 1) << 7) -#ifdef ERTS_SMP -#define ERTS_SSI_AUX_WORK_DD (((erts_aint32_t) 1) << 8) -#define ERTS_SSI_AUX_WORK_DD_THR_PRGR (((erts_aint32_t) 1) << 9) -#endif -#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 10) +/* + * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative + * eachother. Most frequent - lowest bit number. + */ -#if !HAVE_ERTS_MSEG -# undef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK -#endif +#define ERTS_SSI_AUX_WORK_DD (((erts_aint32_t) 1) << 0) +#define ERTS_SSI_AUX_WORK_DD_THR_PRGR (((erts_aint32_t) 1) << 1) +#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC (((erts_aint32_t) 1) << 2) +#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM (((erts_aint32_t) 1) << 3) +#define ERTS_SSI_AUX_WORK_ASYNC_READY (((erts_aint32_t) 1) << 4) +#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN (((erts_aint32_t) 1) << 5) +#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR (((erts_aint32_t) 1) << 6) +#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 7) +#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 8) +#define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 9) +#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 10) +#define ERTS_SSI_AUX_WORK_REAP_PORTS (((erts_aint32_t) 1) << 11) typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo; @@ -344,6 +342,7 @@ struct ErtsRunQueue_ { int len; int wakeup_other; int wakeup_other_reds; + int halt_in_progress; struct { int len; @@ -394,9 +393,22 @@ do { \ } while (0) typedef struct { + int enabled; + Uint64 start; + struct { + Uint64 total; + Uint64 start; + int currently; + } working; +} ErtsSchedWallTime; + +typedef struct { int sched_id; ErtsSchedulerData *esdp; ErtsSchedulerSleepInfo *ssi; +#ifdef ERTS_SMP + ErtsThrPrgrVal current_thr_prgr; +#endif struct { int ix; #ifdef ERTS_SMP @@ -457,6 +469,8 @@ struct ErtsSchedulerData_ { ErtsSchedAllocData alloc_data; + ErtsSchedWallTime sched_wall_time; + #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC erts_alloc_verify_func_t verify_unused_temp_alloc; Allctr_t *verify_unused_temp_alloc_data; @@ -669,6 +683,10 @@ struct process { Uint seq_trace_lastcnt; Eterm seq_trace_token; /* Sequential trace token (tuple size 5 see below) */ +#ifdef USE_VM_PROBES + Eterm dt_utag; /* Place to store the dynamc trace user tag */ + Uint dt_utag_flags; /* flag field for the dt_utag */ +#endif BeamInstr initial[3]; /* Initial module(0), function(1), arity(2), often used instead of pointer to funcinfo instruction, hence the BeamInstr datatype */ BeamInstr* current; /* Current Erlang function, part of the funcinfo: @@ -984,6 +1002,14 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; #define SEQ_TRACE_PRINT (1 << 2) #define SEQ_TRACE_TIMESTAMP (1 << 3) +#ifdef USE_VM_PROBES +#define DT_UTAG_PERMANENT (1 << 0) +#define DT_UTAG_SPREADING (1 << 1) +#define DT_UTAG(P) ((P)->dt_utag) +#define DT_UTAG_FLAGS(P) ((P)->dt_utag_flags) +#endif + + #ifdef ERTS_SMP /* Status flags ... */ #define ERTS_PROC_SFLG_PENDADD2SCHEDQ (((Uint32) 1) << 0) /* Pending @@ -1064,6 +1090,8 @@ void erts_late_init_process(void); void erts_early_init_scheduling(int); void erts_init_scheduling(int, int); +Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable); + ErtsProcList *erts_proclist_create(Process *); void erts_proclist_destroy(ErtsProcList *); int erts_proclist_same(ErtsProcList *, Process *); @@ -1645,4 +1673,6 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi) #endif - +void erl_halt(int code); +extern erts_smp_atomic32_t erts_halt_progress; +extern int erts_halt_code; diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c index a7ccea7403..bff9d246a3 100644 --- a/erts/emulator/beam/erl_sched_spec_pre_alloc.c +++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011. All Rights Reserved. + * Copyright Ericsson AB 2011-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -227,7 +227,6 @@ fetch_remote(erts_sspa_chunk_header_t *chdr, int max) ERTS_THR_MEMORY_BARRIER; else { chdr->head.next.unref_end = (erts_sspa_blk_t *) ilast; - ERTS_THR_MEMORY_BARRIER; chdr->head.next.thr_progress = erts_thr_progress_later(); erts_atomic32_set_relb(&chdr->tail.data.um_refc_ix, um_refc_ix); diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index 63179dfad4..a32e9d9d7c 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2011. All Rights Reserved. + * Copyright Ericsson AB 2005-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -61,6 +61,11 @@ typedef erts_spinlock_t erts_smp_spinlock_t; typedef erts_rwlock_t erts_smp_rwlock_t; void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ +#define ERTS_SMP_MEMORY_BARRIER ERTS_THR_MEMORY_BARRIER +#define ERTS_SMP_WRITE_MEMORY_BARRIER ERTS_THR_WRITE_MEMORY_BARRIER +#define ERTS_SMP_READ_MEMORY_BARRIER ERTS_THR_READ_MEMORY_BARRIER +#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER + #else /* #ifdef ERTS_SMP */ #define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0} @@ -95,6 +100,11 @@ typedef struct { int gcc_is_buggy; } erts_smp_spinlock_t; typedef struct { int gcc_is_buggy; } erts_smp_rwlock_t; #endif +#define ERTS_SMP_MEMORY_BARRIER +#define ERTS_SMP_WRITE_MEMORY_BARRIER +#define ERTS_SMP_READ_MEMORY_BARRIER +#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER + #endif /* #ifdef ERTS_SMP */ ERTS_GLB_INLINE void erts_smp_thr_init(erts_smp_thr_init_data_t *id); @@ -206,13 +216,8 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */ /* - * Functions implementing atomic operations with with no (nob), - * full (mb), acquire (acqb), release (relb), read (rb), and - * write (wb) memory barriers. - * - * If SMP support has been disabled, they are mapped to functions - * that performs the same operation, but aren't atomic and don't - * imply memory barriers. + * See "Documentation of atomics and memory barriers" at the top + * of erl_threads.h for info on atomics. */ #ifdef ERTS_SMP @@ -239,6 +244,11 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_dw_atomic_read_relb erts_dw_atomic_read_relb #define erts_smp_dw_atomic_cmpxchg_relb erts_dw_atomic_cmpxchg_relb +#define erts_smp_dw_atomic_init_ddrb erts_dw_atomic_init_ddrb +#define erts_smp_dw_atomic_set_ddrb erts_dw_atomic_set_ddrb +#define erts_smp_dw_atomic_read_ddrb erts_dw_atomic_read_ddrb +#define erts_smp_dw_atomic_cmpxchg_ddrb erts_dw_atomic_cmpxchg_ddrb + #define erts_smp_dw_atomic_init_rb erts_dw_atomic_init_rb #define erts_smp_dw_atomic_set_rb erts_dw_atomic_set_rb #define erts_smp_dw_atomic_read_rb erts_dw_atomic_read_rb @@ -307,6 +317,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb #define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb +#define erts_smp_atomic_init_ddrb erts_atomic_init_ddrb +#define erts_smp_atomic_set_ddrb erts_atomic_set_ddrb +#define erts_smp_atomic_read_ddrb erts_atomic_read_ddrb +#define erts_smp_atomic_inc_read_ddrb erts_atomic_inc_read_ddrb +#define erts_smp_atomic_dec_read_ddrb erts_atomic_dec_read_ddrb +#define erts_smp_atomic_inc_ddrb erts_atomic_inc_ddrb +#define erts_smp_atomic_dec_ddrb erts_atomic_dec_ddrb +#define erts_smp_atomic_add_read_ddrb erts_atomic_add_read_ddrb +#define erts_smp_atomic_add_ddrb erts_atomic_add_ddrb +#define erts_smp_atomic_read_bor_ddrb erts_atomic_read_bor_ddrb +#define erts_smp_atomic_read_band_ddrb erts_atomic_read_band_ddrb +#define erts_smp_atomic_xchg_ddrb erts_atomic_xchg_ddrb +#define erts_smp_atomic_cmpxchg_ddrb erts_atomic_cmpxchg_ddrb + #define erts_smp_atomic_init_rb erts_atomic_init_rb #define erts_smp_atomic_set_rb erts_atomic_set_rb #define erts_smp_atomic_read_rb erts_atomic_read_rb @@ -393,6 +417,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb #define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb +#define erts_smp_atomic32_init_ddrb erts_atomic32_init_ddrb +#define erts_smp_atomic32_set_ddrb erts_atomic32_set_ddrb +#define erts_smp_atomic32_read_ddrb erts_atomic32_read_ddrb +#define erts_smp_atomic32_inc_read_ddrb erts_atomic32_inc_read_ddrb +#define erts_smp_atomic32_dec_read_ddrb erts_atomic32_dec_read_ddrb +#define erts_smp_atomic32_inc_ddrb erts_atomic32_inc_ddrb +#define erts_smp_atomic32_dec_ddrb erts_atomic32_dec_ddrb +#define erts_smp_atomic32_add_read_ddrb erts_atomic32_add_read_ddrb +#define erts_smp_atomic32_add_ddrb erts_atomic32_add_ddrb +#define erts_smp_atomic32_read_bor_ddrb erts_atomic32_read_bor_ddrb +#define erts_smp_atomic32_read_band_ddrb erts_atomic32_read_band_ddrb +#define erts_smp_atomic32_xchg_ddrb erts_atomic32_xchg_ddrb +#define erts_smp_atomic32_cmpxchg_ddrb erts_atomic32_cmpxchg_ddrb + #define erts_smp_atomic32_init_rb erts_atomic32_init_rb #define erts_smp_atomic32_set_rb erts_atomic32_set_rb #define erts_smp_atomic32_read_rb erts_atomic32_read_rb @@ -445,6 +483,11 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_dw_atomic_read_relb erts_no_dw_atomic_read #define erts_smp_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg +#define erts_smp_dw_atomic_init_ddrb erts_no_dw_atomic_init +#define erts_smp_dw_atomic_set_ddrb erts_no_dw_atomic_set +#define erts_smp_dw_atomic_read_ddrb erts_no_dw_atomic_read +#define erts_smp_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg + #define erts_smp_dw_atomic_init_rb erts_no_dw_atomic_init #define erts_smp_dw_atomic_set_rb erts_no_dw_atomic_set #define erts_smp_dw_atomic_read_rb erts_no_dw_atomic_read @@ -513,6 +556,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_xchg_relb erts_no_atomic_xchg #define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg +#define erts_smp_atomic_init_ddrb erts_no_atomic_set +#define erts_smp_atomic_set_ddrb erts_no_atomic_set +#define erts_smp_atomic_read_ddrb erts_no_atomic_read +#define erts_smp_atomic_inc_read_ddrb erts_no_atomic_inc_read +#define erts_smp_atomic_dec_read_ddrb erts_no_atomic_dec_read +#define erts_smp_atomic_inc_ddrb erts_no_atomic_inc +#define erts_smp_atomic_dec_ddrb erts_no_atomic_dec +#define erts_smp_atomic_add_read_ddrb erts_no_atomic_add_read +#define erts_smp_atomic_add_ddrb erts_no_atomic_add +#define erts_smp_atomic_read_bor_ddrb erts_no_atomic_read_bor +#define erts_smp_atomic_read_band_ddrb erts_no_atomic_read_band +#define erts_smp_atomic_xchg_ddrb erts_no_atomic_xchg +#define erts_smp_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg + #define erts_smp_atomic_init_rb erts_no_atomic_set #define erts_smp_atomic_set_rb erts_no_atomic_set #define erts_smp_atomic_read_rb erts_no_atomic_read @@ -599,6 +656,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg #define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg +#define erts_smp_atomic32_init_ddrb erts_no_atomic32_set +#define erts_smp_atomic32_set_ddrb erts_no_atomic32_set +#define erts_smp_atomic32_read_ddrb erts_no_atomic32_read +#define erts_smp_atomic32_inc_read_ddrb erts_no_atomic32_inc_read +#define erts_smp_atomic32_dec_read_ddrb erts_no_atomic32_dec_read +#define erts_smp_atomic32_inc_ddrb erts_no_atomic32_inc +#define erts_smp_atomic32_dec_ddrb erts_no_atomic32_dec +#define erts_smp_atomic32_add_read_ddrb erts_no_atomic32_add_read +#define erts_smp_atomic32_add_ddrb erts_no_atomic32_add +#define erts_smp_atomic32_read_bor_ddrb erts_no_atomic32_read_bor +#define erts_smp_atomic32_read_band_ddrb erts_no_atomic32_read_band +#define erts_smp_atomic32_xchg_ddrb erts_no_atomic32_xchg +#define erts_smp_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg + #define erts_smp_atomic32_init_rb erts_no_atomic32_set #define erts_smp_atomic32_set_rb erts_no_atomic32_set #define erts_smp_atomic32_read_rb erts_no_atomic32_read diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c index 75f8209c3b..9ef83746c5 100644 --- a/erts/emulator/beam/erl_thr_progress.c +++ b/erts/emulator/beam/erl_thr_progress.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011. All Rights Reserved. + * Copyright Ericsson AB 2011-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -33,10 +33,11 @@ * This module keeps track of the progress of a set of managed threads. Only * threads that behave well can be allowed to be managed. A managed thread * should update its thread progress frequently. Currently only scheduler - * threads and the aux_thread are managed threads. We typically do not want - * any async threads as managed threads since they cannot guarantee a - * frequent update of thread progress, since they execute user implemented - * driver code. + * threads, the system-message-dispatcher threads, and the aux-thread are + * managed threads. We typically do not want any async threads as managed + * threads since they cannot guarantee a frequent update of thread progress, + * since they execute user implemented driver code that is assumed to be + * time consuming. * * erts_thr_progress_current() returns the global current thread progress * value of managed threads. I.e., the latest progress value that all @@ -112,8 +113,10 @@ * * On 32-bit systems we therefore need a double word atomic. */ - +#undef read_acqb #define read_acqb erts_thr_prgr_read_acqb__ +#undef read_nob +#define read_nob erts_thr_prgr_read_nob__ #ifdef ARCH_64 @@ -129,12 +132,6 @@ set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) erts_atomic_set_nob(atmc, val); } -static ERTS_INLINE ErtsThrPrgrVal -read_nob(ERTS_THR_PRGR_ATOMIC *atmc) -{ - return (ErtsThrPrgrVal) erts_atomic_read_nob(atmc); -} - static ERTS_INLINE void init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) { @@ -143,52 +140,44 @@ init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) #else -#undef dw_sint_to_val -#define dw_sint_to_val erts_thr_prgr_dw_sint_to_val__ +#undef dw_aint_to_val +#define dw_aint_to_val erts_thr_prgr_dw_aint_to_val__ static void -val_to_dw_sint(ethr_dw_sint_t *dw_sint, ErtsThrPrgrVal val) +val_to_dw_aint(erts_dw_aint_t *dw_aint, ErtsThrPrgrVal val) { #ifdef ETHR_SU_DW_NAINT_T__ - dw_sint->dw_sint = (ETHR_SU_DW_NAINT_T__) val; + dw_aint->dw_sint = (ETHR_SU_DW_NAINT_T__) val; #else - dw_sint->sint[ETHR_DW_SINT_LOW_WORD] - = (ethr_sint_t) (val & 0xffffffff); - dw_sint->sint[ETHR_DW_SINT_HIGH_WORD] - = (ethr_sint_t) ((val >> 32) & 0xffffffff); + dw_aint->sint[ERTS_DW_AINT_LOW_WORD] + = (erts_aint_t) (val & 0xffffffff); + dw_aint->sint[ERTS_DW_AINT_HIGH_WORD] + = (erts_aint_t) ((val >> 32) & 0xffffffff); #endif } static ERTS_INLINE void set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) { - ethr_dw_sint_t dw_sint; - val_to_dw_sint(&dw_sint, val); - erts_dw_atomic_set_mb(atmc, &dw_sint); + erts_dw_aint_t dw_aint; + val_to_dw_aint(&dw_aint, val); + erts_dw_atomic_set_mb(atmc, &dw_aint); } static ERTS_INLINE void set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) { - ethr_dw_sint_t dw_sint; - val_to_dw_sint(&dw_sint, val); - erts_dw_atomic_set_nob(atmc, &dw_sint); -} - -static ERTS_INLINE ErtsThrPrgrVal -read_nob(ERTS_THR_PRGR_ATOMIC *atmc) -{ - ethr_dw_sint_t dw_sint; - erts_dw_atomic_read_nob(atmc, &dw_sint); - return erts_thr_prgr_dw_sint_to_val__(&dw_sint); + erts_dw_aint_t dw_aint; + val_to_dw_aint(&dw_aint, val); + erts_dw_atomic_set_nob(atmc, &dw_aint); } static ERTS_INLINE void init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) { - ethr_dw_sint_t dw_sint; - val_to_dw_sint(&dw_sint, val); - erts_dw_atomic_init_nob(atmc, &dw_sint); + erts_dw_aint_t dw_aint; + val_to_dw_aint(&dw_aint, val); + erts_dw_atomic_init_nob(atmc, &dw_aint); } #endif @@ -1222,9 +1211,9 @@ erts_thr_progress_block(void) } void -erts_thr_progress_fatal_error_block(SWord timeout) +erts_thr_progress_fatal_error_block(SWord timeout, + ErtsThrPrgrData *tmp_tpd_bufp) { - ErtsThrPrgrData tpd_buf; ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL); erts_aint32_t bc; SWord time_left = timeout; @@ -1248,7 +1237,7 @@ erts_thr_progress_fatal_error_block(SWord timeout) * since we never complete an unblock after a fatal error * block. */ - tpd = &tpd_buf; + tpd = tmp_tpd_bufp; init_tmp_thr_prgr_data(tpd); } diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h index 68d14174b9..a71724b813 100644 --- a/erts/emulator/beam/erl_thr_progress.h +++ b/erts/emulator/beam/erl_thr_progress.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011. All Rights Reserved. + * Copyright Ericsson AB 2011-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -44,7 +44,6 @@ #define erts_smp_thr_progress_unblock erts_thr_progress_unblock #define erts_smp_thr_progress_is_blocking erts_thr_progress_is_blocking -void erts_thr_progress_fatal_error_block(SWord timeout); void erts_thr_progress_block(void); void erts_thr_progress_unblock(void); int erts_thr_progress_is_blocking(void); @@ -73,6 +72,10 @@ typedef struct { ErtsThrPrgrVal current; } previous; } ErtsThrPrgrData; + +void erts_thr_progress_fatal_error_block(SWord timeout, + ErtsThrPrgrData *tmp_tpd_bufp); + #endif /* ERTS_SMP */ #endif @@ -86,6 +89,7 @@ typedef struct { #ifdef ERTS_SMP #define ERTS_THR_PRGR_VAL_WAITING (~((ErtsThrPrgrVal) 0)) +#define ERTS_THR_PRGR_INVALID (~((ErtsThrPrgrVal) 0)) extern erts_tsd_key_t erts_thr_prgr_data_key__; @@ -127,14 +131,20 @@ void erts_thr_progress_dbg_print_state(void); #ifdef ARCH_32 #define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t -ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_dw_sint_to_val__(ethr_dw_sint_t *dw_sint); +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint); #endif +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc); ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc); +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc); ERTS_GLB_INLINE int erts_thr_progress_is_managed_thread(void); +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current_to_later__(ErtsThrPrgrVal val); +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later_than(ErtsThrPrgrVal val); ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later(void); ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current(void); ERTS_GLB_INLINE int erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2); +ERTS_GLB_INLINE int erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val); +ERTS_GLB_INLINE int erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2); ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -142,33 +152,61 @@ ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val); #ifdef ARCH_64 ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc) +{ + return (ErtsThrPrgrVal) erts_atomic_read_nob(atmc); +} + +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc) { return (ErtsThrPrgrVal) erts_atomic_read_acqb(atmc); } +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc) +{ + return (ErtsThrPrgrVal) erts_atomic_read_mb(atmc); +} + #else /* ARCH_32 */ ERTS_GLB_INLINE ErtsThrPrgrVal -erts_thr_prgr_dw_sint_to_val__(ethr_dw_sint_t *dw_sint) +erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint) { #ifdef ETHR_SU_DW_NAINT_T__ - return (ErtsThrPrgrVal) dw_sint->dw_sint; + return (ErtsThrPrgrVal) dw_aint->dw_sint; #else ErtsThrPrgrVal res; - res = (ErtsThrPrgrVal) ((Uint32) dw_sint->sint[ETHR_DW_SINT_HIGH_WORD]); + res = (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_HIGH_WORD]); res <<= 32; - res |= (ErtsThrPrgrVal) ((Uint32) dw_sint->sint[ETHR_DW_SINT_LOW_WORD]); + res |= (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_LOW_WORD]); return res; #endif } ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc) +{ + erts_dw_aint_t dw_aint; + erts_dw_atomic_read_nob(atmc, &dw_aint); + return erts_thr_prgr_dw_aint_to_val__(&dw_aint); +} + +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc) { - ethr_dw_sint_t dw_sint; - erts_dw_atomic_read_acqb(atmc, &dw_sint); - return erts_thr_prgr_dw_sint_to_val__(&dw_sint); + erts_dw_aint_t dw_aint; + erts_dw_atomic_read_acqb(atmc, &dw_aint); + return erts_thr_prgr_dw_aint_to_val__(&dw_aint); +} + +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc) +{ + erts_dw_aint_t dw_aint; + erts_dw_atomic_read_mb(atmc, &dw_aint); + return erts_thr_prgr_dw_aint_to_val__(&dw_aint); } #endif @@ -181,9 +219,8 @@ erts_thr_progress_is_managed_thread(void) } ERTS_GLB_INLINE ErtsThrPrgrVal -erts_thr_progress_later(void) +erts_thr_progress_current_to_later__(ErtsThrPrgrVal val) { - ErtsThrPrgrVal val = erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current); if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)2))) return ((ErtsThrPrgrVal) 0); else if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)1))) @@ -193,9 +230,26 @@ erts_thr_progress_later(void) } ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_progress_later_than(ErtsThrPrgrVal val) +{ + ERTS_THR_MEMORY_BARRIER; + return erts_thr_progress_current_to_later__(val); +} + +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_progress_later(void) +{ + ErtsThrPrgrVal val = erts_thr_prgr_read_mb__(&erts_thr_prgr__.current); + return erts_thr_progress_current_to_later__(val); +} + +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current(void) { - return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current); + if (erts_thr_progress_is_managed_thread()) + return erts_thr_prgr_read_nob__(&erts_thr_prgr__.current); + else + return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current); } ERTS_GLB_INLINE int @@ -217,13 +271,29 @@ erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val0) } ERTS_GLB_INLINE int -erts_thr_progress_has_reached(ErtsThrPrgrVal val) +erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val) { - ErtsThrPrgrVal current; - current = erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current); - if (current == val) + if (this == val) return 1; - return erts_thr_progress_has_passed__(current, val); + return erts_thr_progress_has_passed__(this, val); +} + +ERTS_GLB_INLINE int +erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2) +{ + if (val1 == val2) + return 0; + if (erts_thr_progress_has_passed__(val1, val2)) + return 1; + else + return -1; +} + +ERTS_GLB_INLINE int +erts_thr_progress_has_reached(ErtsThrPrgrVal val) +{ + ErtsThrPrgrVal current = erts_thr_progress_current(); + return erts_thr_progress_has_reached_this(current, val); } #endif diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c index efb8c635d7..70949ece76 100644 --- a/erts/emulator/beam/erl_thr_queue.c +++ b/erts/emulator/beam/erl_thr_queue.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011. All Rights Reserved. + * Copyright Ericsson AB 2011-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -418,10 +418,9 @@ clean(ErtsThrQ_t *q, int max_ops, int do_notify) } if (q->head.unref_end == (ErtsThrQElement_t *) ilast) - ERTS_THR_MEMORY_BARRIER; + ERTS_SMP_MEMORY_BARRIER; else { q->head.next.unref_end = (ErtsThrQElement_t *) ilast; - ERTS_THR_MEMORY_BARRIER; #ifdef ERTS_SMP q->head.next.thr_progress = erts_thr_progress_later(); #endif diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index 065e7077c0..ee47c98009 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2011. All Rights Reserved. + * Copyright Ericsson AB 2001-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -25,6 +25,235 @@ #ifndef ERL_THREAD_H__ #define ERL_THREAD_H__ +/* + * --- Documentation of atomics and memory barriers -------------------------- + * + * The following explicit memory barriers exist: + * + * - ERTS_THR_MEMORY_BARRIER + * Full memory barrier. Orders both loads, and stores. No + * load or store is allowed to be reordered over the + * barrier. + * - ERTS_THR_WRITE_MEMORY_BARRIER + * Write barrier. Orders *only* stores. These are not + * allowed to be reordered over the barrier. + * - ERTS_THR_READ_MEMORY_BARRIER + * Read barrier. Orders *only* loads. These are not + * allowed to be reordered over the barrier. + * - ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER + * Data dependency read barrier. Orders *only* loads + * according to data dependency across the barrier. + * + * If thread support has been disabled, these barriers will become no-ops. + * + * If the prefix ERTS_THR_ is replaced with ERTS_SMP_, the barriers will + * be enabled only in the SMP enabled runtime system. + * + * --- Atomic operations --- + * + * Atomics operations exist for 32-bit, word size, and double word size + * integers. Function prototypes are listed below. + * + * Each function implementing an atomic operation exist with the following + * implied memory barrier semantics. Not all combinations are useful, but + * all of them exist for simplicity. <B> is suffix in function name: + * + * - <B> - Description + * + * - mb - Full memory barrier. Orders both loads, and + * stores before, and after the atomic operation. + * No load or store is allowed to be reordered + * over the atomic operation. + * - relb - Release barrier. Orders both loads, and + * stores appearing *before* the atomic + * operation. These are not allowed to be + * reordered over the atomic operation. + * - acqb - Acquire barrier. Orders both loads, and stores + * appearing *after* the atomic operation. These + * are not allowed to be reordered over the + * atomic operation. + * - wb - Write barrier. Orders *only* stores. These are + * not allowed to be reordered over the barrier. + * Store in atomic operation is ordered *after* + * the barrier. + * - rb - Read barrier. Orders *only* loads. These are + * not allowed to be reordered over the barrier. + * Load in atomic operation is ordered *before* + * the barrier. + * - ddrb - Data dependency read barrier. Orders *only* + * loads according to data dependency across the + * barrier. Load in atomic operation is ordered + * before the barrier. + * + * If thread support has been disabled, these functions are mapped to + * functions that performs the same operation, but aren't atomic + * and don't imply any memory barriers. + * + * If the atomic operations are prefixed with erts_smp_ instead of only + * erts_ the atomic operations will only be atomic in the SMP enabled + * runtime system, and will be mapped to non-atomic operations without + * memory barriers in the runtime system without SMP support. Atomic + * operations with erts_smp_ prefix should use the atomic types + * erts_smp_atomic32_t, erts_smp_atomic_t, and erts_smp_dw_atomic_t + * instead of erts_atomic32_t, erts_atomic_t, and erts_dw_atomic_t. The + * integer data types erts_aint32_t, erts_aint_t, and erts_dw_atomic_t + * are the same. + * + * --- 32-bit atomic operations --- + * + * The following 32-bit atomic operations exist. <B> should be + * replaced with a supported memory barrier (see above). Note + * that sizeof(erts_atomic32_t) might be larger than 4! + * + * + * Initialize (not necessarily the same as the set operation): + * void erts_atomic32_init_<B>(erts_atomic32_t *atmc, + * erts_aint32_t val); + * + * Set value: + * void erts_atomic32_set_<B>(erts_atomic32_t *atmc, + * erts_aint32_t val); + * + * Read; returns current value: + * erts_aint32_t erts_atomic32_read_<B>(erts_atomic32_t *atmc); + * + * Increment; returns resulting value: + * erts_aint32_t erts_atomic32_inc_read_<B>(erts_atomic32_t *atmc); + * + * Decrement; returns resulting value: + * erts_aint32_t erts_atomic32_dec_read_<B>(erts_atomic32_t *atmc); + * + * Increment: + * void erts_atomic32_inc_<B>(erts_atomic32_t *atmc); + * + * Decrement: + * void erts_atomic32_dec_<B>(erts_atomic32_t *atmc); + * + * Add value; returns resulting value: + * erts_aint32_t erts_atomic32_add_read_<B>(erts_atomic32_t *atmc, + * erts_aint32_t val); + * + * Add value: + * void erts_atomic32_add_<B>(erts_atomic32_t *atmc, + * erts_aint32_t val); + * + * Bitwise-or; returns previous value: + * erts_aint32_t erts_atomic32_read_bor_<B>(erts_atomic32_t *atmc, + * erts_aint32_t val); + * + * Bitwise-and; returns previous value: + * erts_aint32_t erts_atomic32_read_band_<B>(erts_atomic32_t *atmc, + * erts_aint32_t val); + * + * Exchange; returns previous value: + * erts_aint32_t erts_atomic32_xchg_<B>(erts_atomic32_t *atmc, + * erts_aint32_t val); + * + * Compare and exchange; returns previous or current value. If + * returned value equals 'exp' the value was changed to 'new'; + * otherwise not: + * erts_aint32_t erts_atomic32_cmpxchg_<B>(erts_atomic32_t *a, + * erts_aint32_t new, + * erts_aint32_t exp); + * + * --- Word size atomic operations --- + * + * The following word size (same size as sizeof(void *)) atomic + * operations exist. <B> should be replaced with a supported + * memory barrier (see above). Note that sizeof(erts_atomic_t) + * might be larger than sizeof(void *)! + * + * Initialize (not necessarily the same as the set operation): + * void erts_atomic_init_<B>(erts_atomic_t *atmc, + * erts_aint_t val); + * + * Set value; + * void erts_atomic_set_<B>(erts_atomic_t *atmc, + * erts_aint_t val); + * + * Read; returns current value: + * erts_aint_t erts_atomic_read_<B>(erts_atomic_t *atmc); + * + * Increment; returns resulting value: + * erts_aint_t erts_atomic_inc_read_<B>(erts_atomic_t *atmc); + * + * Decrement; returns resulting value: + * erts_aint_t erts_atomic_dec_read_<B>(erts_atomic_t *atmc); + * + * Increment: + * void erts_atomic_inc_<B>(erts_atomic_t *atmc); + * + * Decrement: + * void erts_atomic_dec_<B>(erts_atomic_t *atmc); + * + * Add value; returns resulting value: + * erts_aint_t erts_atomic_add_read_<B>(erts_atomic_t *atmc, + * erts_aint_t val); + * + * Add value: + * void erts_atomic_add_<B>(erts_atomic_t *atmc, + * erts_aint_t val); + * + * Bitwise-or; returns previous value: + * erts_aint_t erts_atomic_read_bor_<B>(erts_atomic_t *atmc, + * erts_aint_t val); + * + * Bitwise-and; returns previous value: + * erts_aint_t erts_atomic_read_band_<B>(erts_atomic_t *atmc, + * erts_aint_t val); + * + * Exchange; returns previous value: + * erts_aint_t erts_atomic_xchg_<B>(erts_atomic_t *atmc, + * erts_aint_t val); + * + * Compare and exchange; returns previous or current value. If + * returned value equals 'exp' the value was changed to 'new'; + * otherwise not: + * erts_aint_t erts_atomic_cmpxchg_<B>(erts_atomic_t *a, + * erts_aint_t new, + * erts_aint_t exp); + * + * --- Double word size atomic operations --- + * + * The following double word atomic operations exist. <B> should be + * replaced with a supported memory barrier (see above). + * + * Note that sizeof(erts_dw_atomic_t) usually is larger than + * 2*sizeof(void *)! + * + * The erts_dw_aint_t data type should be accessed as if it was defined + * like this: + * + * typedef struct { + * erts_aint_t sint[2]; + * } erts_dw_aint_t; + * + * Most significant word is 'sint[ERTS_DW_AINT_HIGH_WORD]' and least + * significant word is 'sint[ERTS_DW_AINT_LOW_WORD]'. + * + * + * Initialize (not necessarily the same as the set operation): + * void erts_dw_atomic_init_<B>(erts_dw_atomic_t *var, + * erts_dw_aint_t *val); + * + * Set; value is written into 'val': + * void erts_dw_atomic_set_<B>(erts_dw_atomic_t *var, + * erts_dw_aint_t *val); + * + * Read; value is written into 'val': + * void erts_dw_atomic_read_<B>(erts_dw_atomic_t *var, + * erts_dw_aint_t *val); + * + * Compare and exchange; returns a value != 0 if exchange was + * made; otherwise 0. 'new_val' contains new value to set. If 'exp_act' + * contains the same value as in memory when the function is called, + * 'new' is written to memory; otherwise, not. If exchange was not + * made, 'exp_act' contains the actual value in memory: + * int erts_dw_atomic_cmpxchg_<B>(erts_dw_atomic_t *var, + * erts_dw_aint_t *new, + * erts_dw_aint_t *exp_act); + */ + #define ERTS_SPIN_BODY ETHR_SPIN_BODY #include "sys.h" @@ -52,6 +281,9 @@ typedef Sint32 erts_no_atomic32_t; #endif #define ERTS_THR_MEMORY_BARRIER ETHR_MEMORY_BARRIER +#define ERTS_THR_WRITE_MEMORY_BARRIER ETHR_WRITE_MEMORY_BARRIER +#define ERTS_THR_READ_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER +#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER ETHR_READ_DEPEND_MEMORY_BARRIER #ifdef ERTS_ENABLE_LOCK_COUNT #define erts_mtx_lock(L) erts_mtx_lock_x(L, __FILE__, __LINE__) @@ -113,6 +345,9 @@ typedef ethr_ts_event erts_tse_t; #define erts_aint32_t ethr_sint32_t #define erts_atomic32_t ethr_atomic32_t +#define ERTS_DW_AINT_HIGH_WORD ETHR_DW_SINT_HIGH_WORD +#define ERTS_DW_AINT_LOW_WORD ETHR_DW_SINT_LOW_WORD + /* spinlock */ typedef struct { ethr_spinlock_t slck; @@ -149,6 +384,9 @@ __decl_noreturn void __noreturn erts_thr_fatal_error(int, char *); #else /* #ifdef USE_THREADS */ #define ERTS_THR_MEMORY_BARRIER +#define ERTS_THR_WRITE_MEMORY_BARRIER +#define ERTS_THR_READ_MEMORY_BARRIER +#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER #define ERTS_THR_OPTS_DEFAULT_INITER 0 typedef int erts_thr_opts_t; @@ -361,18 +599,13 @@ ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set, ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #endif /* #ifdef HAVE_ETHR_SIG_FUNCS */ +#ifdef USE_THREADS + /* - * Functions implementing atomic operations with with no (nob), - * full (mb), acquire (acqb), release (relb), read (rb), and - * write (wb) memory barriers. - * - * If thread support has been disabled, they are mapped to - * functions that performs the same operation, but aren't atomic - * and don't imply memory barriers. + * See "Documentation of atomics and memory barriers" at the top + * of this file for info on atomics. */ -#ifdef USE_THREADS - /* Double word size atomics */ #define erts_dw_atomic_init_nob ethr_dw_atomic_init @@ -395,6 +628,11 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_dw_atomic_read_relb ethr_dw_atomic_read_relb #define erts_dw_atomic_cmpxchg_relb ethr_dw_atomic_cmpxchg_relb +#define erts_dw_atomic_init_ddrb ethr_dw_atomic_init_ddrb +#define erts_dw_atomic_set_ddrb ethr_dw_atomic_set_ddrb +#define erts_dw_atomic_read_ddrb ethr_dw_atomic_read_ddrb +#define erts_dw_atomic_cmpxchg_ddrb ethr_dw_atomic_cmpxchg_ddrb + #define erts_dw_atomic_init_rb ethr_dw_atomic_init_rb #define erts_dw_atomic_set_rb ethr_dw_atomic_set_rb #define erts_dw_atomic_read_rb ethr_dw_atomic_read_rb @@ -463,6 +701,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_xchg_relb ethr_atomic_xchg_relb #define erts_atomic_cmpxchg_relb ethr_atomic_cmpxchg_relb +#define erts_atomic_init_ddrb ethr_atomic_init_ddrb +#define erts_atomic_set_ddrb ethr_atomic_set_ddrb +#define erts_atomic_read_ddrb ethr_atomic_read_ddrb +#define erts_atomic_inc_read_ddrb ethr_atomic_inc_read_ddrb +#define erts_atomic_dec_read_ddrb ethr_atomic_dec_read_ddrb +#define erts_atomic_inc_ddrb ethr_atomic_inc_ddrb +#define erts_atomic_dec_ddrb ethr_atomic_dec_ddrb +#define erts_atomic_add_read_ddrb ethr_atomic_add_read_ddrb +#define erts_atomic_add_ddrb ethr_atomic_add_ddrb +#define erts_atomic_read_bor_ddrb ethr_atomic_read_bor_ddrb +#define erts_atomic_read_band_ddrb ethr_atomic_read_band_ddrb +#define erts_atomic_xchg_ddrb ethr_atomic_xchg_ddrb +#define erts_atomic_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb + #define erts_atomic_init_rb ethr_atomic_init_rb #define erts_atomic_set_rb ethr_atomic_set_rb #define erts_atomic_read_rb ethr_atomic_read_rb @@ -549,6 +801,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_xchg_relb ethr_atomic32_xchg_relb #define erts_atomic32_cmpxchg_relb ethr_atomic32_cmpxchg_relb +#define erts_atomic32_init_ddrb ethr_atomic32_init_ddrb +#define erts_atomic32_set_ddrb ethr_atomic32_set_ddrb +#define erts_atomic32_read_ddrb ethr_atomic32_read_ddrb +#define erts_atomic32_inc_read_ddrb ethr_atomic32_inc_read_ddrb +#define erts_atomic32_dec_read_ddrb ethr_atomic32_dec_read_ddrb +#define erts_atomic32_inc_ddrb ethr_atomic32_inc_ddrb +#define erts_atomic32_dec_ddrb ethr_atomic32_dec_ddrb +#define erts_atomic32_add_read_ddrb ethr_atomic32_add_read_ddrb +#define erts_atomic32_add_ddrb ethr_atomic32_add_ddrb +#define erts_atomic32_read_bor_ddrb ethr_atomic32_read_bor_ddrb +#define erts_atomic32_read_band_ddrb ethr_atomic32_read_band_ddrb +#define erts_atomic32_xchg_ddrb ethr_atomic32_xchg_ddrb +#define erts_atomic32_cmpxchg_ddrb ethr_atomic32_cmpxchg_ddrb + #define erts_atomic32_init_rb ethr_atomic32_init_rb #define erts_atomic32_set_rb ethr_atomic32_set_rb #define erts_atomic32_read_rb ethr_atomic32_read_rb @@ -601,6 +867,11 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_dw_atomic_read_relb erts_no_dw_atomic_read #define erts_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg +#define erts_dw_atomic_init_ddrb erts_no_dw_atomic_init +#define erts_dw_atomic_set_ddrb erts_no_dw_atomic_set +#define erts_dw_atomic_read_ddrb erts_no_dw_atomic_read +#define erts_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg + #define erts_dw_atomic_init_rb erts_no_dw_atomic_init #define erts_dw_atomic_set_rb erts_no_dw_atomic_set #define erts_dw_atomic_read_rb erts_no_dw_atomic_read @@ -669,6 +940,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_xchg_relb erts_no_atomic_xchg #define erts_atomic_cmpxchg_relb erts_no_atomic_cmpxchg +#define erts_atomic_init_ddrb erts_no_atomic_set +#define erts_atomic_set_ddrb erts_no_atomic_set +#define erts_atomic_read_ddrb erts_no_atomic_read +#define erts_atomic_inc_read_ddrb erts_no_atomic_inc_read +#define erts_atomic_dec_read_ddrb erts_no_atomic_dec_read +#define erts_atomic_inc_ddrb erts_no_atomic_inc +#define erts_atomic_dec_ddrb erts_no_atomic_dec +#define erts_atomic_add_read_ddrb erts_no_atomic_add_read +#define erts_atomic_add_ddrb erts_no_atomic_add +#define erts_atomic_read_bor_ddrb erts_no_atomic_read_bor +#define erts_atomic_read_band_ddrb erts_no_atomic_read_band +#define erts_atomic_xchg_ddrb erts_no_atomic_xchg +#define erts_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg + #define erts_atomic_init_rb erts_no_atomic_set #define erts_atomic_set_rb erts_no_atomic_set #define erts_atomic_read_rb erts_no_atomic_read @@ -755,6 +1040,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_xchg_relb erts_no_atomic32_xchg #define erts_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg +#define erts_atomic32_init_ddrb erts_no_atomic32_set +#define erts_atomic32_set_ddrb erts_no_atomic32_set +#define erts_atomic32_read_ddrb erts_no_atomic32_read +#define erts_atomic32_inc_read_ddrb erts_no_atomic32_inc_read +#define erts_atomic32_dec_read_ddrb erts_no_atomic32_dec_read +#define erts_atomic32_inc_ddrb erts_no_atomic32_inc +#define erts_atomic32_dec_ddrb erts_no_atomic32_dec +#define erts_atomic32_add_read_ddrb erts_no_atomic32_add_read +#define erts_atomic32_add_ddrb erts_no_atomic32_add +#define erts_atomic32_read_bor_ddrb erts_no_atomic32_read_bor +#define erts_atomic32_read_band_ddrb erts_no_atomic32_read_band +#define erts_atomic32_xchg_ddrb erts_no_atomic32_xchg +#define erts_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg + #define erts_atomic32_init_rb erts_no_atomic32_set #define erts_atomic32_set_rb erts_no_atomic32_set #define erts_atomic32_read_rb erts_no_atomic32_read diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 70c3e7612f..4261cd03be 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -125,8 +125,13 @@ do { \ enqueue_sys_msg_unlocked(SYS_MSG_TYPE_TRACE, (FPID), (TPID), (MSG), (BP)); \ } while(0) #else +#ifdef USE_VM_PROBES #define ERTS_ENQ_TRACE_MSG(FPID, TPROC, MSG, BP) \ - erts_queue_message((TPROC), NULL, (BP), (MSG), NIL) + erts_queue_message((TPROC), NULL, (BP), (MSG), NIL, NIL) +#else +#define ERTS_ENQ_TRACE_MSG(FPID, TPROC, MSG, BP) \ + erts_queue_message((TPROC), NULL, (BP), (MSG), NIL) +#endif #endif /* @@ -544,7 +549,7 @@ send_to_port(Process *c_p, Eterm message, */ static void -profile_send(Eterm profiler, Eterm message) { +profile_send(Eterm from, Eterm message) { Uint sz = 0; ErlHeapFragment *bp = NULL; Uint *hp = NULL; @@ -552,6 +557,11 @@ profile_send(Eterm profiler, Eterm message) { Process *profile_p = NULL; ErlOffHeap *off_heap = NULL; + Eterm profiler = erts_get_system_profile(); + + /* do not profile profiler pid */ + if (from == profiler) return; + if (is_internal_port(profiler)) { Port *profiler_port = NULL; @@ -578,7 +588,11 @@ profile_send(Eterm profiler, Eterm message) { hp = erts_alloc_message_heap(sz, &bp, &off_heap, profile_p, 0); msg = copy_struct(message, sz, &hp, &bp->off_heap); - erts_queue_message(profile_p, NULL, bp, msg, NIL); + erts_queue_message(profile_p, NULL, bp, msg, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); } } @@ -989,9 +1003,13 @@ seq_trace_update_send(Process *p) { Eterm seq_tracer = erts_get_system_seq_tracer(); ASSERT((is_tuple(SEQ_TRACE_TOKEN(p)) || is_nil(SEQ_TRACE_TOKEN(p)))); - if ( (p->id == seq_tracer) || (SEQ_TRACE_TOKEN(p) == NIL)) + if ( (p->id == seq_tracer) || (SEQ_TRACE_TOKEN(p) == NIL) +#ifdef USE_VM_PROBES + || (SEQ_TRACE_TOKEN(p) == am_have_dt_utag) +#endif + ) { return 0; - + } SEQ_TRACE_TOKEN_SENDER(p) = p->id; /* Internal pid */ SEQ_TRACE_TOKEN_SERIAL(p) = make_small(++(p -> seq_trace_clock)); @@ -1173,7 +1191,11 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SEQTRACE, NIL, NIL, mess, bp); erts_smp_mtx_unlock(&smq_mtx); #else - erts_queue_message(tracer, NULL, bp, mess, NIL); /* trace_token must be NIL here */ + erts_queue_message(tracer, NULL, bp, mess, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); /* trace_token must be NIL here */ #endif } } @@ -2464,7 +2486,11 @@ monitor_long_gc(Process *p, Uint time) { #ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->id, NIL, msg, bp); #else - erts_queue_message(monitor_p, NULL, bp, msg, NIL); + erts_queue_message(monitor_p, NULL, bp, msg, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); #endif } @@ -2536,7 +2562,11 @@ monitor_large_heap(Process *p) { #ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->id, NIL, msg, bp); #else - erts_queue_message(monitor_p, NULL, bp, msg, NIL); + erts_queue_message(monitor_p, NULL, bp, msg, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); #endif } @@ -2566,7 +2596,11 @@ monitor_generic(Process *p, Eterm type, Eterm spec) { #ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->id, NIL, msg, bp); #else - erts_queue_message(monitor_p, NULL, bp, msg, NIL); + erts_queue_message(monitor_p, NULL, bp, msg, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); #endif } @@ -2581,7 +2615,6 @@ profile_scheduler(Eterm scheduler_id, Eterm state) { Uint Ms, s, us; #ifndef ERTS_SMP - Eterm profiler; #define LOCAL_HEAP_SIZE (4 + 7) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2616,8 +2649,7 @@ profile_scheduler(Eterm scheduler_id, Eterm state) { make_small(active_sched), timestamp); hp += 7; #ifndef ERTS_SMP - profiler = erts_get_system_profile(); - profile_send(profiler, msg); + profile_send(NIL, msg); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE #else @@ -2632,7 +2664,6 @@ profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint M Eterm *hp, msg, timestamp; #ifndef ERTS_SMP - Eterm profiler; #define LOCAL_HEAP_SIZE (4 + 7) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2653,8 +2684,7 @@ profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint M timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4; msg = TUPLE6(hp, am_profile, am_scheduler, scheduler_id, state, no_schedulers, timestamp); hp += 7; #ifndef ERTS_SMP - profiler = erts_get_system_profile(); - profile_send(profiler, msg); + profile_send(NIL, msg); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE #else @@ -2897,7 +2927,6 @@ profile_runnable_port(Port *p, Eterm status) { Eterm count = make_small(0); #ifndef ERTS_SMP - Eterm profiler; #define LOCAL_HEAP_SIZE (4 + 6) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); @@ -2922,12 +2951,11 @@ profile_runnable_port(Port *p, Eterm status) { msg = TUPLE5(hp, am_profile, p->id, status, count, timestamp); hp += 6; #ifndef ERTS_SMP - profiler = erts_get_system_profile(); - profile_send(profiler, msg); + profile_send(p->id, msg); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE #else - enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp); + enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, p->id, NIL, msg, bp); #endif erts_smp_mtx_unlock(&smq_mtx); } @@ -2938,7 +2966,6 @@ profile_runnable_proc(Process *p, Eterm status){ Uint Ms, s, us; Eterm *hp, msg, timestamp; Eterm where = am_undefined; - Eterm profiler; #ifndef ERTS_SMP #define LOCAL_HEAP_SIZE (4 + 6 + 4) @@ -2951,12 +2978,6 @@ profile_runnable_proc(Process *p, Eterm status){ ErlHeapFragment *bp; Uint hsz = 4 + 6 + 4; #endif - profiler = erts_get_system_profile(); - - /* Do not profile profiler pid */ - if (profiler == p->id) { - return; - } if (!p->current) { p->current = find_function_from_pc(p->i); @@ -2983,11 +3004,11 @@ profile_runnable_proc(Process *p, Eterm status){ timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4; msg = TUPLE5(hp, am_profile, p->id, status, where, timestamp); hp += 6; #ifndef ERTS_SMP - profile_send(profiler, msg); + profile_send(p->id, msg); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE #else - enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp); + enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, p->id, NIL, msg, bp); #endif erts_smp_mtx_unlock(&smq_mtx); } @@ -3365,7 +3386,11 @@ sys_msg_dispatcher_func(void *unused) } else { queue_proc_msg: - erts_queue_message(proc,&proc_locks,smqp->bp,smqp->msg,NIL); + erts_queue_message(proc,&proc_locks,smqp->bp,smqp->msg,NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); #ifdef DEBUG_PRINTOUTS erts_fprintf(stderr, "delivered\n"); #endif diff --git a/erts/emulator/beam/erlang_dtrace.d b/erts/emulator/beam/erlang_dtrace.d new file mode 100644 index 0000000000..587e51cb67 --- /dev/null +++ b/erts/emulator/beam/erlang_dtrace.d @@ -0,0 +1,726 @@ +/* + * %CopyrightBegin% + * + * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011. + * All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * A note on probe naming: if "__" appears in a provider probe + * definition, then two things happen during compilation: + * + * 1. The "__" will turn into a hypen, "-", for the probe name. + * 2. The "__" will turn into a single underscore, "_", for the + * macro names and function definitions that the compiler and + * C developers will see. + * + * We'll try to use the following naming convention. We're a bit + * limited because, as a USDT probe, we can only specify the 4th part + * of the probe name, e.g. erlang*:::mumble. The 2nd part of the + * probe name is always going to be "beam" or "beam.smp", and the 3rd + * part of the probe name will always be the name of the function + * that's calling the probe. + * + * So, all probes will be have names defined in this file using the + * convention category__name or category__sub_category__name. This + * will translate to probe names of category-name or + * category-sub_category-name. + * + * Each of "category", "sub_category", and "name" may have underscores + * but may not have hyphens. + */ + +provider erlang { + /** + * Fired when a message is sent from one local process to another. + * + * NOTE: The 'size' parameter is in machine-dependent words and + * that the actual size of any binary terms in the message + * are not included. + * + * @param sender the PID (string form) of the sender + * @param receiver the PID (string form) of the receiver + * @param size the size of the message being delivered (words) + * @param token_label for the sender's sequential trace token + * @param token_previous count for the sender's sequential trace token + * @param token_current count for the sender's sequential trace token + */ + probe message__send(char *sender, char *receiver, uint32_t size, + int token_label, int token_previous, int token_current); + + /** + * Fired when a message is sent from a local process to a remote process. + * + * NOTE: The 'size' parameter is in machine-dependent words and + * that the actual size of any binary terms in the message + * are not included. + * + * @param sender the PID (string form) of the sender + * @param node_name the Erlang node name (string form) of the receiver + * @param receiver the PID/name (string form) of the receiver + * @param size the size of the message being delivered (words) + * @param token_label for the sender's sequential trace token + * @param token_previous count for the sender's sequential trace token + * @param token_current count for the sender's sequential trace token + */ + probe message__send__remote(char *sender, char *node_name, char *receiver, + uint32_t size, + int token_label, int token_previous, int token_current); + + /** + * Fired when a message is queued to a local process. This probe + * will not fire if the sender's pid == receiver's pid. + * + * NOTE: The 'size' parameter is in machine-dependent words and + * that the actual size of any binary terms in the message + * are not included. + * + * NOTE: In cases of messages in external format (i.e. from another + * Erlang node), we probably don't know the message size + * without performing substantial extra computation. To + * avoid the extra CPU overhead, the message size may be + * reported as -1, which can appear to a D script as 4294967295. + * + * @param receiver the PID (string form) of the receiver + * @param size the size of the message being delivered (words) + * @param queue_len length of the queue of the receiving process + * @param token_label for the sender's sequential trace token + * @param token_previous count for the sender's sequential trace token + * @param token_current count for the sender's sequential trace token + */ + probe message__queued(char *receiver, uint32_t size, uint32_t queue_len, + int token_label, int token_previous, int token_current); + + /** + * Fired when a message is 'receive'd by a local process and removed + * from its mailbox. + * + * NOTE: The 'size' parameter is in machine-dependent words and + * that the actual size of any binary terms in the message + * are not included. + * + * NOTE: In cases of messages in external format (i.e. from another + * Erlang node), we probably don't know the message size + * without performing substantial extra computation. To + * avoid the extra CPU overhead, the message size may be + * reported as -1, which can appear to a D script as 4294967295. + * + * @param receiver the PID (string form) of the receiver + * @param size the size of the message being delivered (words) + * @param queue_len length of the queue of the receiving process + * @param token_label for the sender's sequential trace token + * @param token_previous count for the sender's sequential trace token + * @param token_current count for the sender's sequential trace token + */ + probe message__receive(char *receiver, uint32_t size, uint32_t queue_len, + int token_label, int token_previous, int token_current); + + /** + * Fired when an Eterm structure is being copied. + * + * NOTE: Due to the placement of this probe, the process ID of + * owner of the Eterm is not available. + * + * @param size the size of the structure + */ + probe copy__struct(uint32_t size); + + /** + * Fired when an Eterm is being copied onto a process. + * + * @param proc the PID (string form) of the recipient process + * @param size the size of the structure + */ + probe copy__object(char *proc, uint32_t size); + + /* PID, Module, Function, Arity */ + + /** + * Fired whenever a user function is being called locally. + * + * @param p the PID (string form) of the process + * @param mfa the m:f/a of the function + * @param depth the stack depth + */ + probe local__function__entry(char *p, char *mfa, int depth); + + /** + * Fired whenever a user function is called externally + * (through an export entry). + * + * @param p the PID (string form) of the process + * @param mfa the m:f/a of the function + * @param depth the stack depth + */ + probe global__function__entry(char *p, char *mfa, int depth); + + /** + * Fired whenever a user function returns. + * + * @param p the PID (string form) of the process + * @param mfa the m:f/a of the function + * @param depth the stack depth + */ + probe function__return(char *p, char *mfa, int depth); + + /** + * Fired whenever a Built In Function is called. + * + * @param p the PID (string form) of the process + * @param mfa the m:f/a of the function + */ + probe bif__entry(char *p, char *mfa); + + /** + * Fired whenever a Built In Function returns. + * + * @param p the PID (string form) of the process + * @param mfa the m:f/a of the function + */ + probe bif__return(char *p, char *mfa); + + /** + * Fired whenever a Native Function is called. + * + * @param p the PID (string form) of the process + * @param mfa the m:f/a of the function + */ + probe nif__entry(char *p, char *mfa); + + /** + * Fired whenever a Native Function returns. + * + * @param p the PID (string form) of the process + * @param mfa the m:f/a of the function + */ + probe nif__return(char *p, char *mfa); + + /** + * Fired when a major GC is starting. + * + * @param p the PID (string form) of the exiting process + * @param need the number of words needed on the heap + */ + probe gc_major__start(char *p, int need); + + /** + * Fired when a minor GC is starting. + * + * @param p the PID (string form) of the exiting process + * @param need the number of words needed on the heap + */ + probe gc_minor__start(char *p, int need); + + /** + * Fired when a major GC is starting. + * + * @param p the PID (string form) of the exiting process + * @param reclaimed the amount of space reclaimed + */ + probe gc_major__end(char *p, int reclaimed); + + /** + * Fired when a minor GC is starting. + * + * @param p the PID (string form) of the exiting process + * @param reclaimed the amount of space reclaimed + */ + probe gc_minor__end(char *p, int reclaimed); + + /** + * Fired when a process is spawned. + * + * @param p the PID (string form) of the new process. + * @param mfa the m:f/a of the function + */ + probe process__spawn(char *p, char *mfa); + + /** + * Fired when a process is exiting. + * + * @param p the PID (string form) of the exiting process + * @param reason the reason for the exit (may be truncated) + */ + probe process__exit(char *p, char *reason); + + /** + * Fired when exit signal is delivered to a local process. + * + * @param sender the PID (string form) of the exiting process + * @param receiver the PID (string form) of the process receiving EXIT signal + * @param reason the reason for the exit (may be truncated) + */ + probe process__exit_signal(char *sender, char *receiver, char *reason); + + /** + * Fired when exit signal is delivered to a remote process. + * + * @param sender the PID (string form) of the exiting process + * @param node_name the Erlang node name (string form) of the receiver + * @param receiver the PID (string form) of the process receiving EXIT signal + * @param reason the reason for the exit (may be truncated) + * @param token_label for the sender's sequential trace token + * @param token_previous count for the sender's sequential trace token + * @param token_current count for the sender's sequential trace token + */ + probe process__exit_signal__remote(char *sender, char *node_name, + char *receiver, char *reason, + int token_label, int token_previous, int token_current); + + /** + * Fired when a process is scheduled. + * + * @param p the PID (string form) of the newly scheduled process + * @param mfa the m:f/a of the function it should run next + */ + probe process__scheduled(char *p, char *mfa); + + /** + * Fired when a process is unscheduled. + * + * @param p the PID (string form) of the process that has been + * unscheduled. + */ + probe process__unscheduled(char *p); + + /** + * Fired when a process goes into hibernation. + * + * @param p the PID (string form) of the process entering hibernation + * @param mfa the m:f/a of the location to resume + */ + probe process__hibernate(char *p, char *mfa); + + /** + * Fired when a process is unblocked after a port has been unblocked. + * + * @param p the PID (string form) of the process that has been + * unscheduled. + * @param port the port that is no longer busy (i.e., is now unblocked) + */ + probe process__port_unblocked(char *p, char *port); + + /** + * Fired when process' heap is growing. + * + * @param p the PID (string form) + * @param old_size the size of the old heap + * @param new_size the size of the new heap + */ + probe process__heap_grow(char *p, int old_size, int new_size); + + /** + * Fired when process' heap is shrinking. + * + * @param p the PID (string form) + * @param old_size the size of the old heap + * @param new_size the size of the new heap + */ + probe process__heap_shrink(char *p, int old_size, int new_size); + + /* network distribution */ + + /** + * Fired when network distribution event monitor events are triggered. + * + * @param node the name of the reporting node + * @param what the type of event, e.g., nodeup, nodedown + * @param monitored_node the name of the monitored node + * @param type the type of node, e.g., visible, hidden + * @param reason the reason term, e.g., normal, connection_closed, term() + */ + probe dist__monitor(char *node, char *what, char *monitored_node, + char *type, char *reason); + + /** + * Fired when network distribution port is busy (i.e. blocked), + * usually due to the remote node not consuming distribution + * data quickly enough. + * + * @param node the name of the reporting node + * @param port the port ID of the busy port + * @param remote_node the name of the remote node. + * @param pid the PID (string form) of the local process that has + * become unschedulable until the port becomes unblocked. + */ + probe dist__port_busy(char *node, char *port, char *remote_node, + char *pid); + + /** + * Fired when network distribution's driver's "output" callback is called + * + * @param node the name of the reporting node + * @param port the port ID of the busy port + * @param remote_node the name of the remote node. + * @param bytes the number of bytes written + */ + probe dist__output(char *node, char *port, char *remote_node, int bytes); + + /** + * Fired when network distribution's driver's "outputv" callback is called + * + * @param node the name of the reporting node + * @param port the port ID of the busy port + * @param remote_node the name of the remote node. + * @param bytes the number of bytes written + */ + probe dist__outputv(char *node, char *port, char *remote_node, int bytes); + + /** + * Fired when network distribution port is no longer busy (i.e. blocked). + * + * NOTE: This probe may fire multiple times after the same single + * dist-port_busy probe firing. + * + * @param node the name of the reporting node + * @param port the port ID of the busy port + * @param remote_node the name of the remote node. + */ + probe dist__port_not_busy(char *node, char *port, char *remote_node); + + /* ports */ + + /** + * Fired when new port is opened. + * + * @param process the PID (string form) + * @param port_name the string used when the port was opened + * @param port the Port (string form) of the new port + */ + probe port__open(char *process, char *port_name, char *port); + + /** + * Fired when port_command is issued. + * + * @param process the PID (string form) + * @param port the Port (string form) + * @param port_name the string used when the port was opened + * @param command_type type of the issued command, one of: "close", "command" or "connect" + */ + probe port__command(char *process, char *port, char *port_name, char *command_type); + + /** + * Fired when port_control is issued. + * + * @param process the PID (string form) + * @param port the Port (string form) + * @param port_name the string used when the port was opened + * @param command_no command number that has been issued to the port + */ + probe port__control(char *process, char *port, char *port_name, int command_no); + + /** + * Fired when port is closed via port_close/1 (reason = 'normal') + * or is sent an exit signal. + * + * @param process the PID (string form) + * @param port the Port (string form) + * @param port_name the string used when the port was opened + * @param reason Erlang term representing the exit signal, e.g. 'normal' + */ + probe port__exit(char *process, char *port, char *port_name, + char *new_process); + + /** + * Fired when port_connect is issued. + * + * @param process the PID (string form) of the current port owner + * @param port the Port (string form) + * @param port_name the string used when the port was opened + * @param new_process the PID (string form) of the new port owner + */ + probe port__connect(char *process, char *port, char *port_name, + char *new_process); + + /** + * Fired when a port is busy (i.e. blocked) + * + * @param port the port ID of the busy port + */ + probe port__busy(char *port); + + /** + * Fired when a port is no longer busy (i.e. no longer blocked) + * + * @param port the port ID of the not busy port + */ + probe port__not_busy(char *port); + + /* drivers */ + + /** + * Fired when drivers's "init" callback is called. + * + * @param name the name of the driver + * @param major the major version number + * @param minor the minor version number + * @param flags the flags argument + */ + probe driver__init(char *name, int major, int minor, int flags); + + /** + * Fired when drivers's "start" callback is called. + * + * @param process the PID (string form) of the calling process + * @param name the name of the driver + * @param port the Port (string form) of the driver's port + */ + probe driver__start(char *process, char *name, char *port); + + /** + * Fired when drivers's "stop" callback is called. + * + * @param process the PID (string form) of the calling process + * @param name the name of the driver + * @param port the Port (string form) of the driver's port + */ + probe driver__stop(char *process, char *name, char *port); + + /** + * Fired when drivers's "finish" callback is called. + * + * @param name the name of the driver + */ + probe driver__finish(char *name); + + /** + * Fired when drivers's "flush" callback is called. + * + * @param process the PID (string form) + * @param port the Port (string form) + * @param port_name the string used when the port was opened + */ + probe driver__flush(char *process, char *port, char *port_name); + + /** + * Fired when driver's "output" callback is called + * + * @param process the PID (string form) + * @param port the Port (string form) + * @param port_name the string used when the port was opened + * @param bytes the number of bytes written + */ + probe driver__output(char *node, char *port, char *port_name, int bytes); + + /** + * Fired when driver's "outputv" callback is called + * + * @param process the PID (string form) + * @param port the Port (string form) + * @param port_name the string used when the port was opened + * @param bytes the number of bytes written + */ + probe driver__outputv(char *node, char *port, char *port_name, int bytes); + + /** + * Fired when driver's "control" callback is called + * + * @param process the PID (string form) + * @param port the Port (string form) + * @param port_name the string used when the port was opened + * @param command the command # + * @param bytes the number of bytes written + */ + probe driver__control(char *node, char *port, char *port_name, + int command, int bytes); + + /** + * Fired when driver's "call" callback is called + * + * @param process the PID (string form) + * @param port the Port (string form) + * @param port_name the string used when the port was opened + * @param command the command # + * @param bytes the number of bytes written + */ + probe driver__call(char *node, char *port, char *port_name, + int command, int bytes); + + /** + * Fired when driver's "event" callback is called + * + * @param process the PID (string form) + * @param port the Port (string form) + * @param port_name the string used when the port was opened + */ + probe driver__event(char *node, char *port, char *port_name); + + /** + * Fired when driver's "ready_input" callback is called + * + * @param process the PID (string form) + * @param port the Port (string form) + * @param port_name the string used when the port was opened + */ + probe driver__ready_input(char *node, char *port, char *port_name); + + /** + * Fired when driver's "read_output" callback is called + * + * @param process the PID (string form) + * @param port the Port (string form) + * @param port_name the string used when the port was opened + */ + probe driver__ready_output(char *node, char *port, char *port_name); + + /** + * Fired when driver's "timeout" callback is called + * + * @param process the PID (string form) + * @param port the Port (string form) + * @param port_name the string used when the port was opened + */ + probe driver__timeout(char *node, char *port, char *port_name); + + /** + * Fired when drivers's "ready_async" callback is called. + * + * @param process the PID (string form) + * @param port the Port (string form) + * @param port_name the string used when the port was opened + */ + probe driver__ready_async(char *process, char *port, char *port_name); + + /** + * Fired when driver's "process_exit" callback is called + * + * @param process the PID (string form) + * @param port the Port (string form) + * @param port_name the string used when the port was opened + */ + probe driver__process_exit(char *node, char *port, char *port_name); + + /** + * Fired when driver's "stop_select" callback is called + * + * @param name the name of the driver + */ + probe driver__stop_select(char *name); + + + /* Async driver pool */ + + /** + * Show the post-add length of the async driver thread pool member's queue. + * + * NOTE: The port name is not available: additional lock(s) must + * be acquired in order to get the port name safely in an SMP + * environment. The same is true for the aio__pool_get probe. + * + * @param port the Port (string form) + * @param new queue length + */ + probe aio_pool__add(char *, int); + + /** + * Show the post-get length of the async driver thread pool member's queue. + * + * @param port the Port (string form) + * @param new queue length + */ + probe aio_pool__get(char *, int); + + /* Probes for efile_drv.c */ + + /** + * Entry into the efile_drv.c file I/O driver + * + * For a list of command numbers used by this driver, see the section + * "Guide to probe arguments" in ../../../README.md. That section + * also contains explanation of the various integer and string + * arguments that may be present when any particular probe fires. + * + * NOTE: Not all Linux platforms (using SystemTap) can support + * arguments beyond arg9. + * + * + * TODO: Adding the port string, args[10], is a pain. Making that + * port string available to all the other efile_drv.c probes + * will be more pain. Is the pain worth it? If yes, then + * add them everywhere else and grit our teeth. If no, then + * rip it out. + * + * @param thread-id number of the scheduler Pthread arg0 + * @param tag number: {thread-id, tag} uniquely names a driver operation + * @param user-tag string arg2 + * @param command number arg3 + * @param string argument 1 arg4 + * @param string argument 2 arg5 + * @param integer argument 1 arg6 + * @param integer argument 2 arg7 + * @param integer argument 3 arg8 + * @param integer argument 4 arg9 + * @param port the port ID of the busy port args[10] + */ + probe efile_drv__entry(int, int, char *, int, char *, char *, + int64_t, int64_t, int64_t, int64_t, char *); + + /** + * Entry into the driver's internal work function. Computation here + * is performed by a async worker pool Pthread. + * + * @param thread-id number + * @param tag number + * @param command number + */ + probe efile_drv__int_entry(int, int, int); + + /** + * Return from the driver's internal work function. + * + * @param thread-id number + * @param tag number + * @param command number + */ + probe efile_drv__int_return(int, int, int); + + /** + * Return from the efile_drv.c file I/O driver + * + * @param thread-id number arg0 + * @param tag number arg1 + * @param user-tag string arg2 + * @param command number arg3 + * @param Success? 1 is success, 0 is failure arg4 + * @param If failure, the errno of the error. arg5 + */ + probe efile_drv__return(int, int, char *, int, int, int); + +/* + * NOTE: + * For formatting int64_t arguments within a D script, see: + * + * http://mail.opensolaris.org/pipermail/dtrace-discuss/2006-November/002830.html + * Summary: + * "1) you don't need the 'l' printf() modifiers with DTrace ever" + */ + +/* + * NOTE: For file_drv_return + SMP + R14B03 (and perhaps other + * releases), the sched-thread-id will be the same as the + * work-thread-id: erl_async.c's async_main() function + * will call the asynchronous invoke function and then + * immediately call the drivers ready_async function while + * inside the same I/O worker pool thread. + * For R14B03's source, see erl_async.c lines 302-317. + */ +}; + +#pragma D attributes Evolving/Evolving/Common provider erlang provider +#pragma D attributes Private/Private/Common provider erlang module +#pragma D attributes Private/Private/Common provider erlang function +#pragma D attributes Evolving/Evolving/Common provider erlang name +#pragma D attributes Evolving/Evolving/Common provider erlang args diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 152dbcf085..9d52ed4e98 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -3118,6 +3118,9 @@ decoded_size(byte *ep, byte* endp, int internal_tags) case LARGE_BIG_EXT: CHKSIZE(4); n = get_int32(ep); + if (n > BIG_ARITY_MAX*sizeof(ErtsDigit)) { + return -1; + } SKIP2(n,4+1); /* skip, size,sign,digits */ heap_size += 1+1+(n+sizeof(Eterm)-1)/sizeof(Eterm); /* XXX: 1 too much? */ break; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index f1335f600d..b000e2c5d4 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -806,6 +806,8 @@ do { \ /* Port uses port specific locking (opposed to driver specific locking) */ #define ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK ((Uint32) (1 << 13)) #define ERTS_PORT_SFLG_INVALID ((Uint32) (1 << 14)) +/* Last port to terminate halts the emulator */ +#define ERTS_PORT_SFLG_HALT ((Uint32) (1 << 15)) #ifdef DEBUG /* Only debug: make sure all flags aren't cleared unintentionally */ #define ERTS_PORT_SFLG_PORT_DEBUG ((Uint32) (1 << 31)) @@ -899,14 +901,9 @@ void loaded(int, void *); /* config.c */ __decl_noreturn void __noreturn erl_exit(int n, char*, ...); -__decl_noreturn void __noreturn erl_exit0(char *, int, int n, char*, ...); +__decl_noreturn void __noreturn erl_exit_flush_async(int n, char*, ...); void erl_error(char*, va_list); -#define ERL_EXIT0(n,f) erl_exit0(__FILE__, __LINE__, n, f) -#define ERL_EXIT1(n,f,a) erl_exit0(__FILE__, __LINE__, n, f, a) -#define ERL_EXIT2(n,f,a,b) erl_exit0(__FILE__, __LINE__, n, f, a, b) -#define ERL_EXIT3(n,f,a,b,c) erl_exit0(__FILE__, __LINE__, n, f, a, b, c) - /* copy.c */ void init_copy(void); Eterm copy_object(Eterm, Process*); @@ -1977,4 +1974,46 @@ erts_alloc_message_heap(Uint size, # define UseTmpHeapNoproc(Size) /* Nothing */ # define UnUseTmpHeapNoproc(Size) /* Nothing */ #endif /* HEAP_ON_C_STACK */ + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +#include "dtrace-wrapper.h" + +ERTS_GLB_INLINE void +dtrace_pid_str(Eterm pid, char *process_buf) +{ + erts_snprintf(process_buf, DTRACE_TERM_BUF_SIZE, "<%lu.%lu.%lu>", + pid_channel_no(pid), + pid_number(pid), + pid_serial(pid)); +} + +ERTS_GLB_INLINE void +dtrace_proc_str(Process *process, char *process_buf) +{ + dtrace_pid_str(process->id, process_buf); +} + +ERTS_GLB_INLINE void +dtrace_port_str(Port *port, char *port_buf) +{ + erts_snprintf(port_buf, DTRACE_TERM_BUF_SIZE, "#Port<%lu.%lu>", + port_channel_no(port->id), + port_number(port->id)); +} + +ERTS_GLB_INLINE void +dtrace_fun_decode(Process *process, + Eterm module, Eterm function, int arity, + char *process_buf, char *mfa_buf) +{ + if (process_buf) { + dtrace_proc_str(process, process_buf); + } + + erts_snprintf(mfa_buf, DTRACE_TERM_BUF_SIZE, "%T:%T/%d", + module, function, arity); +} +#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ + #endif /* !__GLOBAL_H__ */ diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 49cd0e5f53..8a2a43bebd 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -43,6 +43,7 @@ #include "erl_version.h" #include "error.h" #include "erl_async.h" +#include "dtrace-wrapper.h" extern ErlDrvEntry fd_driver_entry; extern ErlDrvEntry vanilla_driver_entry; @@ -180,6 +181,20 @@ typedef struct line_buf_context { #define LINEBUF_INITIAL 100 +#ifdef USE_VM_PROBES +#define DTRACE_FORMAT_COMMON_PID_AND_PORT(PID, PORT) \ + DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); \ + \ + dtrace_pid_str((PID), process_str); \ + dtrace_port_str((PORT), port_str); +#define DTRACE_FORMAT_COMMON_PROC_AND_PORT(PID, PORT) \ + DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); \ + \ + dtrace_proc_str((PID), process_str); \ + dtrace_port_str((PORT), port_str); +#endif /* The 'number' field in a port now has two parts: the lowest bits contain the index in the port table, and the higher bits are a counter @@ -639,6 +654,12 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ trace_sched_ports_where(port, am_in, am_start); } port->caller = pid; +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(driver_start)) { + DTRACE_FORMAT_COMMON_PID_AND_PORT(pid, port) + DTRACE3(driver_start, process_str, driver->name, port_str); + } +#endif fpe_was_unmasked = erts_block_fpe(); drv_data = (*driver->start)((ErlDrvPort)(port_ix), name, opts); @@ -743,7 +764,7 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */ return (ErlDrvTermData) -1; /* pid does not exist */ } if ((port_num = get_free_port()) < 0) { - errno = ENFILE; + errno = SYSTEM_LIMIT; erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); erts_smp_mtx_unlock(&erts_driver_list_lock); return (ErlDrvTermData) -1; @@ -818,6 +839,11 @@ erts_smp_xports_unlock(Port *prt) #define SET_VEC(iov, bv, bin, ptr, len, vlen) do { \ (iov)->iov_base = (ptr); \ (iov)->iov_len = (len); \ + if (sizeof((iov)->iov_len) < sizeof(len) \ + /* Check if (len) overflowed (iov)->iov_len */ \ + && ((len) >> (sizeof((iov)->iov_len)*CHAR_BIT)) != 0) { \ + goto L_overflow; \ + } \ *(bv)++ = (bin); \ (iov)++; \ (vlen)++; \ @@ -1146,15 +1172,31 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) ivp[0].iov_len = 0; bvp[0] = NULL; ev.vsize = io_list_to_vec(list, ivp+1, bvp+1, cbin, blimit); + if (ev.vsize < 0) { + if (ivp != iv) { + erts_free(ERTS_ALC_T_TMP, (void *) ivp); + } + if (bvp != bv) { + erts_free(ERTS_ALC_T_TMP, (void *) bvp); + } + driver_free_binary(cbin); + goto bad_value; + } ev.vsize++; #if 0 /* This assertion may say something useful, but it can be falsified during the emulator test suites. */ - ASSERT((ev.vsize >= 0) && (ev.vsize == vsize)); + ASSERT(ev.vsize == vsize); #endif ev.size = size; /* total size */ ev.iov = ivp; ev.binv = bvp; +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(driver_outputv)) { + DTRACE_FORMAT_COMMON_PID_AND_PORT(caller_id, p) + DTRACE4(driver_outputv, process_str, port_str, p->name, size); + } +#endif fpe_was_unmasked = erts_block_fpe(); (*drv->outputv)((ErlDrvData)p->drv_data, &ev); erts_unblock_fpe(fpe_was_unmasked); @@ -1174,8 +1216,21 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) buf = erts_alloc(ERTS_ALC_T_TMP, size+1); r = io_list_to_buf(list, buf, size); +#ifdef USE_VM_PROBES + if(DTRACE_ENABLED(port_command)) { + DTRACE_FORMAT_COMMON_PID_AND_PORT(caller_id, p) + DTRACE4(port_command, process_str, port_str, p->name, "command"); + } +#endif + if (r >= 0) { size -= r; +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(driver_output)) { + DTRACE_FORMAT_COMMON_PID_AND_PORT(caller_id, p) + DTRACE4(driver_output, process_str, port_str, p->name, size); + } +#endif fpe_was_unmasked = erts_block_fpe(); (*drv->output)((ErlDrvData)p->drv_data, buf, size); erts_unblock_fpe(fpe_was_unmasked); @@ -1199,6 +1254,12 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) */ buf = erts_alloc(ERTS_ALC_T_TMP, size+1); r = io_list_to_buf(list, buf, size); +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(driver_output)) { + DTRACE_FORMAT_COMMON_PID_AND_PORT(caller_id, p) + DTRACE4(driver_output, process_str, port_str, p->name, size); + } +#endif fpe_was_unmasked = erts_block_fpe(); (*drv->output)((ErlDrvData)p->drv_data, buf, size); erts_unblock_fpe(fpe_was_unmasked); @@ -1514,7 +1575,11 @@ deliver_result(Eterm sender, Eterm pid, Eterm res) hp = erts_alloc_message_heap(sz_res + 3, &bp, &ohp, rp, &rp_locks); res = copy_struct(res, sz_res, &hp, ohp); tuple = TUPLE2(hp, sender, res); - erts_queue_message(rp, &rp_locks, bp, tuple, NIL); + erts_queue_message(rp, &rp_locks, bp, tuple, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); erts_smp_proc_unlock(rp, rp_locks); erts_smp_proc_dec_refc(rp); } @@ -1603,7 +1668,11 @@ static void deliver_read_message(Port* prt, Eterm to, tuple = TUPLE2(hp, prt->id, tuple); hp += 3; - erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined); + erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined +#ifdef USE_VM_PROBES + , NIL +#endif + ); erts_smp_proc_unlock(rp, rp_locks); erts_smp_proc_dec_refc(rp); } @@ -1756,7 +1825,11 @@ deliver_vec_message(Port* prt, /* Port */ tuple = TUPLE2(hp, prt->id, tuple); hp += 3; - erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined); + erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined +#ifdef USE_VM_PROBES + , NIL +#endif + ); erts_smp_proc_unlock(rp, rp_locks); erts_smp_proc_dec_refc(rp); } @@ -1795,6 +1868,12 @@ static void flush_port(Port *p) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); if (p->drv_ptr->flush != NULL) { +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(driver_flush)) { + DTRACE_FORMAT_COMMON_PID_AND_PORT(p->connected, p) + DTRACE3(driver_flush, process_str, port_str, p->name); + } +#endif if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) { trace_sched_ports_where(p, am_in, am_flush); } @@ -1822,6 +1901,7 @@ terminate_port(Port *prt) Eterm send_closed_port_id; Eterm connected_id = NIL /* Initialize to silence compiler */; erts_driver_t *drv; + int halt; ERTS_SMP_CHK_NO_PROC_LOCKS; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); @@ -1829,6 +1909,8 @@ terminate_port(Port *prt) ASSERT(!prt->nlinks); ASSERT(!prt->monitors); + /* prt->status may be altered by kill_port()below */ + halt = (prt->status & ERTS_PORT_SFLG_HALT) != 0; if (prt->status & ERTS_PORT_SFLG_SEND_CLOSED) { erts_port_status_band_set(prt, ~ERTS_PORT_SFLG_SEND_CLOSED); send_closed_port_id = prt->id; @@ -1847,6 +1929,12 @@ terminate_port(Port *prt) drv = prt->drv_ptr; if ((drv != NULL) && (drv->stop != NULL)) { int fpe_was_unmasked = erts_block_fpe(); +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(driver_stop)) { + DTRACE_FORMAT_COMMON_PID_AND_PORT(prt->connected, prt) + DTRACE3(driver_stop, process_str, drv->name, port_str); + } +#endif (*drv->stop)((ErlDrvData)prt->drv_data); erts_unblock_fpe(fpe_was_unmasked); #ifdef ERTS_SMP @@ -1880,6 +1968,10 @@ terminate_port(Port *prt) * We don't want to send the closed message until after the * port has been removed from the port table (in kill_port()). */ + if (halt && (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0)) { + erts_smp_port_unlock(prt); /* We will exit and never return */ + erl_exit_flush_async(erts_halt_code, ""); + } if (is_internal_port(send_closed_port_id)) deliver_result(send_closed_port_id, connected_id, am_closed); @@ -2004,6 +2096,19 @@ erts_do_exit_port(Port *p, Eterm from, Eterm reason) rreason = (reason == am_kill) ? am_killed : reason; +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(port_exit)) { + DTRACE_CHARBUF(from_str, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(rreason_str, 64); + + erts_snprintf(from_str, sizeof(from_str), "%T", from); + dtrace_port_str(p, port_str); + erts_snprintf(rreason_str, sizeof(rreason_str), "%T", rreason); + DTRACE4(port_exit, from_str, port_str, p->name, rreason_str); + } +#endif + if ((p->status & (ERTS_PORT_SFLGS_DEAD | ERTS_PORT_SFLG_EXITING | ERTS_PORT_SFLG_IMMORTAL)) @@ -2104,6 +2209,13 @@ void erts_port_command(Process *proc, if (tp[2] == am_close) { erts_port_status_bor_set(port, ERTS_PORT_SFLG_SEND_CLOSED); erts_do_exit_port(port, pid, am_normal); + +#ifdef USE_VM_PROBES + if(DTRACE_ENABLED(port_command)) { + DTRACE_FORMAT_COMMON_PROC_AND_PORT(proc, port) + DTRACE4(port_command, process_str, port_str, port->name, "close"); + } +#endif goto done; } else if (is_tuple_arity(tp[2], 2)) { tp = tuple_val(tp[2]); @@ -2111,6 +2223,12 @@ void erts_port_command(Process *proc, if (erts_write_to_port(caller_id, port, tp[2]) == 0) goto done; } else if ((tp[1] == am_connect) && is_internal_pid(tp[2])) { +#ifdef USE_VM_PROBES + if(DTRACE_ENABLED(port_command)) { + DTRACE_FORMAT_COMMON_PROC_AND_PORT(proc, port) + DTRACE4(port_command, process_str, port_str, port->name, "connect"); + } +#endif port->connected = tp[2]; deliver_result(port->id, pid, am_connected); goto done; @@ -2213,6 +2331,15 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist) erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); ERTS_SMP_CHK_NO_PROC_LOCKS; +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(port_control) || DTRACE_ENABLED(driver_control)) { + DTRACE_FORMAT_COMMON_PROC_AND_PORT(p, prt); + DTRACE4(port_control, process_str, port_str, prt->name, command); + DTRACE5(driver_control, process_str, port_str, prt->name, + command, to_len); + } +#endif + /* * Call the port's control routine. */ @@ -2353,6 +2480,10 @@ print_port_info(int to, void *arg, int i) void set_busy_port(ErlDrvPort port_num, int on) { +#ifdef USE_VM_PROBES + DTRACE_CHARBUF(port_str, 16); +#endif + ERTS_SMP_CHK_NO_PROC_LOCKS; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(&erts_port[port_num])); @@ -2360,12 +2491,26 @@ set_busy_port(ErlDrvPort port_num, int on) if (on) { erts_port_status_bor_set(&erts_port[port_num], ERTS_PORT_SFLG_PORT_BUSY); +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(port_busy)) { + erts_snprintf(port_str, sizeof(port_str), + "%T", erts_port[port_num].id); + DTRACE1(port_busy, port_str); + } +#endif } else { ErtsProcList* plp = erts_port[port_num].suspended; erts_port_status_band_set(&erts_port[port_num], ~ERTS_PORT_SFLG_PORT_BUSY); erts_port[port_num].suspended = NULL; +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(port_not_busy)) { + erts_snprintf(port_str, sizeof(port_str), + "%T", erts_port[port_num].id); + DTRACE1(port_not_busy, port_str); + } +#endif if (erts_port[port_num].dist_entry) { /* * Processes suspended on distribution ports are @@ -2383,6 +2528,28 @@ set_busy_port(ErlDrvPort port_num, int on) */ if (plp) { +#ifdef USE_VM_PROBES + /* + * Hrm, for blocked dist ports, plp always seems to be NULL. + * That's not so fun. + * Well, another way to get the same info is using a D + * script to correlate an earlier process-port_blocked+pid + * event with a later process-scheduled event. That's + * subject to the multi-CPU races with how events are + * handled, but hey, that way works most of the time. + */ + if (DTRACE_ENABLED(process_port_unblocked)) { + DTRACE_CHARBUF(pid_str, 16); + ErtsProcList* plp2 = plp; + + erts_snprintf(port_str, sizeof(port_str), + "%T", erts_port[port_num]); + while (plp2 != NULL) { + erts_snprintf(pid_str, sizeof(pid_str), "%T", plp2->pid); + DTRACE2(process_port_unblocked, pid_str, port_str); + } + } +#endif /* First proc should be resumed last */ if (plp->next) { erts_resume_processes(plp->next); @@ -2429,6 +2596,14 @@ void erts_raw_port_command(Port* p, byte* buf, Uint len) p->drv_ptr->name ? p->drv_ptr->name : "unknown"); p->caller = NIL; +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(driver_output)) { + DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); + + dtrace_port_str(p, port_str); + DTRACE4(driver_output, "-raw-", port_str, p->name, len); + } +#endif fpe_was_unmasked = erts_block_fpe(); (*p->drv_ptr->output)((ErlDrvData)p->drv_data, (char*) buf, (int) len); erts_unblock_fpe(fpe_was_unmasked); @@ -2444,6 +2619,12 @@ int async_ready(Port *p, void* data) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); ASSERT(!(p->status & ERTS_PORT_SFLGS_DEAD)); if (p->drv_ptr->ready_async != NULL) { +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(driver_ready_async)) { + DTRACE_FORMAT_COMMON_PID_AND_PORT(p->connected, p) + DTRACE3(driver_ready_async, process_str, port_str, p->name); + } +#endif (*p->drv_ptr->ready_async)((ErlDrvData)p->drv_data, data); need_free = 0; #ifdef ERTS_SMP @@ -2638,7 +2819,11 @@ void driver_report_exit(int ix, int status) hp += 3; tuple = TUPLE2(hp, prt->id, tuple); - erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined); + erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined +#ifdef USE_VM_PROBES + , NIL +#endif + ); erts_smp_proc_unlock(rp, rp_locks); erts_smp_proc_dec_refc(rp); @@ -3188,7 +3373,11 @@ driver_deliver_term(ErlDrvPort port, HRelease(rp, hp_end, hp); } /* send message */ - erts_queue_message(rp, &rp_locks, bp, mess, am_undefined); + erts_queue_message(rp, &rp_locks, bp, mess, am_undefined +#ifdef USE_VM_PROBES + , NIL +#endif + ); } else { if (b2t.ix > b2t.used) @@ -3912,7 +4101,7 @@ int driver_pushqv(ErlDrvPort ix, ErlIOVec* vec, ErlDrvSizeT skip) ErlDrvSizeT driver_deq(ErlDrvPort ix, ErlDrvSizeT size) { ErlIOQueue* q = drvport2ioq(ix); - int len; + ErlDrvSizeT len; if ((q == NULL) || (q->size < size)) return -1; @@ -4419,6 +4608,12 @@ void erts_fire_port_monitor(Port *prt, Eterm ref) ASSERT(callback != NULL); ref_to_driver_monitor(ref,&drv_monitor); DRV_MONITOR_UNLOCK_PDL(prt); +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(driver_process_exit)) { + DTRACE_FORMAT_COMMON_PID_AND_PORT(prt->connected, prt) + DTRACE3(driver_process_exit, process_str, port_str, prt->name); + } +#endif fpe_was_unmasked = erts_block_fpe(); (*callback)((ErlDrvData) (prt->drv_data), &drv_monitor); erts_unblock_fpe(fpe_was_unmasked); @@ -4862,6 +5057,8 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle) else { int res; int fpe_was_unmasked = erts_block_fpe(); + DTRACE4(driver_init, drv->name, drv->version.major, drv->version.minor, + drv->flags); res = (*de->init)(); erts_unblock_fpe(fpe_was_unmasked); return res; diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index fc53a88a3a..b2fc571032 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -883,6 +883,95 @@ call_ext_last u==3 u$func:erlang:hibernate/3 D => i_hibernate call_ext_only u==3 u$func:erlang:hibernate/3 => i_hibernate # +# If VM probes are not enabled, we want to short-circult calls to +# the dt tag BIFs to make them as cheap as possible. +# + +%unless USE_VM_PROBES + +call_ext Arity u$func:erlang:dt_get_tag/0 => \ + move a=am_undefined r +call_ext_last Arity u$func:erlang:dt_get_tag/0 D => \ + move a=am_undefined r | deallocate D | return +call_ext_only Arity u$func:erlang:dt_get_tag/0 => \ + move a=am_undefined r | return + +move Any r | call_ext Arity u$func:erlang:dt_put_tag/1 => \ + move a=am_undefined r +move Any r | call_ext_last Arity u$func:erlang:dt_put_tag/1 D => \ + move a=am_undefined r | deallocate D | return +move Any r | call_ext_only Arity u$func:erlang:dt_put_tag/1 => \ + move a=am_undefined r | return +call_ext Arity u$func:erlang:dt_put_tag/1 => \ + move a=am_undefined r +call_ext_last Arity u$func:erlang:dt_put_tag/1 D => \ + move a=am_undefined r | deallocate D | return +call_ext_only Arity u$func:erlang:dt_put_tag/1 => \ + move a=am_undefined r | return + +call_ext Arity u$func:erlang:dt_get_tag_data/0 => \ + move a=am_undefined r +call_ext_last Arity u$func:erlang:dt_get_tag_data/0 D => \ + move a=am_undefined r | deallocate D | return +call_ext_only Arity u$func:erlang:dt_get_tag_data/0 => \ + move a=am_undefined r | return + +move Any r | call_ext Arity u$func:erlang:dt_spread_tag/1 => \ + move a=am_true r +move Any r | call_ext_last Arity u$func:erlang:dt_spread_tag/1 D => \ + move a=am_true r | deallocate D | return +move Any r | call_ext_only Arity u$func:erlang:dt_spread_tag/1 => \ + move a=am_true r | return +call_ext Arity u$func:erlang:dt_spread_tag/1 => \ + move a=am_true r +call_ext_last Arity u$func:erlang:dt_spread_tag/1 D => \ + move a=am_true r | deallocate D | return +call_ext_only Arity u$func:erlang:dt_spread_tag/1 => \ + move a=am_true r | return + +move Any r | call_ext Arity u$func:erlang:dt_restore_tag/1 => \ + move a=am_true r +move Any r | call_ext_last Arity u$func:erlang:dt_restore_tag/1 D => \ + move a=am_true r | deallocate D | return +move Any r | call_ext_only Arity u$func:erlang:dt_restore_tag/1 => \ + move a=am_true r | return +call_ext Arity u$func:erlang:dt_restore_tag/1 => \ + move a=am_true r +call_ext_last Arity u$func:erlang:dt_restore_tag/1 D => \ + move a=am_true r | deallocate D | return +call_ext_only Arity u$func:erlang:dt_restore_tag/1 => \ + move a=am_true r | return + +move Any r | call_ext Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \ + move Any r +move Any r | call_ext_last Arity u$func:erlang:dt_prepend_vm_tag_data/1 D => \ + move Any r | deallocate D | return +move Any r | call_ext_only Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \ + move Any r | return +call_ext Arity u$func:erlang:dt_prepend_vm_tag_data/1 => +call_ext_last Arity u$func:erlang:dt_prepend_vm_tag_data/1 D => \ + deallocate D | return +call_ext_only Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \ + return + +move Any r | call_ext Arity u$func:erlang:dt_append_vm_tag_data/1 => \ + move Any r +move Any r | call_ext_last Arity u$func:erlang:dt_append_vm_tag_data/1 D => \ + move Any r | deallocate D | return +move Any r | call_ext_only Arity u$func:erlang:dt_append_vm_tag_data/1 => \ + move Any r | return +call_ext Arity u$func:erlang:dt_append_vm_tag_data/1 => +call_ext_last Arity u$func:erlang:dt_append_vm_tag_data/1 D => \ + deallocate D | return +call_ext_only Arity u$func:erlang:dt_append_vm_tag_data/1 => \ + return + +# Can happen after one of the transformations above. +move Discarded r | move Something r => move Something r + +%endif + +# # The general case for BIFs that have no special instructions. # A BIF used in the tail must be followed by a return instruction. # diff --git a/erts/emulator/beam/register.h b/erts/emulator/beam/register.h index 97bab3ab71..bf15e63554 100644 --- a/erts/emulator/beam/register.h +++ b/erts/emulator/beam/register.h @@ -41,7 +41,7 @@ struct port; typedef struct reg_proc { HashBucket bucket; /* MUST BE LOCATED AT TOP OF STRUCT!!! */ - Process *p; /* The process registerd (only one of this and + Process *p; /* The process registered (only one of this and 'pt' is non-NULL */ struct port *pt; /* The port registered */ Eterm name; /* Atom name */ diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index eb6f2f8516..7b2bb81f62 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -511,7 +511,7 @@ __decl_noreturn void __noreturn erl_exit(int n, char*, ...); /* Some special erl_exit() codes: */ #define ERTS_INTR_EXIT INT_MIN /* called from signal handler */ #define ERTS_ABORT_EXIT (INT_MIN + 1) /* no crash dump; only abort() */ -#define ERTS_DUMP_EXIT (127) /* crash dump; then exit() */ +#define ERTS_DUMP_EXIT (INT_MIN + 2) /* crash dump; then exit() */ Eterm erts_check_io_info(void *p); diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 49b6618f73..5ab51fab50 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1697,7 +1697,11 @@ static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len) erts_queue_error_logger_message(from, tuple3, bp); } #else - erts_queue_message(p, NULL /* only used for smp build */, bp, tuple3, NIL); + erts_queue_message(p, NULL /* only used for smp build */, bp, tuple3, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); #endif return 0; } diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c index b132991a3b..b33a8d210b 100644 --- a/erts/emulator/drivers/common/efile_drv.c +++ b/erts/emulator/drivers/common/efile_drv.c @@ -100,7 +100,7 @@ #endif #include <stdlib.h> -// Need (NON)BLOCKING macros for sendfile +/* Need (NON)BLOCKING macros for sendfile */ #ifndef WANT_NONBLOCKING #define WANT_NONBLOCKING #endif @@ -112,6 +112,7 @@ #include "erl_threads.h" #include "zlib.h" #include "gzio.h" +#include "dtrace-wrapper.h" #include <ctype.h> #include <sys/types.h> @@ -119,6 +120,39 @@ void erl_exit(int n, char *fmt, ...); static ErlDrvSysInfo sys_info; +/* For explanation of this var, see comment for same var in erl_async.c */ +static unsigned gcc_optimizer_hack = 0; + +#ifdef USE_VM_PROBES + +#define DTRACE_EFILE_BUFSIZ 128 + +#define DTRACE_INVOKE_SETUP(op) \ + do { DTRACE3(efile_drv_int_entry, d->sched_i1, d->sched_i2, op); } while (0) +#define DTRACE_INVOKE_SETUP_BY_NAME(op) \ + struct t_data *d = (struct t_data *) data ; \ + DTRACE_INVOKE_SETUP(op) +#define DTRACE_INVOKE_RETURN(op) \ + do { DTRACE3(efile_drv_int_return, d->sched_i1, d->sched_i2, \ + op); } while (0) ; gcc_optimizer_hack++ ; + +/* Assign human-friendlier id numbers to scheduler & I/O worker threads */ +int dt_driver_idnum = 0; +int dt_driver_io_worker_base = 5000; +erts_mtx_t dt_driver_mutex; +pthread_key_t dt_driver_key; + +typedef struct { + int thread_num; + Uint64 tag; +} dt_private; + +dt_private *get_dt_private(int); +#else /* USE_VM_PROBES */ +#define DTRACE_INVOKE_SETUP(op) do {} while (0) +#define DTRACE_INVOKE_SETUP_BY_NAME(op) do {} while (0) +#define DTRACE_INVOKE_RETURN(op) do {} while (0) +#endif /* USE_VM_PROBES */ /* #define TRACE 1 */ #ifdef TRACE @@ -156,11 +190,11 @@ static ErlDrvSysInfo sys_info; * DARWIN. The testcase t_sendfile_crashduring reproduces * this error when using +A 10. */ -#if !defined(DARWIN) -#define USE_THRDS_FOR_SENDFILE (sys_info.async_threads > 0) -#else +#if defined(__APPLE__) && defined(__MACH__) #define USE_THRDS_FOR_SENDFILE 0 -#endif /* !DARWIN */ +#else +#define USE_THRDS_FOR_SENDFILE (sys_info.async_threads > 0) +#endif /* defined(__APPLE__) && defined(__MACH__) */ @@ -174,6 +208,9 @@ static ErlDrvSysInfo sys_info; #ifdef FILENAMES_16BIT +#ifdef USE_VM_PROBES +#error 16bit characters in filenames and dtrace in combination is not supported. +#endif # define FILENAME_BYTELEN(Str) filename_len_16bit(Str) # define FILENAME_COPY(To,From) filename_cpy_16bit((To),(From)) # define FILENAME_CHARSIZE 2 @@ -259,6 +296,7 @@ static void file_stop_select(ErlDrvEvent event, void* _); enum e_timer {timer_idle, timer_again, timer_write}; #ifdef HAVE_SENDFILE enum e_sendfile {sending, not_sending}; +static void free_sendfile(void *data); #endif /* HAVE_SENDFILE */ struct t_data; @@ -288,6 +326,10 @@ typedef struct { ErlDrvPDL q_mtx; /* Mutex for the driver queue, known by the emulator. Also used for mutual exclusion when accessing field(s) below. */ size_t write_buffered; +#ifdef USE_VM_PROBES + int idnum; /* Unique ID # for this driver thread/desc */ + char port_str[DTRACE_TERM_BUF_SIZE]; +#endif } file_descriptor; @@ -385,6 +427,11 @@ struct t_data void (*free)(void *); int again; int reply; +#ifdef USE_VM_PROBES + int sched_i1; + Uint64 sched_i2; + char sched_utag[DTRACE_EFILE_BUFSIZ+1]; +#endif int result_ok; Efile_error errInfo; int flags; @@ -445,6 +492,8 @@ struct t_data } fadvise; #ifdef HAVE_SENDFILE struct { + ErlDrvPort port; + ErlDrvPDL q_mtx; int out_fd; off_t offset; Uint64 nbytes; @@ -455,8 +504,6 @@ struct t_data char b[1]; }; - - #define EF_ALLOC(S) driver_alloc((S)) #define EF_REALLOC(P, S) driver_realloc((P), (S)) #define EF_SAFE_ALLOC(S) ef_safe_alloc((S)) @@ -485,7 +532,7 @@ static void *ef_safe_realloc(void *op, Uint s) * ErlIOVec manipulation functions. */ -/* char EV_CHAR(ErlIOVec *ev, int p, int q) */ +/* char EV_CHAR_P(ErlIOVec *ev, int p, int q) */ #define EV_CHAR_P(ev, p, q) \ (((char *)(ev)->iov[(q)].iov_base) + (p)) @@ -680,6 +727,11 @@ file_init(void) : 0); driver_system_info(&sys_info, sizeof(ErlDrvSysInfo)); +#ifdef USE_VM_PROBES + erts_mtx_init(&dt_driver_mutex, "efile_drv dtrace mutex"); + pthread_key_create(&dt_driver_key, NULL); +#endif /* USE_VM_PROBES */ + return 0; } @@ -719,6 +771,10 @@ file_start(ErlDrvPort port, char* command) desc->write_error = 0; MUTEX_INIT(desc->q_mtx, port); /* Refc is one, referenced by emulator now */ desc->write_buffered = 0; +#ifdef USE_VM_PROBES + dtrace_drvport_str(port, desc->port_str); + get_dt_private(0); /* throw away return value */ +#endif /* USE_VM_PROBES */ return (ErlDrvData) desc; } @@ -738,8 +794,10 @@ static void do_close(int flags, SWord fd) { static void invoke_close(void *data) { struct t_data *d = (struct t_data *) data; + DTRACE_INVOKE_SETUP(FILE_CLOSE); d->again = 0; do_close(d->flags, d->fd); + DTRACE_INVOKE_RETURN(FILE_CLOSE); } /********************************************************************* @@ -752,15 +810,6 @@ file_stop(ErlDrvData e) TRACE_C('p'); -#ifdef HAVE_SENDFILE - if (desc->sendfile_state == sending && !USE_THRDS_FOR_SENDFILE) { - driver_select(desc->port,(ErlDrvEvent)(long)desc->d->c.sendfile.out_fd, - ERL_DRV_WRITE|ERL_DRV_USE,0); - } else if (desc->sendfile_state == sending) { - SET_NONBLOCKING(desc->d->c.sendfile.out_fd); - } -#endif /* HAVE_SENDFILE */ - if (desc->fd != FILE_FD_INVALID) { do_close(desc->flags, desc->fd); desc->fd = FILE_FD_INVALID; @@ -978,49 +1027,63 @@ static void invoke_name(void *data, int (*f)(Efile_error *, char *)) static void invoke_mkdir(void *data) { + DTRACE_INVOKE_SETUP_BY_NAME(FILE_MKDIR); invoke_name(data, efile_mkdir); + DTRACE_INVOKE_RETURN(FILE_MKDIR); } static void invoke_rmdir(void *data) { + DTRACE_INVOKE_SETUP_BY_NAME(FILE_RMDIR); invoke_name(data, efile_rmdir); + DTRACE_INVOKE_RETURN(FILE_RMDIR); } static void invoke_delete_file(void *data) { + DTRACE_INVOKE_SETUP_BY_NAME(FILE_DELETE); invoke_name(data, efile_delete_file); + DTRACE_INVOKE_RETURN(FILE_DELETE); } static void invoke_chdir(void *data) { + DTRACE_INVOKE_SETUP_BY_NAME(FILE_CHDIR); invoke_name(data, efile_chdir); + DTRACE_INVOKE_RETURN(FILE_CHDIR); } static void invoke_fdatasync(void *data) { struct t_data *d = (struct t_data *) data; int fd = (int) d->fd; + DTRACE_INVOKE_SETUP(FILE_FDATASYNC); d->again = 0; d->result_ok = efile_fdatasync(&d->errInfo, fd); + DTRACE_INVOKE_RETURN(FILE_FDATASYNC); } static void invoke_fsync(void *data) { struct t_data *d = (struct t_data *) data; int fd = (int) d->fd; + DTRACE_INVOKE_SETUP(FILE_FSYNC); d->again = 0; d->result_ok = efile_fsync(&d->errInfo, fd); + DTRACE_INVOKE_RETURN(FILE_FSYNC); } static void invoke_truncate(void *data) { struct t_data *d = (struct t_data *) data; int fd = (int) d->fd; + DTRACE_INVOKE_SETUP(FILE_TRUNCATE); d->again = 0; d->result_ok = efile_truncate_file(&d->errInfo, &fd, d->flags); + DTRACE_INVOKE_RETURN(FILE_TRUNCATE); } static void invoke_read(void *data) @@ -1028,6 +1091,7 @@ static void invoke_read(void *data) struct t_data *d = (struct t_data *) data; int status, segment; size_t size, read_size; + DTRACE_INVOKE_SETUP(FILE_READ); segment = d->again && d->c.read.bin_size >= 2*FILE_SEGMENT_READ; if (segment) { @@ -1062,6 +1126,7 @@ static void invoke_read(void *data) } else { d->again = 0; } + DTRACE_INVOKE_RETURN(FILE_READ); } static void free_read(void *data) @@ -1078,6 +1143,7 @@ static void invoke_read_line(void *data) int status; size_t read_size; int local_loop = (d->again == 0); + DTRACE_INVOKE_SETUP(FILE_READ_LINE); do { size_t size = (d->c.read_line.binp)->orig_size - @@ -1169,6 +1235,7 @@ static void invoke_read_line(void *data) break; } } while (local_loop); + DTRACE_INVOKE_RETURN(FILE_READ_LINE); } static void free_read_line(void *data) @@ -1184,6 +1251,7 @@ static void invoke_read_file(void *data) struct t_data *d = (struct t_data *) data; size_t read_size; int chop; + DTRACE_INVOKE_SETUP(FILE_READ_FILE); if (! d->c.read_file.binp) { /* First invocation only */ int fd; @@ -1220,12 +1288,14 @@ static void invoke_read_file(void *data) &read_size); if (d->result_ok) { d->c.read_file.offset += read_size; - if (chop) return; /* again */ + if (chop) goto chop_done; /* again */ } close: efile_closefile((int) d->fd); done: d->again = 0; + chop_done: + DTRACE_INVOKE_RETURN(FILE_READ_FILE); } static void free_read_file(void *data) @@ -1245,6 +1315,7 @@ static void invoke_preadv(void *data) ErlIOVec *ev = &c->eiov; size_t bytes_read_so_far = 0; unsigned char *p = (unsigned char *)ev->iov[0].iov_base + 4+4+8*c->cnt; + DTRACE_INVOKE_SETUP(FILE_PREADV); while (c->cnt < c->n) { size_t read_size = ev->iov[1 + c->cnt].iov_len - c->size; @@ -1266,7 +1337,7 @@ static void invoke_preadv(void *data) bytes_read_so_far += bytes_read; if (chop && bytes_read == read_size) { c->size += bytes_read; - return; + goto done; } ASSERT(bytes_read <= read_size); ev->iov[1 + c->cnt].iov_len = bytes_read + c->size; @@ -1277,7 +1348,7 @@ static void invoke_preadv(void *data) if (d->again && bytes_read_so_far >= FILE_SEGMENT_READ && c->cnt < c->n) { - return; + goto done; } } else { /* In case of a read error, ev->size will not be correct, @@ -1288,6 +1359,8 @@ static void invoke_preadv(void *data) } } d->again = 0; + done: + DTRACE_INVOKE_RETURN(FILE_PREADV); } static void free_preadv(void *data) { @@ -1309,6 +1382,7 @@ static void invoke_ipread(void *data) size_t bytes_read = 0; char buf[2*sizeof(Uint32)]; Uint32 offset, size; + DTRACE_INVOKE_SETUP(FILE_IPREAD); /* Read indirection header */ if (! efile_pread(&d->errInfo, (int) d->fd, c->offsets[0], @@ -1347,14 +1421,17 @@ static void invoke_ipread(void *data) /* Read data block */ d->invoke = invoke_preadv; invoke_preadv(data); + DTRACE_INVOKE_RETURN(FILE_IPREAD); return; error: d->result_ok = 0; d->again = 0; + DTRACE_INVOKE_RETURN(FILE_IPREAD); return; done: d->result_ok = !0; d->again = 0; + DTRACE_INVOKE_RETURN(FILE_IPREAD); } /* invoke_writev and invoke_pwritev are the only thread functions that @@ -1377,6 +1454,7 @@ static void invoke_writev(void *data) { size_t size; size_t p; int segment; + DTRACE_INVOKE_SETUP(FILE_WRITE); segment = d->again && d->c.writev.size >= 2*FILE_SEGMENT_WRITE; if (segment) { @@ -1385,7 +1463,11 @@ static void invoke_writev(void *data) { size = d->c.writev.size; } - /* Copy the io vector to avoid locking the port que while writing */ + /* Copy the io vector to avoid locking the port que while writing, + * also, both we and efile_writev might/will change the SysIOVec + * when segmenting or due to partial write and we do not want to + * tamper with the actual queue that we get from driver_peekq + */ MUTEX_LOCK(d->c.writev.q_mtx); /* Lock before accessing the port queue */ iov0 = driver_peekq(d->c.writev.port, &iovlen); @@ -1424,7 +1506,7 @@ static void invoke_writev(void *data) { } else { d->result_ok = efile_writev(&d->errInfo, d->flags, (int) d->fd, - iov, iovcnt, size); + iov, iovcnt); } } else if (iovlen == 0) { d->result_ok = 1; @@ -1446,6 +1528,7 @@ static void invoke_writev(void *data) { TRACE_F(("w%lu", (unsigned long)size)); } + DTRACE_INVOKE_RETURN(FILE_WRITE); } static void free_writev(void *data) { @@ -1459,34 +1542,40 @@ static void free_writev(void *data) { static void invoke_pwd(void *data) { struct t_data *d = (struct t_data *) data; + DTRACE_INVOKE_SETUP(FILE_PWD); d->again = 0; d->result_ok = efile_getdcwd(&d->errInfo,d->drive, d->b+1, RESBUFSIZE-1); + DTRACE_INVOKE_RETURN(FILE_PWD); } static void invoke_readlink(void *data) { struct t_data *d = (struct t_data *) data; char resbuf[RESBUFSIZE]; /* Result buffer. */ + DTRACE_INVOKE_SETUP(FILE_READLINK); d->again = 0; d->result_ok = efile_readlink(&d->errInfo, d->b, resbuf+1, RESBUFSIZE-1); if (d->result_ok != 0) FILENAME_COPY((char *) d->b + 1, resbuf+1); + DTRACE_INVOKE_RETURN(FILE_READLINK); } static void invoke_altname(void *data) { struct t_data *d = (struct t_data *) data; char resbuf[RESBUFSIZE]; /* Result buffer. */ + DTRACE_INVOKE_SETUP(FILE_ALTNAME); d->again = 0; d->result_ok = efile_altname(&d->errInfo, d->b, resbuf+1, RESBUFSIZE-1); if (d->result_ok != 0) FILENAME_COPY((char *) d->b + 1, resbuf+1); + DTRACE_INVOKE_RETURN(FILE_ALTNAME); } static void invoke_pwritev(void *data) { @@ -1499,6 +1588,7 @@ static void invoke_pwritev(void *data) { size_t p; int segment; size_t size, write_size; + DTRACE_INVOKE_SETUP(FILE_PWRITEV); segment = d->again && c->size >= 2*FILE_SEGMENT_WRITE; if (segment) { @@ -1578,6 +1668,7 @@ static void invoke_pwritev(void *data) { } done: EF_FREE(iov); /* Free our copy of the vector, nothing to restore */ + DTRACE_INVOKE_RETURN(FILE_PWRITEV); } static void free_pwritev(void *data) { @@ -1593,9 +1684,14 @@ static void invoke_flstat(void *data) { struct t_data *d = (struct t_data *) data; + DTRACE3(efile_drv_int_entry, d->sched_i1, d->sched_i2, + d->command == FILE_LSTAT ? FILE_LSTAT : FILE_FSTAT); d->again = 0; d->result_ok = efile_fileinfo(&d->errInfo, &d->info, d->b, d->command == FILE_LSTAT); + DTRACE3(efile_drv_int_entry, d->sched_i1, d->sched_i2, + d->command == FILE_LSTAT ? FILE_LSTAT : FILE_FSTAT); + gcc_optimizer_hack++; } static void invoke_link(void *data) @@ -1603,10 +1699,12 @@ static void invoke_link(void *data) struct t_data *d = (struct t_data *) data; char *name = d->b; char *new_name; + DTRACE_INVOKE_SETUP(FILE_LINK); d->again = 0; new_name = name+FILENAME_BYTELEN(name)+FILENAME_CHARSIZE; d->result_ok = efile_link(&d->errInfo, name, new_name); + DTRACE_INVOKE_RETURN(FILE_LINK); } static void invoke_symlink(void *data) @@ -1614,10 +1712,12 @@ static void invoke_symlink(void *data) struct t_data *d = (struct t_data *) data; char *name = d->b; char *new_name; + DTRACE_INVOKE_SETUP(FILE_SYMLINK); d->again = 0; new_name = name+FILENAME_BYTELEN(name)+FILENAME_CHARSIZE; d->result_ok = efile_symlink(&d->errInfo, name, new_name); + DTRACE_INVOKE_RETURN(FILE_SYMLINK); } static void invoke_rename(void *data) @@ -1625,24 +1725,29 @@ static void invoke_rename(void *data) struct t_data *d = (struct t_data *) data; char *name = d->b; char *new_name; + DTRACE_INVOKE_SETUP(FILE_RENAME); d->again = 0; new_name = name+FILENAME_BYTELEN(name)+FILENAME_CHARSIZE; d->result_ok = efile_rename(&d->errInfo, name, new_name); + DTRACE_INVOKE_RETURN(FILE_RENAME); } static void invoke_write_info(void *data) { struct t_data *d = (struct t_data *) data; + DTRACE_INVOKE_SETUP(FILE_WRITE_INFO); d->again = 0; d->result_ok = efile_write_info(&d->errInfo, &d->info, d->b); + DTRACE_INVOKE_RETURN(FILE_WRITE_INFO); } static void invoke_lseek(void *data) { struct t_data *d = (struct t_data *) data; int status; + DTRACE_INVOKE_SETUP(FILE_LSEEK); d->again = 0; if (d->flags & EFILE_COMPRESSED) { @@ -1667,6 +1772,7 @@ static void invoke_lseek(void *data) &d->c.lseek.location); } d->result_ok = status; + DTRACE_INVOKE_RETURN(FILE_LSEEK); } static void invoke_readdir(void *data) @@ -1677,6 +1783,7 @@ static void invoke_readdir(void *data) size_t n = 0, total = 0; struct t_readdir_buf *b = NULL; int res = 0; + DTRACE_INVOKE_SETUP(FILE_READDIR); d->again = 0; d->errInfo.posix_errno = 0; @@ -1712,13 +1819,14 @@ static void invoke_readdir(void *data) } while(res); d->result_ok = (d->errInfo.posix_errno == 0); + DTRACE_INVOKE_RETURN(FILE_READDIR); } static void invoke_open(void *data) { struct t_data *d = (struct t_data *) data; - int status = 1; /* Status of open call. */ + DTRACE_INVOKE_SETUP(FILE_OPEN); d->again = 0; if ((d->flags & EFILE_COMPRESSED) == 0) { @@ -1751,6 +1859,7 @@ static void invoke_open(void *data) } d->result_ok = status; + DTRACE_INVOKE_RETURN(FILE_OPEN); } static void invoke_fadvise(void *data) @@ -1760,9 +1869,11 @@ static void invoke_fadvise(void *data) off_t offset = (off_t) d->c.fadvise.offset; off_t length = (off_t) d->c.fadvise.length; int advise = (int) d->c.fadvise.advise; + DTRACE_INVOKE_SETUP(FILE_FADVISE); d->again = 0; d->result_ok = efile_fadvise(&d->errInfo, fd, offset, length, advise); + DTRACE_INVOKE_RETURN(FILE_FADVISE); } #ifdef HAVE_SENDFILE @@ -1779,26 +1890,31 @@ static void invoke_sendfile(void *data) d->c.sendfile.written += nbytes; - if (result == 1) { - if (USE_THRDS_FOR_SENDFILE) { - d->result_ok = 0; - } else if (d->c.sendfile.nbytes == 0 && nbytes != 0) { - d->result_ok = 1; - } else if ((d->c.sendfile.nbytes - nbytes) != 0) { - d->result_ok = 1; - d->c.sendfile.nbytes -= nbytes; - } else { - d->result_ok = 0; - } + if (result == 1 || (result == 0 && USE_THRDS_FOR_SENDFILE)) { + d->result_ok = 0; } else if (result == 0 && (d->errInfo.posix_errno == EAGAIN || d->errInfo.posix_errno == EINTR)) { + if ((d->c.sendfile.nbytes - nbytes) != 0) { d->result_ok = 1; + if (d->c.sendfile.nbytes != 0) + d->c.sendfile.nbytes -= nbytes; + } else + d->result_ok = 0; } else { d->result_ok = -1; } } static void free_sendfile(void *data) { + struct t_data *d = (struct t_data *)data; + if (USE_THRDS_FOR_SENDFILE) { + SET_NONBLOCKING(d->c.sendfile.out_fd); + } else { + MUTEX_LOCK(d->c.sendfile.q_mtx); + driver_deq(d->c.sendfile.port,1); + MUTEX_UNLOCK(d->c.sendfile.q_mtx); + driver_select(d->c.sendfile.port, (ErlDrvEvent)(long)d->c.sendfile.out_fd, ERL_DRV_USE_NO_CALLBACK|ERL_DRV_WRITE, 0); + } EF_FREE(data); } @@ -1808,7 +1924,7 @@ static void file_ready_output(ErlDrvData data, ErlDrvEvent event) switch (fd->d->command) { case FILE_SENDFILE: - driver_select(fd->port, event, + driver_select(fd->d->c.sendfile.port, event, (int)ERL_DRV_WRITE,(int) 0); invoke_sendfile((void *)fd->d); file_async_ready(data, (ErlDrvThreadData)fd->d); @@ -1822,6 +1938,15 @@ static void file_stop_select(ErlDrvEvent event, void* _) { } + +static int flush_sendfile(file_descriptor *desc,void *_) { + if (desc->sendfile_state == sending) { + desc->d->result_ok = -1; + desc->d->errInfo.posix_errno = ECONNABORTED; + file_async_ready((ErlDrvData)desc,(ErlDrvThreadData)desc->d); + } + return 1; +} #endif /* HAVE_SENDFILE */ @@ -1829,6 +1954,7 @@ static void free_readdir(void *data) { struct t_data *d = (struct t_data *) data; struct t_readdir_buf *b1 = d->c.read_dir.first_buf; + while (b1) { struct t_readdir_buf *b2 = b1; b1 = b1->next; @@ -1897,12 +2023,16 @@ static void cq_execute(file_descriptor *desc) { DRIVER_ASYNC(d->level, desc, d->invoke, void_ptr=d, d->free); } -static int async_write(file_descriptor *desc, int *errp, - int reply, Uint32 reply_size) { +static struct t_data *async_write(file_descriptor *desc, int *errp, + int reply, Uint32 reply_size +#ifdef USE_VM_PROBES + ,Sint64 *dt_i1, Sint64 *dt_i2, Sint64 *dt_i3 +#endif +) { struct t_data *d; if (! (d = EF_ALLOC(sizeof(struct t_data) - 1))) { if (errp) *errp = ENOMEM; - return -1; + return NULL; } TRACE_F(("w%lu", (unsigned long)desc->write_buffered)); d->command = FILE_WRITE; @@ -1911,6 +2041,13 @@ static int async_write(file_descriptor *desc, int *errp, d->c.writev.port = desc->port; d->c.writev.q_mtx = desc->q_mtx; d->c.writev.size = desc->write_buffered; +#ifdef USE_VM_PROBES + if (dt_i1 != NULL) { + *dt_i1 = d->fd; + *dt_i2 = d->flags; + *dt_i3 = d->c.writev.size; + } +#endif d->reply = reply; d->c.writev.free_size = 0; d->c.writev.reply_size = reply_size; @@ -1919,18 +2056,49 @@ static int async_write(file_descriptor *desc, int *errp, d->level = 1; cq_enq(desc, d); desc->write_buffered = 0; - return 0; + return d; } -static int flush_write(file_descriptor *desc, int *errp) { - int result; +static int flush_write(file_descriptor *desc, int *errp +#ifdef USE_VM_PROBES + , dt_private *dt_priv, char *dt_utag +#endif +) { + int result = 0; +#ifdef USE_VM_PROBES + Sint64 dt_i1 = 0, dt_i2 = 0, dt_i3 = 0; +#endif + struct t_data *d = NULL; + MUTEX_LOCK(desc->q_mtx); if (desc->write_buffered > 0) { - result = async_write(desc, errp, 0, 0); - } else { - result = 0; + if ((d = async_write(desc, errp, 0, 0 +#ifdef USE_VM_PROBES + ,&dt_i1, &dt_i2, &dt_i3 +#endif + )) == NULL) { + result = -1; + } } MUTEX_UNLOCK(desc->q_mtx); +#ifdef USE_VM_PROBES + if (d != NULL) { + d->sched_i1 = dt_priv->thread_num; + d->sched_i2 = dt_priv->tag; + d->sched_utag[0] = '\0'; + if (dt_utag != NULL) { + if (dt_utag[0] == '\0') { + dt_utag = NULL; + } else { + strncpy(d->sched_utag, dt_utag, sizeof(d->sched_utag) - 1); + d->sched_utag[sizeof(d->sched_utag) - 1] = '\0'; + } + } + DTRACE11(efile_drv_entry, dt_priv->thread_num, dt_priv->tag++, + dt_utag, FILE_WRITE, + NULL, NULL, dt_i1, dt_i2, dt_i3, 0, desc->port_str); + } +#endif /* USE_VM_PROBES */ return result; } @@ -1943,9 +2111,17 @@ static int check_write_error(file_descriptor *desc, int *errp) { return 0; } -static int flush_write_check_error(file_descriptor *desc, int *errp) { +static int flush_write_check_error(file_descriptor *desc, int *errp +#ifdef USE_VM_PROBES + , dt_private *dt_priv, char *dt_utag +#endif + ) { int r; - if ( (r = flush_write(desc, errp)) != 0) { + if ( (r = flush_write(desc, errp +#ifdef USE_VM_PROBES + , dt_priv, dt_utag +#endif + )) != 0) { check_write_error(desc, NULL); return r; } else { @@ -1953,12 +2129,16 @@ static int flush_write_check_error(file_descriptor *desc, int *errp) { } } -static int async_lseek(file_descriptor *desc, int *errp, int reply, - Sint64 offset, int origin) { +static struct t_data *async_lseek(file_descriptor *desc, int *errp, int reply, + Sint64 offset, int origin +#ifdef USE_VM_PROBES + , Sint64 *dt_i1, Sint64 *dt_i2, Sint64 *dt_i3 +#endif + ) { struct t_data *d; if (! (d = EF_ALLOC(sizeof(struct t_data)))) { *errp = ENOMEM; - return -1; + return NULL; } d->flags = desc->flags; d->fd = desc->fd; @@ -1966,11 +2146,18 @@ static int async_lseek(file_descriptor *desc, int *errp, int reply, d->reply = reply; d->c.lseek.offset = offset; d->c.lseek.origin = origin; +#ifdef USE_VM_PROBES + if (dt_i1 != NULL) { + *dt_i1 = d->fd; + *dt_i2 = d->c.lseek.offset; + *dt_i3 = d->c.lseek.origin; + } +#endif d->invoke = invoke_lseek; d->free = free_data; d->level = 1; cq_enq(desc, d); - return 0; + return d; } static void flush_read(file_descriptor *desc) { @@ -1982,18 +2169,45 @@ static void flush_read(file_descriptor *desc) { } } -static int lseek_flush_read(file_descriptor *desc, int *errp) { +static int lseek_flush_read(file_descriptor *desc, int *errp +#ifdef USE_VM_PROBES + ,dt_private *dt_priv, char *dt_utag +#endif + ) { int r = 0; size_t read_size = desc->read_size; +#ifdef USE_VM_PROBES + Sint64 dt_i1 = 0, dt_i2 = 0, dt_i3 = 0; +#endif + struct t_data *d; + + flush_read(desc); if (read_size != 0) { - flush_read(desc); - if ((r = async_lseek(desc, errp, 0, - -((ssize_t)read_size), EFILE_SEEK_CUR)) - < 0) { - return r; - } - } else { - flush_read(desc); + if ((d = async_lseek(desc, errp, 0, + -((ssize_t)read_size), EFILE_SEEK_CUR +#ifdef USE_VM_PROBES + , &dt_i1, &dt_i2, &dt_i3 +#endif + )) == NULL) { + r = -1; + } else { +#ifdef USE_VM_PROBES + d->sched_i1 = dt_priv->thread_num; + d->sched_i2 = dt_priv->tag; + d->sched_utag[0] = '\0'; + if (dt_utag != NULL) { + if (dt_utag[0] == '\0') { + dt_utag = NULL; + } else { + strncpy(d->sched_utag, dt_utag, sizeof(d->sched_utag) - 1); + d->sched_utag[sizeof(d->sched_utag) - 1] = '\0'; + } + } + DTRACE11(efile_drv_entry, dt_priv->thread_num, dt_priv->tag++, + dt_utag, FILE_LSEEK, + NULL, NULL, dt_i1, dt_i2, dt_i3, 0, desc->port_str); +#endif /* USE_VM_PROBES */ + } } return r; } @@ -2010,11 +2224,23 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) struct t_data *d = (struct t_data *) data; char header[5]; /* result code + count */ char resbuf[RESBUFSIZE]; /* Result buffer. */ - +#ifdef USE_VM_PROBES + int sched_i1 = d->sched_i1, sched_i2 = d->sched_i2, command = d->command, + result_ok = d->result_ok, + posix_errno = d->result_ok ? 0 : d->errInfo.posix_errno; + DTRACE_CHARBUF(sched_utag, DTRACE_EFILE_BUFSIZ+1); + + sched_utag[0] = '\0'; + if (DTRACE_ENABLED(efile_drv_return)) { + strncpy(sched_utag, d->sched_utag, DTRACE_EFILE_BUFSIZ); + sched_utag[DTRACE_EFILE_BUFSIZ] = '\0'; + } +#endif /* USE_VM_PROBES */ TRACE_C('r'); if (try_again(desc, d)) { + /* DTRACE TODO: what kind of probe makes sense here? */ return; } @@ -2212,6 +2438,9 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) if (d->reply) { TRACE_C('K'); reply_ok(desc); +#ifdef USE_VM_PROBES + result_ok = 1; +#endif } free_data(data); break; @@ -2244,42 +2473,31 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) #ifdef HAVE_SENDFILE case FILE_SENDFILE: if (d->result_ok == -1) { - desc->sendfile_state = not_sending; if (d->errInfo.posix_errno == ECONNRESET || d->errInfo.posix_errno == ENOTCONN || d->errInfo.posix_errno == EPIPE) reply_string_error(desc,"closed"); else reply_error(desc, &d->errInfo); - if (USE_THRDS_FOR_SENDFILE) { - SET_NONBLOCKING(d->c.sendfile.out_fd); - free_sendfile(data); - } else { - driver_select(desc->port, (ErlDrvEvent)(long)d->c.sendfile.out_fd, - ERL_DRV_USE, 0); - free_sendfile(data); - } - } else if (d->result_ok == 0) { desc->sendfile_state = not_sending; + free_sendfile(data); + } else if (d->result_ok == 0) { reply_Sint64(desc, d->c.sendfile.written); - if (USE_THRDS_FOR_SENDFILE) { - SET_NONBLOCKING(d->c.sendfile.out_fd); - free_sendfile(data); - } else { - driver_select(desc->port, (ErlDrvEvent)(long)d->c.sendfile.out_fd, ERL_DRV_USE, 0); - free_sendfile(data); - } + desc->sendfile_state = not_sending; + free_sendfile(data); } else if (d->result_ok == 1) { // If we are using select to send the rest of the data desc->sendfile_state = sending; desc->d = d; driver_select(desc->port, (ErlDrvEvent)(long)d->c.sendfile.out_fd, - ERL_DRV_USE|ERL_DRV_WRITE, 1); + ERL_DRV_USE_NO_CALLBACK|ERL_DRV_WRITE, 1); } break; #endif default: abort(); } + DTRACE6(efile_drv_return, sched_i1, sched_i2, sched_utag, + command, result_ok, posix_errno); if (desc->write_buffered != 0 && desc->timer_state == timer_idle) { desc->timer_state = timer_write; driver_set_timer(desc->port, desc->write_delay); @@ -2302,7 +2520,15 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) char* name; /* Points to the filename in buf. */ int command; struct t_data *d = NULL; - +#ifdef USE_VM_PROBES + char *dt_utag = NULL; + char *dt_s1 = NULL, *dt_s2 = NULL; + Sint64 dt_i1 = 0; + Sint64 dt_i2 = 0; + Sint64 dt_i3 = 0; + Sint64 dt_i4 = 0; + dt_private *dt_priv = get_dt_private(0); +#endif /* USE_VM_PROBES */ TRACE_C('o'); @@ -2317,6 +2543,10 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE); FILENAME_COPY(d->b, name); +#ifdef USE_VM_PROBES + dt_s1 = d->b; + dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE; +#endif d->command = command; d->invoke = invoke_mkdir; d->free = free_data; @@ -2328,6 +2558,10 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE); FILENAME_COPY(d->b, name); +#ifdef USE_VM_PROBES + dt_s1 = d->b; + dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE; +#endif d->command = command; d->invoke = invoke_rmdir; d->free = free_data; @@ -2339,6 +2573,10 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE); FILENAME_COPY(d->b, name); +#ifdef USE_VM_PROBES + dt_s1 = d->b; + dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE; +#endif d->command = command; d->invoke = invoke_delete_file; d->free = free_data; @@ -2356,6 +2594,11 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) FILENAME_COPY(d->b, name); FILENAME_COPY(d->b + namelen, new_name); +#ifdef USE_VM_PROBES + dt_s1 = d->b; + dt_s2 = d->b + namelen; + dt_utag = buf + namelen + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE; +#endif d->flags = desc->flags; d->fd = fd; d->command = command; @@ -2369,6 +2612,10 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE); FILENAME_COPY(d->b, name); +#ifdef USE_VM_PROBES + dt_s1 = d->b; + dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE; +#endif d->command = command; d->invoke = invoke_chdir; d->free = free_data; @@ -2380,6 +2627,9 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + RESBUFSIZE + 1); d->drive = *(uchar*)buf; +#ifdef USE_VM_PROBES + dt_utag = buf + 1; +#endif d->command = command; d->invoke = invoke_pwd; d->free = free_data; @@ -2395,6 +2645,10 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) FILENAME_CHARSIZE); FILENAME_COPY(d->b, name); +#ifdef USE_VM_PROBES + dt_s1 = d->b; + dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE; +#endif d->dir_handle = NULL; d->command = command; d->invoke = invoke_readdir; @@ -2419,6 +2673,10 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) dir_handle = NULL; resbuf[0] = FILE_RESP_LFNAME; +#ifdef USE_VM_PROBES + dt_s1 = name; + dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE; +#endif /* Fill the buffer with multiple directory listings before sending it to the * receiving process. READDIR_CHUNKS is minimum number of files sent to the * receiver. @@ -2452,6 +2710,17 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) reply_error(desc, &errInfo); return; } +#ifdef USE_VM_PROBES + if (dt_utag != NULL && dt_utag[0] == '\0') { + dt_utag = NULL; + } + + DTRACE11(efile_drv_entry, dt_priv->thread_num, dt_priv->tag, + dt_utag, command, name, dt_s2, + dt_i1, dt_i2, dt_i3, dt_i4, desc->port_str); + DTRACE6(efile_drv_return, dt_priv->thread_num, dt_priv->tag++, + dt_utag, command, 1, 0); +#endif TRACE_C('R'); driver_output2(desc->port, resbuf, 1, NULL, 0); return; @@ -2464,6 +2733,11 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) d->flags = get_int32((uchar*)buf); name = buf+4; FILENAME_COPY(d->b, name); +#ifdef USE_VM_PROBES + dt_i1 = d->flags; + dt_s1 = d->b; + dt_utag = name + FILENAME_BYTELEN(d->b) + FILENAME_CHARSIZE; +#endif d->command = command; d->invoke = invoke_open; d->free = free_data; @@ -2476,6 +2750,10 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) d = EF_SAFE_ALLOC(sizeof(struct t_data)); d->fd = fd; +#ifdef USE_VM_PROBES + dt_utag = name; + dt_i1 = fd; +#endif d->command = command; d->invoke = invoke_fdatasync; d->free = free_data; @@ -2488,6 +2766,10 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) d = EF_SAFE_ALLOC(sizeof(struct t_data)); d->fd = fd; +#ifdef USE_VM_PROBES + dt_utag = name; + dt_i1 = fd; +#endif d->command = command; d->invoke = invoke_fsync; d->free = free_data; @@ -2504,6 +2786,14 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) FILENAME_COPY(d->b, name); d->fd = fd; +#ifdef USE_VM_PROBES + dt_utag = name + FILENAME_BYTELEN(d->b) + FILENAME_CHARSIZE; + if (command == FILE_LSTAT) { + dt_s1 = d->b; + } else { + dt_i1 = fd; + } +#endif d->command = command; d->invoke = invoke_flstat; d->free = free_data; @@ -2517,6 +2807,11 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) d->flags = desc->flags; d->fd = fd; +#ifdef USE_VM_PROBES + dt_utag = name; + dt_i1 = fd; + dt_i2 = d->flags; +#endif d->command = command; d->invoke = invoke_truncate; d->free = free_data; @@ -2537,6 +2832,13 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) d->info.cTime = (time_t)((Sint64)get_int64(buf + 7 * 4)); FILENAME_COPY(d->b, buf + 9*4); +#ifdef USE_VM_PROBES + dt_i1 = d->info.mode; + dt_i2 = d->info.uid; + dt_i3 = d->info.gid; + dt_s1 = d->b; + dt_utag = buf + 9 * 4 + FILENAME_BYTELEN(d->b) + FILENAME_CHARSIZE; +#endif d->command = command; d->invoke = invoke_write_info; d->free = free_data; @@ -2549,6 +2851,10 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + RESBUFSIZE + 1); FILENAME_COPY(d->b, name); +#ifdef USE_VM_PROBES + dt_s1 = d->b; + dt_utag = name + FILENAME_BYTELEN(d->b) + FILENAME_CHARSIZE; +#endif d->command = command; d->invoke = invoke_readlink; d->free = free_data; @@ -2560,6 +2866,10 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) { d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + RESBUFSIZE + 1); FILENAME_COPY(d->b, name); +#ifdef USE_VM_PROBES + dt_s1 = d->b; + dt_utag = name + FILENAME_BYTELEN(d->b) + FILENAME_CHARSIZE; +#endif d->command = command; d->invoke = invoke_altname; d->free = free_data; @@ -2580,6 +2890,11 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) FILENAME_COPY(d->b, name); FILENAME_COPY(d->b + namelen, new_name); +#ifdef USE_VM_PROBES + dt_s1 = d->b; + dt_s2 = d->b + namelen; + dt_utag = buf + namelen + FILENAME_BYTELEN(dt_s2) + FILENAME_CHARSIZE; +#endif d->flags = desc->flags; d->fd = fd; d->command = command; @@ -2601,6 +2916,11 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) FILENAME_COPY(d->b, name); FILENAME_COPY(d->b + namelen, new_name); +#ifdef USE_VM_PROBES + dt_s1 = d->b; + dt_s2 = d->b + namelen; + dt_utag = buf + namelen + FILENAME_BYTELEN(dt_s2) + FILENAME_CHARSIZE; +#endif d->flags = desc->flags; d->fd = fd; d->command = command; @@ -2622,6 +2942,13 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) d->c.fadvise.offset = get_int64((uchar*) buf); d->c.fadvise.length = get_int64(((uchar*) buf) + sizeof(Sint64)); d->c.fadvise.advise = get_int32(((uchar*) buf) + 2 * sizeof(Sint64)); +#ifdef USE_VM_PROBES + dt_i1 = d->fd; + dt_i2 = d->c.fadvise.offset; + dt_i3 = d->c.fadvise.length; + dt_i4 = d->c.fadvise.advise; + dt_utag = buf + 3 * sizeof(Sint64); +#endif goto done; } @@ -2635,6 +2962,22 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) done: if (d) { +#ifdef USE_VM_PROBES + d->sched_i1 = dt_priv->thread_num; + d->sched_i2 = dt_priv->tag; + d->sched_utag[0] = '\0'; + if (dt_utag != NULL) { + if (dt_utag[0] == '\0') { + dt_utag = NULL; + } else { + strncpy(d->sched_utag, dt_utag, sizeof(d->sched_utag) - 1); + d->sched_utag[sizeof(d->sched_utag) - 1] = '\0'; + } + } + DTRACE11(efile_drv_entry, dt_priv->thread_num, dt_priv->tag++, + dt_utag, command, dt_s1, dt_s2, + dt_i1, dt_i2, dt_i3, dt_i4, desc->port_str); +#endif cq_enq(desc, d); } } @@ -2648,13 +2991,24 @@ file_flush(ErlDrvData e) { #ifdef DEBUG int r; #endif +#ifdef USE_VM_PROBES + dt_private *dt_priv = get_dt_private(dt_driver_io_worker_base); +#endif TRACE_C('f'); +#ifdef HAVE_SENDFILE + flush_sendfile(desc, NULL); +#endif + #ifdef DEBUG r = #endif - flush_write(desc, NULL); + flush_write(desc, NULL +#ifdef USE_VM_PROBES + , dt_priv, (desc->d == NULL) ? NULL : desc->d->sched_utag +#endif + ); /* Only possible reason for bad return value is ENOMEM, and * there is nobody to tell... */ @@ -2696,6 +3050,9 @@ static void file_timeout(ErlDrvData e) { file_descriptor *desc = (file_descriptor *)e; enum e_timer timer_state = desc->timer_state; +#ifdef USE_VM_PROBES + dt_private *dt_priv = get_dt_private(dt_driver_io_worker_base); +#endif TRACE_C('t'); @@ -2713,7 +3070,11 @@ file_timeout(ErlDrvData e) { #ifdef DEBUG int r = #endif - flush_write(desc, NULL); + flush_write(desc, NULL +#ifdef USE_VM_PROBES + , dt_priv, (desc->d == NULL) ? NULL : desc->d->sched_utag +#endif + ); /* Only possible reason for bad return value is ENOMEM, and * there is nobody to tell... */ @@ -2734,6 +3095,14 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { char command; int p, q; int err; + struct t_data *d = NULL; +#ifdef USE_VM_PROBES + Sint64 dt_i1 = 0, dt_i2 = 0, dt_i3 = 0; + Sint64 dt_i4 = 0; + char *dt_utag = NULL; + char *dt_s1 = NULL; + dt_private *dt_priv = get_dt_private(dt_driver_io_worker_base); +#endif TRACE_C('v'); @@ -2753,18 +3122,19 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { switch (command) { case FILE_CLOSE: { +#ifdef USE_VM_PROBES + dt_utag = EV_CHAR_P(ev, p, q); +#endif flush_read(desc); - if (flush_write_check_error(desc, &err) < 0) { + if (flush_write_check_error(desc, &err +#ifdef USE_VM_PROBES + , dt_priv, dt_utag +#endif + ) < 0) { reply_posix_error(desc, err); goto done; } - if (ev->size != 1) { - /* Wrong command length */ - reply_posix_error(desc, EINVAL); - goto done; - } if (desc->fd != FILE_FD_INVALID) { - struct t_data *d; if (! (d = EF_ALLOC(sizeof(struct t_data)))) { reply_posix_error(desc, ENOMEM); } else { @@ -2772,6 +3142,10 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { d->reply = !0; d->fd = desc->fd; d->flags = desc->flags; +#ifdef USE_VM_PROBES + dt_i1 = d->fd; + dt_i2 = d->flags; +#endif d->invoke = invoke_close; d->free = free_data; d->level = 2; @@ -2787,8 +3161,21 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { case FILE_READ: { Uint32 sizeH, sizeL; size_t size, alloc_size; - struct t_data *d; - if (flush_write_check_error(desc, &err) < 0) { + + if (!EV_GET_UINT32(ev, &sizeH, &p, &q) + || !EV_GET_UINT32(ev, &sizeL, &p, &q)) { + /* Wrong buffer length to contain the read count */ + reply_posix_error(desc, EINVAL); + goto done; + } +#ifdef USE_VM_PROBES + dt_utag = EV_CHAR_P(ev, p, q); +#endif + if (flush_write_check_error(desc, &err +#ifdef USE_VM_PROBES + , dt_priv, dt_utag +#endif + ) < 0) { reply_posix_error(desc, err); goto done; } @@ -2796,19 +3183,16 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { if (desc->read_bufsize == 0 && desc->read_binp != NULL && desc->read_size > 0) { /* We have allocated a buffer for line mode but should not really have a read-ahead buffer... */ - if (lseek_flush_read(desc, &err) < 0) { + if (lseek_flush_read(desc, &err +#ifdef USE_VM_PROBES + , dt_priv, dt_utag +#endif + ) < 0) { reply_posix_error(desc, err); goto done; } } #endif - if (ev->size != 1+8 - || !EV_GET_UINT32(ev, &sizeH, &p, &q) - || !EV_GET_UINT32(ev, &sizeL, &p, &q)) { - /* Wrong buffer length to contain the read count */ - reply_posix_error(desc, EINVAL); - goto done; - } #if SIZEOF_SIZE_T == 4 if (sizeH != 0) { reply_posix_error(desc, EINVAL); @@ -2884,6 +3268,11 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { d->c.read.bin_offset = desc->read_offset + desc->read_size; d->c.read.bin_size = desc->read_binp->orig_size - d->c.read.bin_offset; d->c.read.size = size; +#ifdef USE_VM_PROBES + dt_i1 = d->fd; + dt_i2 = d->flags; + dt_i3 = d->c.read.size; +#endif driver_binary_inc_refc(d->c.read.binp); d->invoke = invoke_read; d->free = free_read; @@ -2901,12 +3290,22 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { * allocated binary + dealing with offsets and lengts are done in file_async ready * for this OP. */ - struct t_data *d; - if (flush_write_check_error(desc, &err) < 0) { +#ifdef USE_VM_PROBES + dt_utag = EV_CHAR_P(ev, p, q); +#endif + if (flush_write_check_error(desc, &err +#ifdef USE_VM_PROBES + , dt_priv, dt_utag +#endif + ) < 0) { reply_posix_error(desc, err); goto done; } - if (ev->size != 1) { + if (ev->size != 1 +#ifdef USE_VM_PROBES + + FILENAME_BYTELEN(dt_utag) + FILENAME_CHARSIZE +#endif + ) { /* Wrong command length */ reply_posix_error(desc, EINVAL); goto done; @@ -2962,8 +3361,16 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { d->c.read_line.binp = desc->read_binp; d->c.read_line.read_offset = desc->read_offset; d->c.read_line.read_size = desc->read_size; +#ifdef USE_VM_PROBES + dt_i1 = d->fd; + dt_i2 = d->flags; + dt_i3 = d->c.read_line.read_offset; +#endif #if !ALWAYS_READ_LINE_AHEAD d->c.read_line.read_ahead = (desc->read_bufsize > 0); +#ifdef USE_VM_PROBES + dt_i4 = d->c.read_line.read_ahead; +#endif #endif driver_binary_inc_refc(d->c.read.binp); d->invoke = invoke_read_line; @@ -2971,10 +3378,22 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { d->level = 1; cq_enq(desc, d); } goto done; - case FILE_WRITE: { + case FILE_WRITE: { /* Dtrace: The dtrace user tag is not last in message, + but follows the message tag directly. + This is handled specially in prim_file.erl */ ErlDrvSizeT skip = 1; ErlDrvSizeT size = ev->size - skip; - if (lseek_flush_read(desc, &err) < 0) { + +#ifdef USE_VM_PROBES + dt_utag = EV_CHAR_P(ev, p, q); + skip += FILENAME_BYTELEN(dt_utag) + FILENAME_CHARSIZE; + size = ev->size - skip; +#endif + if (lseek_flush_read(desc, &err +#ifdef USE_VM_PROBES + , dt_priv, dt_utag +#endif + ) < 0) { reply_posix_error(desc, err); goto done; } @@ -3001,7 +3420,11 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { driver_set_timer(desc->port, desc->write_delay); } } else { - if (async_write(desc, &err, !0, size) != 0) { + if ((d = async_write(desc, &err, !0, size +#ifdef USE_VM_PROBES + , &dt_i1, &dt_i2, &dt_i3 +#endif + )) == NULL) { MUTEX_UNLOCK(desc->q_mtx); reply_posix_error(desc, err); goto done; @@ -3011,24 +3434,49 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { } } goto done; /* case FILE_WRITE */ - case FILE_PWRITEV: { + case FILE_PWRITEV: { /* Dtrace: The dtrace user tag is not last in message, + but follows the message tag directly. + This is handled specially in prim_file.erl */ Uint32 i, j, n; size_t total; - struct t_data *d; - if (lseek_flush_read(desc, &err) < 0) { - reply_Uint_posix_error(desc, 0, err); - goto done; - } - if (flush_write_check_error(desc, &err) < 0) { - reply_Uint_posix_error(desc, 0, err); - goto done; +#ifdef USE_VM_PROBES + char dt_tmp; + int dt_utag_bytes = 1; + + dt_utag = EV_CHAR_P(ev, p, q); + /* This will work for UTF-8, but not for UTF-16 - extra reminder here */ +#ifdef FILENAMES_16BIT +#error 16bit characters in filenames and dtrace in combination is not supported. +#endif + while (EV_GET_CHAR(ev, &dt_tmp, &p, &q) && dt_tmp != '\0') { + dt_utag_bytes++; } +#endif if (ev->size < 1+4 +#ifdef USE_VM_PROBES + + dt_utag_bytes +#endif || !EV_GET_UINT32(ev, &n, &p, &q)) { /* Buffer too short to contain even the number of pos/size specs */ reply_Uint_posix_error(desc, 0, EINVAL); goto done; } + if (lseek_flush_read(desc, &err +#ifdef USE_VM_PROBES + , dt_priv, dt_utag +#endif + ) < 0) { + reply_Uint_posix_error(desc, 0, err); + goto done; + } + if (flush_write_check_error(desc, &err +#ifdef USE_VM_PROBES + , dt_priv, dt_utag +#endif + ) < 0) { + reply_Uint_posix_error(desc, 0, err); + goto done; + } if (n == 0) { /* Trivial case - nothing to write */ if (ev->size != 1+4) { @@ -3038,7 +3486,11 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { } goto done; } - if (ev->size < 1+4+8*(2*n)) { + if (ev->size < 1+4+8*(2*n) +#ifdef USE_VM_PROBES + + dt_utag_bytes +#endif + ) { /* Buffer too short to contain even the pos/size specs */ reply_Uint_posix_error(desc, 0, EINVAL); goto done; @@ -3053,6 +3505,10 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { d->reply = !0; d->fd = desc->fd; d->flags = desc->flags; +#ifdef USE_VM_PROBES + dt_i1 = d->fd; + dt_i2 = d->flags; +#endif d->c.pwritev.port = desc->port; d->c.pwritev.q_mtx = desc->q_mtx; d->c.pwritev.n = n; @@ -3090,13 +3546,20 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { } } d->c.pwritev.size = total; +#ifdef USE_VM_PROBES + dt_i3 = d->c.pwritev.size; +#endif d->c.pwritev.free_size = 0; if (j == 0) { /* Trivial case - nothing to write */ EF_FREE(d); reply_Uint(desc, 0); } else { - ErlDrvSizeT skip = 1 + 4 + 8*(2*n); + ErlDrvSizeT skip = 1 + 4 + 8 * (2*n) +#ifdef USE_VM_PROBES + + dt_utag_bytes +#endif + ; if (skip + total != ev->size) { /* Actual amount of data does not match * total of all pos/size specs @@ -3117,27 +3580,55 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { } } goto done; /* case FILE_PWRITEV: */ - case FILE_PREADV: { + case FILE_PREADV: { /* Dtrace: The dtrace user tag is not last in message, + but follows the message tag directly. + This is handled specially in prim_file.erl */ register void * void_ptr; Uint32 i, n; - struct t_data *d; ErlIOVec *res_ev; - if (lseek_flush_read(desc, &err) < 0) { +#ifdef USE_VM_PROBES + char dt_tmp; + int dt_utag_bytes = 1; + /* This will work for UTF-8, but not for UTF-16 - extra reminder here */ +#ifdef FILENAMES_16BIT +#error 16bit characters in filenames and dtrace in combination is not supported. +#endif + dt_utag = EV_CHAR_P(ev, p, q); + while (EV_GET_CHAR(ev, &dt_tmp, &p, &q) && dt_tmp != '\0') { + dt_utag_bytes++; + } +#endif + if (lseek_flush_read(desc, &err +#ifdef USE_VM_PROBES + , dt_priv, dt_utag +#endif + ) < 0) { reply_posix_error(desc, err); goto done; } - if (flush_write_check_error(desc, &err) < 0) { + if (flush_write_check_error(desc, &err +#ifdef USE_VM_PROBES + , dt_priv, dt_utag +#endif + ) < 0) { reply_posix_error(desc, err); goto done; } if (ev->size < 1+8 +#ifdef USE_VM_PROBES + + dt_utag_bytes +#endif || !EV_GET_UINT32(ev, &n, &p, &q) || !EV_GET_UINT32(ev, &n, &p, &q)) { /* Buffer too short to contain even the number of pos/size specs */ reply_posix_error(desc, EINVAL); goto done; } - if (ev->size != 1+8+8*(2*n)) { + if (ev->size < 1+8+8*(2*n) +#ifdef USE_VM_PROBES + + dt_utag_bytes +#endif + ) { /* Buffer wrong length to contain the pos/size specs */ reply_posix_error(desc, EINVAL); goto done; @@ -3157,6 +3648,10 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { d->reply = !0; d->fd = desc->fd; d->flags = desc->flags; +#ifdef USE_VM_PROBES + dt_i1 = d->fd; + dt_i2 = d->flags; +#endif d->c.preadv.n = n; d->c.preadv.cnt = 0; d->c.preadv.size = 0; @@ -3184,6 +3679,9 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { #else size = ((size_t)sizeH<<32) | sizeL; #endif +#ifdef USE_VM_PROBES + dt_i3 += size; +#endif if (! (res_ev->binv[i] = driver_alloc_binary(size))) { reply_posix_error(desc, ENOMEM); break; @@ -3230,42 +3728,68 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { } goto done; /* case FILE_PREADV: */ case FILE_LSEEK: { - Sint64 offset; /* Offset for seek */ + Sint64 offset; /* Offset for seek */ Uint32 origin; /* Origin of seek. */ - if (lseek_flush_read(desc, &err) < 0) { - reply_posix_error(desc, err); + + if (ev->size < 1+8+4 + || !EV_GET_UINT64(ev, &offset, &p, &q) + || !EV_GET_UINT32(ev, &origin, &p, &q)) { + /* Wrong length of buffer to contain offset and origin */ + reply_posix_error(desc, EINVAL); goto done; } - if (flush_write_check_error(desc, &err) < 0) { +#ifdef USE_VM_PROBES + dt_utag = EV_CHAR_P(ev, p, q); +#endif + if (lseek_flush_read(desc, &err +#ifdef USE_VM_PROBES + , dt_priv, dt_utag +#endif + ) < 0) { reply_posix_error(desc, err); goto done; } - if (ev->size != 1+8+4 - || !EV_GET_UINT64(ev, &offset, &p, &q) - || !EV_GET_UINT32(ev, &origin, &p, &q)) { - /* Wrong length of buffer to contain offset and origin */ - reply_posix_error(desc, EINVAL); + if (flush_write_check_error(desc, &err +#ifdef USE_VM_PROBES + , dt_priv, dt_utag +#endif + ) < 0) { + reply_posix_error(desc, err); goto done; } - if (async_lseek(desc, &err, !0, offset, origin) < 0) { + if ((d = async_lseek(desc, &err, !0, offset, origin +#ifdef USE_VM_PROBES + , &dt_i1, &dt_i2, &dt_i3 +#endif + )) == NULL) { reply_posix_error(desc, err); goto done; } } goto done; case FILE_READ_FILE: { - struct t_data *d; char *filename; if (ev->size < 1+1) { /* Buffer contains empty name */ reply_posix_error(desc, ENOENT); goto done; } +#ifndef USE_VM_PROBES + /* In the dtrace case, the iov has an extra element, the dtrace utag - we will need + another test to see that + the filename is in a single buffer: */ if (ev->size-1 != ev->iov[q].iov_len-p) { /* Name not in one single buffer */ reply_posix_error(desc, EINVAL); goto done; } +#else + if (((byte *)ev->iov[q].iov_base)[ev->iov[q].iov_len-1] != '\0') { + /* Name not in one single buffer */ + reply_posix_error(desc, EINVAL); + goto done; + } +#endif filename = EV_CHAR_P(ev, p, q); d = EF_ALLOC(sizeof(struct t_data) -1 + FILENAME_BYTELEN(filename) + FILENAME_CHARSIZE); if (! d) { @@ -3276,6 +3800,20 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { d->reply = !0; /* Copy name */ FILENAME_COPY(d->b, filename); +#ifdef USE_VM_PROBES + { + char dt_tmp; + + /* This will work for UTF-8, but not for UTF-16 - extra reminder here */ +#ifdef FILENAMES_16BIT +#error 16bit characters in filenames and dtrace in combination is not supported. +#endif + while (EV_GET_CHAR(ev, &dt_tmp, &p, &q) && dt_tmp != '\0') + ; + dt_s1 = d->b; + dt_utag = EV_CHAR_P(ev, p, q); + } +#endif d->c.read_file.binp = NULL; d->invoke = invoke_read_file; d->free = free_read_file; @@ -3295,7 +3833,6 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { char mode; Sint64 hdr_offset; Uint32 max_size; - struct t_data *d; ErlIOVec *res_ev; int vsize; if (! EV_GET_CHAR(ev, &mode, &p, &q)) { @@ -3307,14 +3844,6 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { reply_posix_error(desc, EINVAL); goto done; } - if (lseek_flush_read(desc, &err) < 0) { - reply_posix_error(desc, err); - goto done; - } - if (flush_write_check_error(desc, &err) < 0) { - reply_posix_error(desc, err); - goto done; - } if (ev->size < 1+1+8+4 || !EV_GET_UINT64(ev, &hdr_offset, &p, &q) || !EV_GET_UINT32(ev, &max_size, &p, &q)) { @@ -3323,6 +3852,25 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { reply_posix_error(desc, EINVAL); goto done; } +#ifdef USE_VM_PROBES + dt_utag = EV_CHAR_P(ev, p, q); +#endif + if (lseek_flush_read(desc, &err +#ifdef USE_VM_PROBES + , dt_priv, dt_utag +#endif + ) < 0) { + reply_posix_error(desc, err); + goto done; + } + if (flush_write_check_error(desc, &err +#ifdef USE_VM_PROBES + , dt_priv, dt_utag +#endif + ) < 0) { + reply_posix_error(desc, err); + goto done; + } /* Create the thread data structure with the contained ErlIOVec * and corresponding binaries for the response */ @@ -3339,6 +3887,12 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { d->flags = desc->flags; d->c.preadv.offsets[0] = hdr_offset; d->c.preadv.size = max_size; +#ifdef USE_VM_PROBES + dt_i1 = d->fd; + dt_i2 = d->flags; + dt_i3 = d->c.preadv.offsets[0]; + dt_i4 = d->c.preadv.size; +#endif res_ev = &d->c.preadv.eiov; /* XXX possible alignment problems here for weird machines */ res_ev->iov = void_ptr = d + 1; @@ -3353,16 +3907,24 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { case FILE_SETOPT: { char opt; + if (ev->size < 1+1 || !EV_GET_CHAR(ev, &opt, &p, &q)) { /* Buffer too short to contain even the option type */ reply_posix_error(desc, EINVAL); goto done; } +#ifdef USE_VM_PROBES + dt_i1 = opt; + dt_utag = EV_CHAR_P(ev, p, q); +#endif switch (opt) { case FILE_OPT_DELAYED_WRITE: { Uint32 sizeH, sizeL, delayH, delayL; if (ev->size != 1+1+4*sizeof(Uint32) +#ifdef USE_VM_PROBES + + FILENAME_BYTELEN(dt_utag) + FILENAME_CHARSIZE +#endif || !EV_GET_UINT32(ev, &sizeH, &p, &q) || !EV_GET_UINT32(ev, &sizeL, &p, &q) || !EV_GET_UINT32(ev, &delayH, &p, &q) @@ -3389,12 +3951,18 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { #else desc->write_delay = ((unsigned long)delayH << 32) | delayL; #endif +#ifdef USE_VM_PROBES + dt_i2 = desc->write_delay; +#endif TRACE_C('K'); reply_ok(desc); } goto done; case FILE_OPT_READ_AHEAD: { Uint32 sizeH, sizeL; if (ev->size != 1+1+2*sizeof(Uint32) +#ifdef USE_VM_PROBES + + FILENAME_BYTELEN(dt_utag)+FILENAME_CHARSIZE +#endif || !EV_GET_UINT32(ev, &sizeH, &p, &q) || !EV_GET_UINT32(ev, &sizeL, &p, &q)) { /* Buffer has wrong length to contain the option values */ @@ -3410,6 +3978,9 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { #else desc->read_bufsize = ((size_t)sizeH << 32) | sizeL; #endif +#ifdef USE_VM_PROBES + dt_i2 = desc->read_bufsize; +#endif TRACE_C('K'); reply_ok(desc); } goto done; @@ -3450,11 +4021,13 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { d->fd = desc->fd; d->command = command; d->invoke = invoke_sendfile; - d->free = NULL; + d->free = free_sendfile; d->level = 2; d->c.sendfile.out_fd = (int) out_fd; d->c.sendfile.written = 0; + d->c.sendfile.port = desc->port; + d->c.sendfile.q_mtx = desc->q_mtx; #if SIZEOF_OFF_T == 4 if (offsetH != 0) { @@ -3470,6 +4043,19 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { if (USE_THRDS_FOR_SENDFILE) { SET_BLOCKING(d->c.sendfile.out_fd); + } else { + /** + * Write a place holder to queue in order to force file_flush + * to be called before the driver is closed. + */ + char tmp[1] = ""; + MUTEX_LOCK(d->c.sendfile.q_mtx); + if (driver_enq(d->c.sendfile.port, tmp, 1)) { + MUTEX_UNLOCK(d->c.sendfile.q_mtx); + reply_posix_error(desc, ENOMEM); + goto done; + } + MUTEX_UNLOCK(d->c.sendfile.q_mtx); } cq_enq(desc, d); @@ -3481,11 +4067,19 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { } /* switch(command) */ - if (lseek_flush_read(desc, &err) < 0) { + if (lseek_flush_read(desc, &err +#ifdef USE_VM_PROBES + , dt_priv, dt_utag +#endif + ) < 0) { reply_posix_error(desc, err); goto done; } - if (flush_write_check_error(desc, &err) < 0) { + if (flush_write_check_error(desc, &err +#ifdef USE_VM_PROBES + , dt_priv, dt_utag +#endif + ) < 0) { reply_posix_error(desc, err); goto done; } else { @@ -3503,5 +4097,50 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { } done: + if (d != NULL) { +#ifdef USE_VM_PROBES + /* + * If d == NULL, then either: + * 1). There was an error of some sort, or + * 2). The command given to us is actually implemented + * by file_output() instead. + * + * Case #1 is probably a TODO item, perhaps? + * Case #2 we definitely don't want to activate a probe. + */ + d->sched_i1 = dt_priv->thread_num; + d->sched_i2 = dt_priv->tag; + d->sched_utag[0] = '\0'; + if (dt_utag != NULL) { + if (dt_utag[0] == '\0') { + dt_utag = NULL; + } else { + strncpy(d->sched_utag, dt_utag, sizeof(d->sched_utag) - 1); + d->sched_utag[sizeof(d->sched_utag) - 1] = '\0'; + } + } + DTRACE11(efile_drv_entry, dt_priv->thread_num, dt_priv->tag++, + dt_utag, command, dt_s1, NULL, dt_i1, dt_i2, dt_i3, dt_i4, + desc->port_str); +#endif + } cq_execute(desc); } + +#ifdef USE_VM_PROBES +dt_private * +get_dt_private(int base) +{ + dt_private *dt_priv = (dt_private *) pthread_getspecific(dt_driver_key); + + if (dt_priv == NULL) { + dt_priv = EF_SAFE_ALLOC(sizeof(dt_private)); + erts_mtx_lock(&dt_driver_mutex); + dt_priv->thread_num = (base + dt_driver_idnum++); + erts_mtx_unlock(&dt_driver_mutex); + dt_priv->tag = 0; + pthread_setspecific(dt_driver_key, dt_priv); + } + return dt_priv; +} +#endif /* USE_VM_PROBES */ diff --git a/erts/emulator/drivers/common/erl_efile.h b/erts/emulator/drivers/common/erl_efile.h index 3868b38137..69ad02633c 100644 --- a/erts/emulator/drivers/common/erl_efile.h +++ b/erts/emulator/drivers/common/erl_efile.h @@ -162,7 +162,7 @@ int efile_write_info(Efile_error* errInfo, Efile_info* pInfo, char *name); int efile_write(Efile_error* errInfo, int flags, int fd, char* buf, size_t count); int efile_writev(Efile_error* errInfo, int flags, int fd, - SysIOVec* iov, int iovcnt, size_t size); + SysIOVec* iov, int iovcnt); int efile_read(Efile_error* errInfo, int flags, int fd, char* buf, size_t count, size_t* pBytesRead); int efile_seek(Efile_error* errInfo, int fd, diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index eeaa4d24ea..d1c2dbf94c 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -553,6 +553,12 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) # define VALGRIND_MAKE_MEM_DEFINED(ptr,size) #endif +/* + Magic errno value used locally for return of {error, system_limit} + - the emulator definition of SYSTEM_LIMIT is not available here. +*/ +#define INET_ERRNO_SYSTEM_LIMIT (15 << 8) + /*---------------------------------------------------------------------------- ** Interface constants. ** @@ -1645,6 +1651,17 @@ static struct erl_drv_entry dummy_sctp_driver_entry = #endif +/* return lowercase string form of errno value */ +static char *errno_str(int err) +{ + switch (err) { + case INET_ERRNO_SYSTEM_LIMIT: + return "system_limit"; + default: + return erl_errno_id(err); + } +} + /* general control reply function */ static ErlDrvSSizeT ctl_reply(int rep, char* buf, ErlDrvSizeT len, char** rbuf, ErlDrvSizeT rsize) @@ -1665,13 +1682,9 @@ static ErlDrvSSizeT ctl_reply(int rep, char* buf, ErlDrvSizeT len, /* general control error reply function */ static ErlDrvSSizeT ctl_error(int err, char** rbuf, ErlDrvSizeT rsize) { - char response[256]; /* Response buffer. */ - char* s; - char* t; + char* s = errno_str(err); - for (s = erl_errno_id(err), t = response; *s; s++, t++) - *t = tolower(*s); - return ctl_reply(INET_REP_ERROR, response, t-response, rbuf, rsize); + return ctl_reply(INET_REP_ERROR, s, strlen(s), rbuf, rsize); } static ErlDrvSSizeT ctl_xerror(char* xerr, char** rbuf, ErlDrvSizeT rsize) @@ -1683,14 +1696,7 @@ static ErlDrvSSizeT ctl_xerror(char* xerr, char** rbuf, ErlDrvSizeT rsize) static ErlDrvTermData error_atom(int err) { - char errstr[256]; - char* s; - char* t; - - for (s = erl_errno_id(err), t = errstr; *s; s++, t++) - *t = tolower(*s); - *t = '\0'; - return driver_mk_atom(errstr); + return driver_mk_atom(errno_str(err)); } @@ -4089,6 +4095,7 @@ static char* buf_to_sockaddr(char* ptr, char* end, struct sockaddr* addr) addr->sa_family = AF_INET; return ptr + sizeof(struct in_addr); } +#if defined(HAVE_IN6) && defined(AF_INET6) case INET_AF_INET6: { struct in6_addr *p = &((struct sockaddr_in6*)addr)->sin6_addr; buf_check(ptr,end,sizeof(struct in6_addr)); @@ -4096,6 +4103,7 @@ static char* buf_to_sockaddr(char* ptr, char* end, struct sockaddr* addr) addr->sa_family = AF_INET6; return ptr + sizeof(struct in6_addr); } +#endif } error: return NULL; @@ -6194,6 +6202,7 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len) type = SCTP_DEFAULT_SEND_PARAM; arg_ptr = (char*) (&arg.sri); arg_sz = sizeof ( arg.sri); + VALGRIND_MAKE_MEM_DEFINED(arg_ptr, arg_sz); /*suppress "uninitialised bytes"*/ break; } case SCTP_OPT_EVENTS: @@ -8050,7 +8059,7 @@ static ErlDrvData tcp_inet_start(ErlDrvPort port, char* args) /* Copy a descriptor, by creating a new port with same settings * as the descriptor desc. - * return NULL on error (ENFILE no ports avail) + * return NULL on error (SYSTEM_LIMIT no ports avail) */ static tcp_descriptor* tcp_inet_copy(tcp_descriptor* desc,SOCKET s, ErlDrvTermData owner, int* err) @@ -8089,7 +8098,7 @@ static tcp_descriptor* tcp_inet_copy(tcp_descriptor* desc,SOCKET s, /* The new port will be linked and connected to the original caller */ port = driver_create_port(port, owner, "tcp_inet", (ErlDrvData) copy_desc); if ((long)port == -1) { - *err = ENFILE; + *err = INET_ERRNO_SYSTEM_LIMIT; FREE(copy_desc); return NULL; } @@ -10299,7 +10308,6 @@ static void packet_inet_command(ErlDrvData e, char* buf, ErlDrvSizeT len) cmsg.hdr.cmsg_level = IPPROTO_SCTP; cmsg.hdr.cmsg_type = SCTP_SNDRCV; cmsg.hdr.cmsg_len = CMSG_LEN(sizeof(*sri)); - VALGRIND_MAKE_MEM_DEFINED(&cmsg, (char*)sri - (char*)&cmsg); /*suppress padding as "uninitialised bytes"*/ data_len = (buf + len) - ptr; /* The whole msg. @@ -10313,6 +10321,7 @@ static void packet_inet_command(ErlDrvData e, char* buf, ErlDrvSizeT len) mhdr.msg_iovlen = 1; mhdr.msg_control = cmsg.ancd; /* For ancilary data */ mhdr.msg_controllen = cmsg.hdr.cmsg_len; + VALGRIND_MAKE_MEM_DEFINED(mhdr.msg_control, mhdr.msg_controllen); /*suppress "uninitialised bytes"*/ mhdr.msg_flags = 0; /* Not used with "sendmsg" */ /* Now do the actual sending. NB: "flags" in "sendmsg" itself are NOT diff --git a/erts/emulator/drivers/common/zlib_drv.c b/erts/emulator/drivers/common/zlib_drv.c index da4a17db1a..a4b02b845e 100644 --- a/erts/emulator/drivers/common/zlib_drv.c +++ b/erts/emulator/drivers/common/zlib_drv.c @@ -64,6 +64,7 @@ static int zlib_init(void); static ErlDrvData zlib_start(ErlDrvPort port, char* buf); static void zlib_stop(ErlDrvData e); +static void zlib_flush(ErlDrvData e); static ErlDrvSSizeT zlib_ctl(ErlDrvData drv_data, unsigned int command, char *buf, ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen); static void zlib_outputv(ErlDrvData drv_data, ErlIOVec *ev); @@ -82,7 +83,7 @@ ErlDrvEntry zlib_driver_entry = { NULL, /* timeout */ zlib_outputv, NULL, /* read_async */ - NULL, /* flush */ + zlib_flush, NULL, /* call */ NULL, /* event */ ERL_DRV_EXTENDED_MARKER, @@ -410,6 +411,13 @@ static void zlib_stop(ErlDrvData e) driver_free(d); } +static void zlib_flush(ErlDrvData drv_data) +{ + ZLibData* d = (ZLibData*) drv_data; + + driver_deq(d->port, driver_sizeq(d->port)); +} + static ErlDrvSSizeT zlib_ctl(ErlDrvData drv_data, unsigned int command, char *buf, ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen) { diff --git a/erts/emulator/drivers/unix/unix_efile.c b/erts/emulator/drivers/unix/unix_efile.c index 7cf0a712ce..dfb6cece14 100644 --- a/erts/emulator/drivers/unix/unix_efile.c +++ b/erts/emulator/drivers/unix/unix_efile.c @@ -1004,13 +1004,11 @@ efile_writev(Efile_error* errInfo, /* Where to return error codes */ * opened */ int fd, /* File descriptor to write to */ SysIOVec* iov, /* Vector of buffer structs. - * The structs are unchanged - * after the call */ - int iovcnt, /* Number of structs in vector */ - size_t size) /* Number of bytes to write */ + * The structs may be changed i.e. + * due to incomplete writes */ + int iovcnt) /* Number of structs in vector */ { int cnt = 0; /* Buffers so far written */ - int p = 0; /* Position in next buffer */ ASSERT(iovcnt >= 0); @@ -1021,66 +1019,47 @@ efile_writev(Efile_error* errInfo, /* Where to return error codes */ #endif while (cnt < iovcnt) { + if ((! iov[cnt].iov_base) || (iov[cnt].iov_len <= 0)) { + /* Empty buffer - skip */ + cnt++; + } else { /* Non-empty buffer */ + ssize_t w; /* Bytes written in this call */ #ifdef HAVE_WRITEV - int w; /* Bytes written in this call */ - int b = iovcnt - cnt; /* Buffers to write */ - if (b > MAXIOV) - b = MAXIOV; - if (iov[cnt].iov_base && iov[cnt].iov_len > 0) { - if (b == 1) { - /* Degenerated io vector */ - do { - w = write(fd, iov[cnt].iov_base + p, iov[cnt].iov_len - p); - } while (w < 0 && errno == EINTR); - } else { - /* Non-empty vector first. - * Adjust pos in first buffer in case of - * previous incomplete writev */ - iov[cnt].iov_base += p; - iov[cnt].iov_len -= p; + int b = iovcnt - cnt; /* Buffers to write */ + /* Use as many buffers as MAXIOV allows */ + if (b > MAXIOV) + b = MAXIOV; + if (b > 1) { do { w = writev(fd, &iov[cnt], b); } while (w < 0 && errno == EINTR); - iov[cnt].iov_base -= p; - iov[cnt].iov_len += p; - } - if (w < 0) - return check_error(-1, errInfo); - } else { - /* Empty vector first - skip */ - cnt++; - continue; - } - ASSERT(w >= 0); - /* Move forward to next vector to write */ - for (; cnt < iovcnt; cnt++) { - if (iov[cnt].iov_base && iov[cnt].iov_len > 0) { - if (w < iov[cnt].iov_len) - break; - else - w -= iov[cnt].iov_len; - } - } - ASSERT(w >= 0); - p = w > 0 ? w : 0; /* Skip p bytes next writev */ -#else /* #ifdef HAVE_WRITEV */ - if (iov[cnt].iov_base && iov[cnt].iov_len > 0) { - /* Non-empty vector */ - int w; /* Bytes written in this call */ - while (p < iov[cnt].iov_len) { - do { - w = write(fd, iov[cnt].iov_base + p, iov[cnt].iov_len - p); - } while (w < 0 && errno == EINTR); - if (w < 0) - return check_error(-1, errInfo); - p += w; + } else + /* Degenerated io vector - use regular write */ +#endif + { + do { + w = write(fd, iov[cnt].iov_base, iov[cnt].iov_len); + } while (w < 0 && errno == EINTR); + ASSERT(w <= iov[cnt].iov_len); + } + if (w < 0) return check_error(-1, errInfo); + /* Move forward to next buffer to write */ + for (; cnt < iovcnt && w > 0; cnt++) { + if (iov[cnt].iov_base && iov[cnt].iov_len > 0) { + if (w < iov[cnt].iov_len) { + /* Adjust the buffer for next write */ + iov[cnt].iov_len -= w; + iov[cnt].iov_base += w; + w = 0; + break; + } else { + w -= iov[cnt].iov_len; + } + } } - } - cnt++; - p = 0; -#endif /* #ifdef HAVE_WRITEV */ + ASSERT(w == 0); + } /* else Non-empty buffer */ } /* while (cnt< iovcnt) */ - size = 0; /* Avoid compiler warning */ return 1; } @@ -1427,10 +1406,9 @@ efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, } #ifdef HAVE_SENDFILE - // For some reason the maximum size_t cannot be used as the max size // 3GB seems to work on all platforms -#define SENDFILE_CHUNK_SIZE ((1 << 30) -1) +#define SENDFILE_CHUNK_SIZE ((1UL << 30) -1) /* * sendfile: The implementation of the sendfile system call varies @@ -1448,6 +1426,14 @@ efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, * you would have to emulate it in linux and on BSD/Darwin some complex * calculations have to be made when using a non blocking socket to figure * out how much of the header/file/trailer was sent in each command. + * + * The semantics of the API is this: + * Return value: 1 if all data was sent and the function does not need to + * be called again. 0 if an error occures OR if there is more data which + * has to be sent (EAGAIN or EINTR will be set appropriately) + * + * The amount of data written in a call is returned through nbytes. + * */ int @@ -1467,9 +1453,12 @@ efile_sendfile(Efile_error* errInfo, int in_fd, int out_fd, written += retval; *nbytes -= retval; } - } while (retval != -1 && retval == SENDFILE_CHUNK_SIZE); - *nbytes = written; - return check_error(retval == -1 ? -1 : 0, errInfo); + } while (retval == SENDFILE_CHUNK_SIZE); + if (written != 0) { + // -1 is not returned by the linux API so we have to simulate it + retval = -1; + errno = EAGAIN; + } #elif defined(__sun) && defined(__SVR4) && defined(HAVE_SENDFILEV) ssize_t retval; size_t len; @@ -1491,8 +1480,6 @@ efile_sendfile(Efile_error* errInfo, int in_fd, int out_fd, written += len; } } while (len == SENDFILE_CHUNK_SIZE); - *nbytes = written; - return check_error(retval == -1 ? -1 : 0, errInfo); #elif defined(DARWIN) int retval; off_t len; @@ -1509,8 +1496,6 @@ efile_sendfile(Efile_error* errInfo, int in_fd, int out_fd, written += len; } } while (len == SENDFILE_CHUNK_SIZE); - *nbytes = written; - return check_error(retval, errInfo); #elif defined(__FreeBSD__) || defined(__DragonFly__) off_t len; int retval; @@ -1526,8 +1511,8 @@ efile_sendfile(Efile_error* errInfo, int in_fd, int out_fd, written += len; } } while(len == SENDFILE_CHUNK_SIZE); +#endif *nbytes = written; return check_error(retval, errInfo); -#endif } #endif /* HAVE_SENDFILE */ diff --git a/erts/emulator/drivers/win32/win_efile.c b/erts/emulator/drivers/win32/win_efile.c index 0d3d334154..606fa1d7de 100644 --- a/erts/emulator/drivers/win32/win_efile.c +++ b/erts/emulator/drivers/win32/win_efile.c @@ -1115,8 +1115,7 @@ efile_writev(Efile_error* errInfo, /* Where to return error codes */ SysIOVec* iov, /* Vector of buffer structs. * The structs are unchanged * after the call */ - int iovcnt, /* Number of structs in vector */ - size_t size) /* Number of bytes to write */ + int iovcnt) /* Number of structs in vector */ { int cnt; /* Buffers so far written */ OVERLAPPED overlapped; diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c index 0199dea99e..e0575c35ff 100644 --- a/erts/emulator/hipe/hipe_gc.c +++ b/erts/emulator/hipe/hipe_gc.c @@ -46,9 +46,14 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) char *src, *oh; Uint src_size, oh_size; + if (!p->hipe.nstack) { + ASSERT(!p->hipe.nsp && !p->hipe.nstend); + return n_htop; + } if (!nstack_walk_init_check(p)) return n_htop; + ASSERT(p->hipe.nsp && p->hipe.nstend); nsp = nstack_walk_nsp_begin(p); nsp_end = p->hipe.nstgraylim; if (nsp_end) @@ -136,9 +141,14 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) char *heap; Uint heap_size, mature_size; + if (!p->hipe.nstack) { + ASSERT(!p->hipe.nsp && !p->hipe.nstend); + return; + } if (!nstack_walk_init_check(p)) return; + ASSERT(p->hipe.nsp && p->hipe.nstend); nsp = nstack_walk_nsp_begin(p); nsp_end = p->hipe.nstgraylim; if (nsp_end) { diff --git a/erts/emulator/pcre/pcre.mk b/erts/emulator/pcre/pcre.mk index b752c11459..352137b341 100644 --- a/erts/emulator/pcre/pcre.mk +++ b/erts/emulator/pcre/pcre.mk @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2011. All Rights Reserved. +# Copyright Ericsson AB 2012. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -17,8 +17,6 @@ # %CopyrightEnd% # -ARFLAGS = rc - PCRE_O = \ pcre_latin_1_table.o \ pcre_compile.o \ @@ -43,7 +41,7 @@ pcre_xclass.o PCRE_OBJS = $(PCRE_O:%=$(PCRE_OBJDIR)/%) -GENINC = pcre/pcre_exec_loop_break_cases.inc +PCRE_GENINC = $(ERL_TOP)/erts/emulator/pcre/pcre_exec_loop_break_cases.inc PCRE_OBJDIR = $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE) @@ -61,12 +59,12 @@ endif $(PCRE_OBJDIR)/%.o: pcre/%.c $(CC) -c $(PCRE_CFLAGS) -o $@ $< -$(GENINC): pcre/pcre_exec.c +$(PCRE_GENINC): pcre/pcre_exec.c for x in `grep -n COST_CHK pcre/pcre_exec.c | grep -v 'COST_CHK(N)' | awk -F: '{print $$1}'`; \ do \ N=`expr $$x + 100`; \ echo "case $$N: goto L_LOOP_COUNT_$${x};"; \ - done > $(GENINC) + done > $(PCRE_GENINC) # Dependencies. @@ -79,7 +77,7 @@ $(PCRE_OBJDIR)/pcre_config.o: pcre/pcre_config.c pcre/pcre_internal.h \ $(PCRE_OBJDIR)/pcre_dfa_exec.o: pcre/pcre_dfa_exec.c pcre/pcre_internal.h \ pcre/local_config.h pcre/pcre.h pcre/ucp.h $(PCRE_OBJDIR)/pcre_exec.o: pcre/pcre_exec.c pcre/pcre_internal.h \ - pcre/local_config.h pcre/pcre.h pcre/ucp.h $(GENINC) + pcre/local_config.h pcre/pcre.h pcre/ucp.h $(PCRE_GENINC) $(PCRE_OBJDIR)/pcre_fullinfo.o: pcre/pcre_fullinfo.c pcre/pcre_internal.h \ pcre/local_config.h pcre/pcre.h pcre/ucp.h $(PCRE_OBJDIR)/pcre_get.o: pcre/pcre_get.c pcre/pcre_internal.h \ diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index 23a4bf1b04..1d173a758a 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -36,6 +36,7 @@ #include "global.h" #include "erl_check_io.h" #include "erl_thr_progress.h" +#include "dtrace-wrapper.h" #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS # define ERTS_DRV_EV_STATE_EXTRA_SIZE 128 @@ -314,6 +315,7 @@ forget_removed(struct pollset_info* psi) erts_smp_mtx_unlock(mtx); if (drv_ptr) { int was_unmasked = erts_block_fpe(); + DTRACE1(driver_stop_select, drv_ptr->name); (*drv_ptr->stop_select) ((ErlDrvEvent) fd, NULL); erts_unblock_fpe(was_unmasked); if (drv_ptr->handle) { @@ -496,6 +498,9 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, ErtsDrvEventState *state; int wake_poller; int ret; +#ifdef USE_VM_PROBES + DTRACE_CHARBUF(name, 64); +#endif ERTS_SMP_LC_ASSERT(erts_drvport2port(ix) && erts_lc_is_port_locked(erts_drvport2port(ix))); @@ -525,6 +530,10 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, if (IS_FD_UNKNOWN(state)) { /* fast track to stop_select callback */ stop_select_fn = erts_drvport2port(ix)->drv_ptr->stop_select; +#ifdef USE_VM_PROBES + strncpy(name, erts_drvport2port(ix)->drv_ptr->name, sizeof(name)-1); + name[sizeof(name)-1] = '\0'; +#endif ret = 0; goto done_unknown; } @@ -661,6 +670,10 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, /* Safe to close fd now as it is not in pollset or there was no need to eject fd (kernel poll) */ stop_select_fn = drv_ptr->stop_select; +#ifdef USE_VM_PROBES + strncpy(name, erts_drvport2port(ix)->drv_ptr->name, sizeof(name)-1); + name[sizeof(name)-1] = '\0'; +#endif } else { /* Not safe to close fd, postpone stop_select callback. */ @@ -686,6 +699,7 @@ done_unknown: erts_smp_mtx_unlock(fd_mtx(fd)); if (stop_select_fn) { int was_unmasked = erts_block_fpe(); + DTRACE1(driver_stop_select, name); (*stop_select_fn)(e, NULL); erts_unblock_fpe(was_unmasked); } diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl index c7617d3b90..a21b055596 100644 --- a/erts/emulator/test/bif_SUITE.erl +++ b/erts/emulator/test/bif_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2011. All Rights Reserved. +%% Copyright Ericsson AB 2005-2012. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -28,7 +28,7 @@ types/1, t_list_to_existing_atom/1,os_env/1,otp_7526/1, binary_to_atom/1,binary_to_existing_atom/1, - atom_to_binary/1,min_max/1]). + atom_to_binary/1,min_max/1, erlang_halt/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -36,7 +36,7 @@ all() -> [types, t_list_to_existing_atom, os_env, otp_7526, display, atom_to_binary, binary_to_atom, binary_to_existing_atom, - min_max]. + min_max, erlang_halt]. groups() -> []. @@ -438,7 +438,55 @@ min_max(Config) when is_list(Config) -> ok. + + +erlang_halt(Config) when is_list(Config) -> + try erlang:halt(undefined) of + _-> ?t:fail({erlang,halt,{undefined}}) + catch error:badarg -> ok end, + try halt(undefined) of + _-> ?t:fail({halt,{undefined}}) + catch error:badarg -> ok end, + try erlang:halt(undefined, []) of + _-> ?t:fail({erlang,halt,{undefined,[]}}) + catch error:badarg -> ok end, + try halt(undefined, []) of + _-> ?t:fail({halt,{undefined,[]}}) + catch error:badarg -> ok end, + try halt(0, undefined) of + _-> ?t:fail({halt,{0,undefined}}) + catch error:badarg -> ok end, + try halt(0, [undefined]) of + _-> ?t:fail({halt,{0,[undefined]}}) + catch error:badarg -> ok end, + try halt(0, [{undefined,true}]) of + _-> ?t:fail({halt,{0,[{undefined,true}]}}) + catch error:badarg -> ok end, + try halt(0, [{flush,undefined}]) of + _-> ?t:fail({halt,{0,[{flush,undefined}]}}) + catch error:badarg -> ok end, + try halt(0, [{flush,true,undefined}]) of + _-> ?t:fail({halt,{0,[{flush,true,undefined}]}}) + catch error:badarg -> ok end, + H = hostname(), + {ok,N1} = slave:start(H, halt_node1), + {badrpc,nodedown} = rpc:call(N1, erlang, halt, []), + {ok,N2} = slave:start(H, halt_node2), + {badrpc,nodedown} = rpc:call(N2, erlang, halt, [0]), + {ok,N3} = slave:start(H, halt_node3), + {badrpc,nodedown} = rpc:call(N3, erlang, halt, [0,[]]), + ok. + + + %% Helpers id(I) -> I. +hostname() -> + hostname(atom_to_list(node())). + +hostname([$@ | Hostname]) -> + list_to_atom(Hostname); +hostname([_C | Cs]) -> + hostname(Cs). diff --git a/erts/emulator/test/call_trace_SUITE.erl b/erts/emulator/test/call_trace_SUITE.erl index 3e2bee06d1..7030ebed3f 100644 --- a/erts/emulator/test/call_trace_SUITE.erl +++ b/erts/emulator/test/call_trace_SUITE.erl @@ -165,10 +165,14 @@ worker_loop() -> worker_foo(_Arg) -> ok. -basic(doc) -> - "Basic test of the call tracing (we trace one process)."; -basic(suite) -> []; -basic(Config) when is_list(Config) -> +%% Basic test of the call tracing (we trace one process). +basic(_Config) -> + case test_server:is_native(lists) of + true -> {skip,"lists is native"}; + false -> basic() + end. + +basic() -> ?line start_tracer(), ?line trace_info(self(), flags), ?line trace_info(self(), tracer), @@ -263,9 +267,15 @@ foo() -> foo0. foo(X) -> X+1. foo(X, Y) -> X+Y. -flags(doc) -> "Test flags (arity, timestamp) for call_trace/3. " - "Also, test the '{tracer,Pid}' option."; -flags(Config) when is_list(Config) -> +%% Test flags (arity, timestamp) for call_trace/3. +%% Also, test the '{tracer,Pid}' option. +flags(_Config) -> + case test_server:is_native(filename) of + true -> {skip,"filename is native"}; + false -> flags() + end. + +flags() -> ?line Tracer = start_tracer_loop(), ?line trace_pid(self(), true, [call,{tracer,Tracer}]), @@ -428,9 +438,14 @@ pam_foo(A, B) -> {ok,A,B}. -change_pam(doc) -> "Test changing PAM programs for a function."; -change_pam(suite) -> []; -change_pam(Config) when is_list(Config) -> +%% Test changing PAM programs for a function. +change_pam(_Config) -> + case test_server:is_native(lists) of + true -> {skip,"lists is native"}; + false -> change_pam() + end. + +change_pam() -> ?line start_tracer(), ?line Self = self(), @@ -468,10 +483,11 @@ change_pam_trace(Prog) -> {match_spec,Prog} = trace_info({erlang,process_info,2}, match_spec), ok. -return_trace(doc) -> "Test the new return trace."; -return_trace(suite) -> []; -return_trace(Config) when is_list(Config) -> - return_trace(). +return_trace(_Config) -> + case test_server:is_native(lists) of + true -> {skip,"lists is native"}; + false -> return_trace() + end. return_trace() -> X = {save,me}, @@ -521,7 +537,7 @@ return_trace() -> ?line {match_spec,Prog2} = trace_info({erlang,atom_to_list,1}, match_spec), ?line lists:seq(2, 7), - ?line atom_to_list(non_literal(nisse)), + ?line _ = atom_to_list(non_literal(nisse)), ?line expect({trace,Self,return_from,{lists,seq,2},[2,3,4,5,6,7]}), ?line expect({trace,Self,return_from,{erlang,atom_to_list,1},"nisse"}), @@ -539,10 +555,11 @@ return_trace() -> nasty() -> exit(good_bye). -exception_trace(doc) -> "Test the new exception trace."; -exception_trace(suite) -> []; -exception_trace(Config) when is_list(Config) -> - exception_trace(). +exception_trace(_Config) -> + case test_server:is_native(lists) of + true -> {skip,"lists is native"}; + false -> exception_trace() + end. exception_trace() -> X = {save,me}, @@ -600,7 +617,7 @@ exception_trace() -> trace_info({erlang,atom_to_list,1}, match_spec), ?line lists:seq(2, 7), - ?line atom_to_list(non_literal(nisse)), + ?line _ = atom_to_list(non_literal(nisse)), ?line expect({trace,Self,return_from,{lists,seq,2},[2,3,4,5,6,7]}), ?line expect({trace,Self,return_from,{erlang,atom_to_list,1},"nisse"}), diff --git a/erts/emulator/test/erts_debug_SUITE.erl b/erts/emulator/test/erts_debug_SUITE.erl index 4dc2fbaae2..76667772c7 100644 --- a/erts/emulator/test/erts_debug_SUITE.erl +++ b/erts/emulator/test/erts_debug_SUITE.erl @@ -23,13 +23,13 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2, - flat_size/1,flat_size_big/1,df/1, + test_size/1,flat_size_big/1,df/1, instructions/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [flat_size, flat_size_big, df, instructions]. + [test_size, flat_size_big, df, instructions]. groups() -> []. @@ -55,16 +55,58 @@ end_per_testcase(_Func, Config) -> Dog=?config(watchdog, Config), ?t:timetrap_cancel(Dog). -flat_size(Config) when is_list(Config) -> - 0 = erts_debug:flat_size([]), - 0 = erts_debug:flat_size(42), - 2 = erts_debug:flat_size([a|b]), - 1 = erts_debug:flat_size({}), - 2 = erts_debug:flat_size({[]}), - 3 = erts_debug:flat_size({a,b}), - 7 = erts_debug:flat_size({a,[b,c]}), +test_size(Config) when is_list(Config) -> + ConsCell1 = id([a|b]), + ConsCell2 = id(ConsCell1), + ConsCellSz = 2, + + 0 = do_test_size([]), + 0 = do_test_size(42), + ConsCellSz = do_test_size(ConsCell1), + 1 = do_test_size({}), + 2 = do_test_size({[]}), + 3 = do_test_size({a,b}), + 7 = do_test_size({a,[b,c]}), + + %% Test internal consistency of sizes, but without testing + %% exact sizes. + Const = id(42), + AnotherConst = id(7), + + %% Fun environment size = 0 (the smallest fun possible) + SimplestFun = fun() -> ok end, + FunSz0 = do_test_size(SimplestFun), + + %% Fun environment size = 1 + FunSz1 = do_test_size(fun() -> Const end), + FunSz1 = FunSz0 + 1, + + %% Fun environment size = 2 + FunSz2 = do_test_size(fun() -> Const+AnotherConst end), + FunSz2 = FunSz1 + 1, + + FunSz1 = do_test_size(fun() -> ConsCell1 end) - do_test_size(ConsCell1), + + %% Test shared data structures. + do_test_size([ConsCell1|ConsCell1], + 3*ConsCellSz, + 2*ConsCellSz), + do_test_size(fun() -> {ConsCell1,ConsCell2} end, + FunSz2 + 2*ConsCellSz, + FunSz2 + ConsCellSz), + do_test_size({SimplestFun,SimplestFun}, + 2*FunSz0+do_test_size({a,b}), + FunSz0+do_test_size({a,b})), ok. +do_test_size(Term) -> + Sz = erts_debug:flat_size(Term), + Sz = erts_debug:size(Term). + +do_test_size(Term, FlatSz, Sz) -> + FlatSz = erts_debug:flat_size(Term), + Sz = erts_debug:size(Term). + flat_size_big(Config) when is_list(Config) -> %% Build a term whose external size only fits in a big num (on 32-bit CPU). flat_size_big_1(16#11111111111111117777777777777777888889999, 0, 16#FFFFFFF). @@ -96,3 +138,6 @@ instructions(Config) when is_list(Config) -> ?line Is = erts_debug:instructions(), ?line _ = [list_to_atom(I) || I <- Is], ok. + +id(I) -> + I. diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index 370363bf9e..6bd7361612 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -859,7 +859,13 @@ resource_holder(Pid,Reply,List) -> threading(doc) -> ["Test the threading API functions (reuse tests from driver API)"]; -threading(Config) when is_list(Config) -> +threading(Config) when is_list(Config) -> + case erlang:system_info(threads) of + true -> threading_do(Config); + false -> {skipped,"No thread support"} + end. + +threading_do(Config) -> ?line Data = ?config(data_dir, Config), ?line File = filename:join(Data, "tester"), ?line {ok,tester,ModBin} = compile:file(File, [binary,return_errors]), diff --git a/erts/emulator/test/nif_SUITE_data/tester.c b/erts/emulator/test/nif_SUITE_data/tester.c index 08466d0f18..257b116322 100644 --- a/erts/emulator/test/nif_SUITE_data/tester.c +++ b/erts/emulator/test/nif_SUITE_data/tester.c @@ -61,6 +61,7 @@ static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) static ERL_NIF_TERM run(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { testcase_run(NULL); + testcase_cleanup(NULL); return enif_make_atom(env, "ok"); } diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl index 0392312a6f..a93dd309c1 100644 --- a/erts/emulator/test/statistics_SUITE.erl +++ b/erts/emulator/test/statistics_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2011. All Rights Reserved. +%% Copyright Ericsson AB 1997-2012. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,6 +29,7 @@ runtime_zero_diff/1, runtime_update/1, runtime_diff/1, run_queue_one/1, + scheduler_wall_time/1, reductions/1, reductions_big/1, garbage_collection/1, io/1, badarg/1]). @@ -51,8 +52,8 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [{group, wall_clock}, {group, runtime}, reductions, - reductions_big, {group, run_queue}, garbage_collection, - io, badarg]. + reductions_big, {group, run_queue}, scheduler_wall_time, + garbage_collection, io, badarg]. groups() -> [{wall_clock, [], @@ -266,11 +267,10 @@ run_queue_one(Config) when is_list(Config) -> run_queue_one_test(Config) when is_list(Config) -> - ?line Hog = spawn_link(?MODULE, hog, [self()]), + ?line _Hog = spawn_link(?MODULE, hog, [self()]), ?line receive - hog_started -> - Hog ! go - end, + hog_started -> ok + end, ?line receive after 100 -> ok end, % Give hog a head start. ?line case statistics(run_queue) of N when N >= 1 -> ok; @@ -280,18 +280,88 @@ run_queue_one_test(Config) when is_list(Config) -> %% CPU-bound process, going at low priority. It will always be ready %% to run. - + hog(Pid) -> ?line process_flag(priority, low), ?line Pid ! hog_started, - ?line receive - go -> hog_iter(0) + ?line Mon = erlang:monitor(process, Pid), + ?line hog_iter(0, Mon). + +hog_iter(N, Mon) when N > 0 -> + receive + {'DOWN', Mon, _, _, _} -> ok + after 0 -> + ?line hog_iter(N-1, Mon) + end; +hog_iter(0, Mon) -> + ?line hog_iter(10000, Mon). + +%%% Tests of statistics(scheduler_wall_time). + +scheduler_wall_time(doc) -> + "Tests that statistics(scheduler_wall_time) works as intended"; +scheduler_wall_time(Config) when is_list(Config) -> + %% Should return undefined if system_flag is not turned on yet + undefined = statistics(scheduler_wall_time), + %% Turn on statistics + false = erlang:system_flag(scheduler_wall_time, true), + try + Schedulers = erlang:system_info(schedulers_online), + %% Let testserver and everyone else finish their work + timer:sleep(500), + %% Empty load + EmptyLoad = get_load(), + {false, _} = {lists:any(fun(Load) -> Load > 50 end, EmptyLoad),EmptyLoad}, + MeMySelfAndI = self(), + StartHog = fun() -> + Pid = spawn(?MODULE, hog, [self()]), + receive hog_started -> MeMySelfAndI ! go end, + Pid + end, + P1 = StartHog(), + %% Max on one, the other schedulers empty (hopefully) + %% Be generous the process can jump between schedulers + %% which is ok and we don't want the test to fail for wrong reasons + _L1 = [S1Load|EmptyScheds1] = get_load(), + {true,_} = {S1Load > 50,S1Load}, + {false,_} = {lists:any(fun(Load) -> Load > 50 end, EmptyScheds1),EmptyScheds1}, + {true,_} = {lists:sum(EmptyScheds1) < 60,EmptyScheds1}, + + %% 50% load + HalfHogs = [StartHog() || _ <- lists:seq(1, (Schedulers-1) div 2)], + HalfLoad = lists:sum(get_load()) div Schedulers, + if Schedulers < 2, HalfLoad > 80 -> ok; %% Ok only one scheduler online and one hog + %% We want roughly 50% load + HalfLoad > 40, HalfLoad < 60 -> ok; + true -> exit({halfload, HalfLoad}) + end, + + %% 100% load + LastHogs = [StartHog() || _ <- lists:seq(1, Schedulers div 2)], + FullScheds = get_load(), + {false,_} = {lists:any(fun(Load) -> Load < 80 end, FullScheds),FullScheds}, + FullLoad = lists:sum(FullScheds) div Schedulers, + if FullLoad > 90 -> ok; + true -> exit({fullload, FullLoad}) + end, + + [exit(Pid, kill) || Pid <- [P1|HalfHogs++LastHogs]], + AfterLoad = get_load(), + {false,_} = {lists:any(fun(Load) -> Load > 5 end, AfterLoad),AfterLoad}, + true = erlang:system_flag(scheduler_wall_time, false) + after + erlang:system_flag(scheduler_wall_time, false) end. -hog_iter(N) when N > 0 -> - ?line hog_iter(N-1); -hog_iter(0) -> - ?line hog_iter(10000). +get_load() -> + Start = erlang:statistics(scheduler_wall_time), + timer:sleep(500), + End = erlang:statistics(scheduler_wall_time), + lists:reverse(lists:sort(load_percentage(lists:sort(Start),lists:sort(End)))). + +load_percentage([{Id, WN, TN}|Ss], [{Id, WP, TP}|Ps]) -> + [100*(WN-WP) div (TN-TP)|load_percentage(Ss, Ps)]; +load_percentage([], []) -> []. garbage_collection(doc) -> diff --git a/erts/emulator/test/trace_port_SUITE.erl b/erts/emulator/test/trace_port_SUITE.erl index 0026da4979..b0ce6f81db 100644 --- a/erts/emulator/test/trace_port_SUITE.erl +++ b/erts/emulator/test/trace_port_SUITE.erl @@ -77,7 +77,8 @@ end_per_testcase(_Func, Config) -> call_trace(doc) -> "Test sending call trace messages to a port."; call_trace(Config) when is_list(Config) -> - case test_server:is_native(?MODULE) of + case test_server:is_native(?MODULE) orelse + test_server:is_native(lists) of true -> {skip,"Native code"}; false -> @@ -128,7 +129,8 @@ bs_sum_c(<<>>, Acc) -> Acc. return_trace(doc) -> "Test the new return trace."; return_trace(Config) when is_list(Config) -> - case test_server:is_native(?MODULE) of + case test_server:is_native(?MODULE) orelse + test_server:is_native(lists) of true -> {skip,"Native code"}; false -> diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops index 58c36c3bdc..ea57000c82 100755 --- a/erts/emulator/utils/beam_makeops +++ b/erts/emulator/utils/beam_makeops @@ -28,6 +28,7 @@ my $verbose = 0; my $hot = 1; my $num_file_opcodes = 0; my $wordsize = 32; +my %defs; # Defines (from command line). # This is shift counts and mask for the packer. my $WHOLE_WORD = ''; @@ -96,6 +97,12 @@ my %unnumbered; my %is_transformed; # +# Pre-processor. +# +my @if_val; +my @if_line; + +# # Code transformations. # my $te_max_vars = 0; # Max number of variables ever needed. @@ -223,6 +230,7 @@ while (@ARGV && $ARGV[0] =~ /^-(.*)/) { ($outdir = shift), next if /^outdir/; ($wordsize = shift), next if /^wordsize/; ($verbose = 1), next if /^v/; + ($defs{$1} = $2), next if /^D(\w+)=(\w+)/; die "$0: Bad option: -$_\n"; } @@ -239,7 +247,43 @@ while (<>) { } next if /^\s*$/; next if /^\#/; - + + # + # Handle %if. + # + if (/^\%if (\w+)/) { + my $name = $1; + my $val = $defs{$name}; + defined $val or error("'$name' is undefined"); + push @if_val, $val; + push @if_line, $.; + next; + } elsif (/^\%unless (\w+)/) { + my $name = $1; + my $val = $defs{$name}; + defined $val or error("'$name' is undefined"); + push @if_val, !$val; + push @if_line, $.; + next; + } elsif (/^\%else$/) { + unless (@if_line) { + error("%else without a preceding %if/%unless"); + } + $if_line[$#if_line] = $.; + $if_val[$#if_val] = !$if_val[$#if_val]; + next; + } elsif (/^\%endif$/) { + unless (@if_line) { + error("%endif without a preceding %if/%unless/%else"); + } + pop @if_val; + pop @if_line; + next; + } + if (@if_val and not $if_val[$#if_val]) { + next; + } + # # Handle assignments. # @@ -349,7 +393,13 @@ while (<>) { $unnumbered{$name,$arity} = 1; } } continue { - close(ARGV) if eof(ARGV); + if (eof(ARGV)) { + close(ARGV); + if (@if_line) { + error("Unterminated %if/%unless/%else at " . + "line $if_line[$#if_line]\n"); + } + } } $num_file_opcodes = @gen_opname; diff --git a/erts/emulator/zlib/Makefile b/erts/emulator/zlib/Makefile deleted file mode 100644 index def8e1aa47..0000000000 --- a/erts/emulator/zlib/Makefile +++ /dev/null @@ -1,23 +0,0 @@ -# -# %CopyrightBegin% -# -# Copyright Ericsson AB 1997-2009. All Rights Reserved. -# -# The contents of this file are subject to the Erlang Public License, -# Version 1.1, (the "License"); you may not use this file except in -# compliance with the License. You should have received a copy of the -# Erlang Public License along with this software. If not, it can be -# retrieved online at http://www.erlang.org/. -# -# Software distributed under the License is distributed on an "AS IS" -# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -# the License for the specific language governing rights and limitations -# under the License. -# -# %CopyrightEnd% -# -# -# Invoke with GNU make or clearmake -C gnu. -# - -include $(ERL_TOP)/make/run_make.mk diff --git a/erts/emulator/zlib/Makefile.in b/erts/emulator/zlib/Makefile.in deleted file mode 100644 index b44a87551d..0000000000 --- a/erts/emulator/zlib/Makefile.in +++ /dev/null @@ -1,102 +0,0 @@ -# Makefile for zlib -# Copyright (C) 1995-1996 Jean-loup Gailly. -# For conditions of distribution and use, see copyright notice in zlib.h - -# %ExternalCopyright% - -# To compile and test, type: -# ./configure; make test -# The call of configure is optional if you don't have special requirements - -# To install /usr/local/lib/libz.* and /usr/local/include/zlib.h, type: -# make install -# To install in $HOME instead of /usr/local, use: -# make install prefix=$HOME - -ARFLAGS = rc -CFLAGS = $(subst -O2, -O3, @CFLAGS@ @DEFS@ @EMU_THR_DEFS@) -#CFLAGS=-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7 -#CFLAGS=-g -DDEBUG -#CFLAGS=-O3 -Wall -Wwrite-strings -Wpointer-arith -Wconversion \ -# -Wstrict-prototypes -Wmissing-prototypes - -VER=1.0.4 - -O = adler32.o compress.o crc32.o uncompr.o deflate.o trees.o \ - zutil.o inflate.o inftrees.o inffast.o -OBJS = $(O:%=$(OBJDIR)/%) - - -#### Begin OTP targets - -include $(ERL_TOP)/make/target.mk - -ifeq ($(TYPE),gcov) -CFLAGS = -O0 -fprofile-arcs -ftest-coverage @DEBUG_CFLAGS@ @DEFS@ @EMU_THR_DEFS@ -else # gcov -ifeq ($(TYPE),debug) -CFLAGS = @DEBUG_CFLAGS@ @DEFS@ @EMU_THR_DEFS@ -endif # debug -endif # gcov - -# On windows we *need* a separate zlib during debug build -OBJDIR= $(ERL_TOP)/erts/emulator/zlib/obj/$(TARGET)/$(TYPE) - -include $(ERL_TOP)/make/$(TARGET)/otp.mk - -ifeq ($(TARGET), win32) -LIBRARY=$(OBJDIR)/z.lib -else -LIBRARY=$(OBJDIR)/libz.a -endif - -all: $(LIBRARY) - -# ---------------------------------------------------- -# Release Target -# ---------------------------------------------------- -include $(ERL_TOP)/make/otp_release_targets.mk - -release_spec: opt - -tests release_tests: - -docs release_docs release_docs_spec: - -clean: - rm -f $(OBJS) $(OBJDIR)/libz.a - -#### end OTP targets - -ifeq ($(TARGET), win32) -$(LIBRARY): $(OBJS) - $(AR) -out:$@ $(OBJS) -else -$(LIBRARY): $(OBJS) - $(AR) $(ARFLAGS) $@ $(OBJS) - -@ ($(RANLIB) $@ || true) 2>/dev/null -endif - -$(OBJDIR)/%.o: %.c - $(CC) -c $(CFLAGS) -o $@ $< - -# DO NOT DELETE THIS LINE -- make depend depends on it. - -adler32.o: zlib.h zconf.h -compress.o: zlib.h zconf.h -crc32.o: zlib.h zconf.h -deflate.o: deflate.h zutil.h zlib.h zconf.h -example.o: zlib.h zconf.h -gzio.o: zutil.h zlib.h zconf.h -infblock.o: infblock.h inftrees.h infcodes.h infutil.h zutil.h zlib.h zconf.h -infcodes.o: zutil.h zlib.h zconf.h -infcodes.o: inftrees.h infblock.h infcodes.h infutil.h inffast.h -inffast.o: zutil.h zlib.h zconf.h inftrees.h -inffast.o: infblock.h infcodes.h infutil.h inffast.h -inflate.o: zutil.h zlib.h zconf.h infblock.h -inftrees.o: zutil.h zlib.h zconf.h inftrees.h -infutil.o: zutil.h zlib.h zconf.h infblock.h inftrees.h infcodes.h infutil.h -minigzip.o: zlib.h zconf.h -trees.o: deflate.h zutil.h zlib.h zconf.h -uncompr.o: zlib.h zconf.h -zutil.o: zutil.h zlib.h zconf.h diff --git a/erts/emulator/zlib/zlib.mk b/erts/emulator/zlib/zlib.mk new file mode 100644 index 0000000000..fa1f159fae --- /dev/null +++ b/erts/emulator/zlib/zlib.mk @@ -0,0 +1,74 @@ +#-*-makefile-*- ; force emacs to enter makefile-mode +# ---------------------------------------------------- +# Make include file for zlib +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 2011-2012. All Rights Reserved. +# +# The contents of this file are subject to the Erlang Public License, +# Version 1.1, (the "License"); you may not use this file except in +# compliance with the License. You should have received a copy of the +# Erlang Public License along with this software. If not, it can be +# retrieved online at http://www.erlang.org/. +# +# Software distributed under the License is distributed on an "AS IS" +# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +# the License for the specific language governing rights and limitations +# under the License. +# +# %CopyrightEnd% +# +# ---------------------------------------------------- +# Copyright for zlib itself see copyright notice in zlib.h + +ZLIB_FILES = \ + adler32 \ + compress \ + crc32 \ + uncompr \ + deflate \ + trees \ + zutil \ + inflate \ + inftrees \ + inffast + +# On windows we *need* a separate zlib during debug build +ZLIB_OBJDIR = $(ERL_TOP)/erts/emulator/zlib/obj/$(TARGET)/$(TYPE) + +ZLIB_OBJS = $(ZLIB_FILES:%=$(ZLIB_OBJDIR)/%.o) +ZLIB_SRC = $(ZLIB_FILES:%=zlib/%.c) + +ifeq ($(TARGET), win32) +ZLIB_LIBRARY = $(ZLIB_OBJDIR)/z.lib +else +ZLIB_LIBRARY = $(ZLIB_OBJDIR)/libz.a +endif + + +ifeq ($(TYPE),gcov) +ZLIB_CFLAGS = -O0 -fprofile-arcs -ftest-coverage $(DEBUG_CFLAGS) $(DEFS) $(THR_DEFS) +else # gcov +ifeq ($(TYPE),debug) +ZLIB_CFLAGS = $(DEBUG_CFLAGS) $(DEFS) $(THR_DEFS) +else # debug +ZLIB_CFLAGS = $(subst -O2, -O3, $(CONFIGURE_CFLAGS) $(DEFS) $(THR_DEFS)) +#ZLIB_CFLAGS=-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7 +#ZLIB_CFLAGS=-g -DDEBUG +#ZLIB_CFLAGS=-O3 -Wall -Wwrite-strings -Wpointer-arith -Wconversion \ +# -Wstrict-prototypes -Wmissing-prototypes +endif # debug +endif # gcov + +ifeq ($(TARGET), win32) +$(ZLIB_LIBRARY): $(ZLIB_OBJS) + $(AR) -out:$@ $(ZLIB_OBJS) +else +$(ZLIB_LIBRARY): $(ZLIB_OBJS) + $(AR) $(ARFLAGS) $@ $(ZLIB_OBJS) + -@ ($(RANLIB) $@ || true) 2>/dev/null +endif + +$(ZLIB_OBJDIR)/%.o: zlib/%.c + $(CC) -c $(ZLIB_CFLAGS) -o $@ $< |