diff options
Diffstat (limited to 'erts/emulator')
46 files changed, 1646 insertions, 692 deletions
diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index a5d8217545..279844adb2 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2011. All Rights Reserved. +# Copyright Ericsson AB 1996-2012. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -28,21 +28,26 @@ Z_LIB=@Z_LIB@ NO_INLINE_FUNCTIONS=false OPCODE_TABLES = $(ERL_TOP)/lib/compiler/src/genop.tab beam/ops.tab +DEBUG_CFLAGS = @DEBUG_CFLAGS@ +CONFIGURE_CFLAGS = @CFLAGS@ + # # Run this make file with TYPE set to the type of emulator you want. # Different versions of the emulator for different uses. The default # is "debug". For a normal version use "opt". # +DEFS=@DEFS@ THR_DEFS=@EMU_THR_DEFS@ M4FLAGS= CREATE_DIRS= LDFLAGS=@LDFLAGS@ +ARFLAGS=rc ifeq ($(TYPE),debug) PURIFY = TYPEMARKER = .debug -TYPE_FLAGS = @DEBUG_CFLAGS@ -DDEBUG +TYPE_FLAGS = $(DEBUG_CFLAGS) -DDEBUG ENABLE_ALLOC_TYPE_VARS += debug ifeq ($(TARGET),win32) TYPE_FLAGS += -DNO_JUMP_TABLE @@ -53,7 +58,7 @@ else ifeq ($(TYPE),purify) PURIFY = purify $(PURIFY_BUILD_OPTIONS) TYPEMARKER = .purify -TYPE_FLAGS = @DEBUG_CFLAGS@ -DPURIFY -DNO_JUMP_TABLE -DERTS_MSEG_FAKE_SEGMENTS +TYPE_FLAGS = $(DEBUG_CFLAGS) -DPURIFY -DNO_JUMP_TABLE -DERTS_MSEG_FAKE_SEGMENTS ENABLE_ALLOC_TYPE_VARS += purify else @@ -67,14 +72,14 @@ else ifeq ($(TYPE),purecov) PURIFY = purecov --follow-child-processes=yes $(PURECOV_BUILD_OPTIONS) TYPEMARKER = .purecov -TYPE_FLAGS = @DEBUG_CFLAGS@ -DPURECOV -DNO_JUMP_TABLE +TYPE_FLAGS = $(DEBUG_CFLAGS) -DPURECOV -DNO_JUMP_TABLE ENABLE_ALLOC_TYPE_VARS += purecov else ifeq ($(TYPE),gcov) PURIFY = TYPEMARKER = .gcov -TYPE_FLAGS = @DEBUG_CFLAGS@ -DERTS_GCOV -DNO_JUMP_TABLE -fprofile-arcs -ftest-coverage -O0 -DERTS_CAN_INLINE=0 -DERTS_INLINE= +TYPE_FLAGS = $(DEBUG_CFLAGS) -DERTS_GCOV -DNO_JUMP_TABLE -fprofile-arcs -ftest-coverage -O0 -DERTS_CAN_INLINE=0 -DERTS_INLINE= ifneq ($(findstring solaris,$(TARGET)),solaris) LIBS += -lgcov endif @@ -84,7 +89,7 @@ else ifeq ($(TYPE),valgrind) PURIFY = TYPEMARKER = .valgrind -TYPE_FLAGS = @DEBUG_CFLAGS@ -DVALGRIND -DNO_JUMP_TABLE -DERTS_MSEG_FAKE_SEGMENTS +TYPE_FLAGS = $(DEBUG_CFLAGS) -DVALGRIND -DNO_JUMP_TABLE -DERTS_MSEG_FAKE_SEGMENTS ENABLE_ALLOC_TYPE_VARS += valgrind else @@ -147,6 +152,15 @@ endif TF_MARKER=$(TYPEMARKER)$(FLAVOR_MARKER) +ifeq ($(FLAVOR)-@ERTS_BUILD_SMP_EMU@,smp-no) +VOID_EMULATOR = '*** SMP emulator disabled by configure' +else +ifeq ($(TYPE)-@HAVE_VALGRIND@,valgrind-no) +VOID_EMULATOR = '*** valgrind emulator disabled by configure' +else +VOID_EMULATOR = +endif +endif OPSYS=@OPSYS@ sol2CFLAGS= @@ -187,7 +201,7 @@ else EMU_CC = @EMU_CC@ endif WFLAGS = @WFLAGS@ -CFLAGS = @STATIC_CFLAGS@ $(TYPE_FLAGS) $(FLAVOR_FLAGS) @DEFS@ $(WFLAGS) $(THR_DEFS) $(ARCHCFLAGS) +CFLAGS = @STATIC_CFLAGS@ $(TYPE_FLAGS) $(FLAVOR_FLAGS) $(DEFS) $(WFLAGS) $(THR_DEFS) $(ARCHCFLAGS) HCC = @HCC@ LD = @LD@ DEXPORT = @DEXPORT@ @@ -263,30 +277,29 @@ CS_PURIFY = CS_TYPE_FLAGS = $(subst QUANTIFY,FAKE_QUANTIFY, \ $(subst PURIFY,FAKE_PURIFY, $(TYPE_FLAGS))) endif -CS_CFLAGS_ = $(CS_TYPE_FLAGS) @DEFS@ $(WFLAGS) +CS_CFLAGS_ = $(CS_TYPE_FLAGS) $(DEFS) $(WFLAGS) ifeq ($(GCC),yes) CS_CFLAGS = $(subst -O2, $(GEN_OPT_FLGS) $(UNROLL_FLG), $(CS_CFLAGS_)) else CS_CFLAGS = $(CS_CFLAGS_) endif CS_LDFLAGS = $(LDFLAGS) -CS_LIBS = -L../lib/internal/$(TARGET) -lerts_internal @ERTS_INTERNAL_X_LIBS@ +CS_LIBS = -L../lib/internal/$(TARGET) -lerts_internal$(TYPEMARKER) @ERTS_INTERNAL_X_LIBS@ LIBS += @TERMCAP_LIB@ -L../lib/internal/$(TARGET) @ERTS_INTERNAL_X_LIBS@ ifdef Z_LIB # Use shared zlib library LIBS += $(Z_LIB) +DEPLIBS = else +DEPLIBS=$(ZLIB_LIBRARY) ifeq ($(TARGET),win32) -LIBS += -L$(ERL_TOP)/erts/emulator/zlib/obj/$(TARGET)/$(TYPE) -lz -DEPLIBS = $(ERL_TOP)/erts/emulator/zlib/obj/$(TARGET)/$(TYPE)/z.lib +LIBS += -L$(ZLIB_OBJDIR) -lz else # Build on darwin fails if -lz is used -LIBS += $(ERL_TOP)/erts/emulator/zlib/obj/$(TARGET)/$(TYPE)/libz.a -DEPLIBS = $(ERL_TOP)/erts/emulator/zlib/obj/$(TARGET)/$(TYPE)/libz.a +LIBS += $(ZLIB_LIBRARY) endif - endif ifeq ($(TARGET),win32) @@ -313,12 +326,9 @@ LIBSCTP = @LIBSCTP@ ORG_THR_LIBS=@EMU_THR_LIBS@ THR_LIB_NAME=@EMU_THR_LIB_NAME@ -ifneq ($(strip $(THR_LIB_NAME)),) -DEPLIBS += $(ERL_TOP)/erts/lib/internal/$(TARGET)/$(LIB_PREFIX)erts_internal_r$(TYPEMARKER)$(LIB_SUFFIX) \ - $(ERL_TOP)/erts/lib/internal/$(TARGET)/$(LIB_PREFIX)ethread$(TYPEMARKER)$(LIB_SUFFIX) -else -DEPLIBS += $(ERL_TOP)/erts/lib/internal/$(TARGET)/$(LIB_PREFIX)erts_internal$(TYPEMARKER)$(LIB_SUFFIX) -endif +ERTS_LIB_DIR = $(ERL_TOP)/erts/lib_src +ERTS_LIB = $(ERTS_LIB_DIR)/obj/$(TARGET)/$(TYPE)/MADE +DEPLIBS += $(ERTS_LIB) THR_LIBS=$(subst -l$(THR_LIB_NAME),-l$(THR_LIB_NAME)$(TYPEMARKER), \ $(subst -lerts_internal_r,-lerts_internal_r$(TYPEMARKER),$(ORG_THR_LIBS))) @@ -354,8 +364,7 @@ OBJDIR = obj/$(TTF_DIR) CREATE_DIRS += $(OBJDIR) \ pcre/obj/$(TARGET)/$(TYPE) \ - zlib/obj/$(TARGET)/$(TYPE) - + $(ZLIB_OBJDIR) BINDIR = $(ERL_TOP)/bin/$(TARGET) @@ -378,15 +387,12 @@ else UNIX_ONLY_BUILDS = endif -ifeq ($(TYPE)-@HAVE_VALGRIND@,valgrind-no) +.PHONY: all +ifdef VOID_EMULATOR all: - @echo '*** valgrind not found by configure' + @echo $(VOID_EMULATOR)' - omitted target all' else -ifeq ($(FLAVOR)-@ERTS_BUILD_SMP_EMU@,smp-no) -all: - @echo '*** Omitted build of emulator with smp support' -else -all: generate erts_lib zlib $(BINDIR)/$(EMULATOR_EXECUTABLE) $(UNIX_ONLY_BUILDS) +all: $(BINDIR)/$(EMULATOR_EXECUTABLE) $(UNIX_ONLY_BUILDS) ifeq ($(OMIT_OMIT_FP),yes) @echo '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *' @echo '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *' @@ -397,37 +403,25 @@ ifeq ($(OMIT_OMIT_FP),yes) @echo '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *' endif endif -endif - -ifdef Z_LIB -zlib: - @echo 'Skip zlib directory, use shared library' -else -zlib: - @set -e ; cd zlib && $(MAKE) TYPE=$(TYPE) $(TYPE) -endif - +include zlib/zlib.mk include pcre/pcre.mk -erts_lib: - cd $(ERL_TOP)/erts/lib_src && $(MAKE) $(TYPE) +$(ERTS_LIB): + cd $(ERTS_LIB_DIR) && $(MAKE) $(TYPE) +.PHONY: clean clean: -ifeq ($(TARGET),win32) - $(RM) -f $(TARGET)/beams.rc -endif - $(RM) -f $(TARGET)/*.c $(TARGET)/*.h $(TARGET)/depend.mk - $(RM) -f $(TARGET)/*/*/*.c $(TARGET)/*/*/*.h $(TARGET)/*/*/*.S - $(RM) -f $(ERL_TOP)/erts/emulator/obj/$(TARGET)/*/*/*.o - $(RM) -f $(BINDIR)/beam $(BINDIR)/beam.* - $(RM) -rf $(BINDIR)/child_setup $(BINDIR)/child_setup.* - $(RM) -f $(BINDIR)/hipe_mkliterals $(BINDIR)/hipe_mkliterals.* - @set -e ; cd zlib && $(MAKE) clean - rm -f $(OBJS) $(OBJDIR)/libepcre.a - -.PHONY: all zlib clean - + $(RM) -f $(GENERATE) + $(RM) -rf $(TARGET)/*.c $(TARGET)/*.h $(TARGET)/*-GENERATED + $(RM) -rf $(TARGET)/*/* + $(RM) -rf obj/$(TARGET) + $(RM) -rf pcre/obj/$(TARGET) $(PCRE_GENINC) + $(RM) -rf zlib/obj/$(TARGET) + $(RM) -rf bin/$(TARGET) + cd $(ERTS_LIB_DIR) && $(MAKE) clean + +.PHONY: docs docs: # ---------------------------------------------------------------------- @@ -440,13 +434,11 @@ ifeq ($(TARGET),win32) RELEASE_INCLUDES += sys/$(ERLANG_OSTYPE)/erl_win_dyn_driver.h endif -ifeq ($(TYPE)-@HAVE_VALGRIND@,valgrind-no) -release_spec: - @echo '*** valgrind not found by configure' -else -ifeq ($(FLAVOR)-@ERTS_BUILD_SMP_EMU@,smp-no) + +.PHONY: release_spec +ifdef VOID_EMULATOR release_spec: - @echo '*** No emulator with smp support to install' + @echo $(VOID_EMULATOR)' - omitted target release_spec (install)' else release_spec: all $(INSTALL_DIR) $(RELSYSDIR) @@ -463,8 +455,8 @@ ifeq ($(ERLANG_OSTYPE), unix) $(INSTALL_PROGRAM) $(BINDIR)/$(CS_EXECUTABLE) $(RELSYSDIR)/bin endif endif -endif +.PHONY: release_docs_spec release_docs_spec: # ---------------------------------------------------------------------- @@ -473,48 +465,41 @@ release_docs_spec: _create_dirs := $(shell mkdir -p $(CREATE_DIRS)) -.PHONY : generate - -GENERATE= $(TTF_DIR)/beam_opcodes.h \ - $(TARGET)/erl_bif_table.c \ - $(TARGET)/erl_version.h \ - $(TTF_DIR)/driver_tab.c \ - $(TTF_DIR)/erl_alloc_types.h - -ifeq ($(TARGET),win32) -GENERATE += $(TARGET)/beams.rc -else -GENERATE += $(TARGET)/preload.c -endif +GENERATE = +HIPE_ASM = ifeq ($(findstring vxworks,$(TARGET)),vxworks) else ifdef HIPE_ENABLED -GENERATE += $(TTF_DIR)/hipe_x86_asm.h \ +HIPE_ASM += $(TTF_DIR)/hipe_x86_asm.h \ $(TTF_DIR)/hipe_amd64_asm.h \ $(TTF_DIR)/hipe_sparc_asm.h \ $(TTF_DIR)/hipe_ppc_asm.h \ - $(TTF_DIR)/hipe_arm_asm.h \ + $(TTF_DIR)/hipe_arm_asm.h + +GENERATE += $(HIPE_ASM) \ $(TTF_DIR)/hipe_literals.h \ $(BINDIR)/hipe_mkliterals$(TF_MARKER) endif endif -ifeq ($(FLAVOR)-@ERTS_BUILD_SMP_EMU@,smp-no) -GENERATE= -endif - -generate: $(GENERATE) - ifdef HIPE_ENABLED OPCODE_TABLES += hipe/hipe_ops.tab endif -$(TTF_DIR)/beam_opcodes.h $(TTF_DIR)/beam_opcodes.c: $(OPCODE_TABLES) utils/beam_makeops +$(TTF_DIR)/beam_cold.h \ +$(TTF_DIR)/beam_hot.h \ +$(TTF_DIR)/beam_opcodes.c \ +$(TTF_DIR)/beam_opcodes.h \ +$(TTF_DIR)/beam_pred_funcs.h \ +$(TTF_DIR)/beam_tr_funcs.h \ + : $(TTF_DIR)/OPCODES-GENERATED +$(TTF_DIR)/OPCODES-GENERATED: $(OPCODE_TABLES) utils/beam_makeops LANG=C $(PERL) utils/beam_makeops \ -wordsize @EXTERNAL_WORD_SIZE@ \ -outdir $(TTF_DIR) \ - -emulator $(OPCODE_TABLES) + -emulator $(OPCODE_TABLES) && echo $? >$(TTF_DIR)/OPCODES-GENERATED +GENERATE += $(TTF_DIR)/OPCODES-GENERATED # bif and atom table ATOMS= beam/atom.names @@ -534,32 +519,44 @@ BIFS += hipe/hipe_perfctr.tab endif endif -TABLES= $(TARGET)/erl_bif_table.c $(TARGET)/erl_bif_table.h \ - $(TARGET)/erl_bif_wrap.c $(TARGET)/erl_bif_list.h \ - $(TARGET)/erl_atom_table.c $(TARGET)/erl_atom_table.h \ - $(TARGET)/erl_pbifs.c - -$(TABLES): $(ATOMS) $(BIFS) utils/make_tables +$(TARGET)/erl_bif_table.c \ +$(TARGET)/erl_bif_table.h \ +$(TARGET)/erl_bif_wrap.c \ +$(TARGET)/erl_bif_list.h \ +$(TARGET)/erl_atom_table.c \ +$(TARGET)/erl_atom_table.h \ +$(TARGET)/erl_pbifs.c \ + : $(TARGET)/TABLES-GENERATED +$(TARGET)/TABLES-GENERATED: $(ATOMS) $(BIFS) utils/make_tables LANG=C $(PERL) utils/make_tables -src $(TARGET) -include $(TARGET)\ - $(ATOMS) $(BIFS) + $(ATOMS) $(BIFS) && echo $? >$(TARGET)/TABLES-GENERATED +GENERATE += $(TARGET)/TABLES-GENERATED $(TTF_DIR)/erl_alloc_types.h: beam/erl_alloc.types utils/make_alloc_types LANG=C $(PERL) utils/make_alloc_types -src $< -dst $@ $(ENABLE_ALLOC_TYPE_VARS) +GENERATE += $(TTF_DIR)/erl_alloc_types.h # version include file $(TARGET)/erl_version.h: ../vsn.mk LANG=C $(PERL) utils/make_version -o $@ $(SYSTEM_VSN) $(VSN)$(SERIALNO) $(TARGET) +GENERATE += $(TARGET)/erl_version.h # driver table $(TTF_DIR)/driver_tab.c: Makefile.in LANG=C $(PERL) utils/make_driver_tab -o $@ $(DRV_OBJS) +GENERATE += $(TTF_DIR)/driver_tab.c + + # Preloaded code. # # This list must be consistent with PRE_LOADED_MODULES in # lib/kernel/src/Makefile. ifeq ($(TARGET),win32) -$(TARGET)/beams.rc: $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ +# On windows the preloaded objects are in a resource object. +PRELOAD_OBJ = $(OBJDIR)/beams.$(RES_EXT) +PRELOAD_SRC = $(TARGET)/beams.rc +$(PRELOAD_SRC): $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ $(ERL_TOP)/erts/preloaded/ebin/init.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_inet.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_file.beam \ @@ -569,7 +566,9 @@ $(TARGET)/beams.rc: $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ $(ERL_TOP)/erts/preloaded/ebin/erlang.beam LANG=C $(PERL) utils/make_preload $(MAKE_PRELOAD_EXTRA) -rc $^ > $@ else -$(TARGET)/preload.c: $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ +PRELOAD_OBJ = $(OBJDIR)/preload.o +PRELOAD_SRC = $(TARGET)/preload.c +$(PRELOAD_SRC): $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ $(ERL_TOP)/erts/preloaded/ebin/init.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_inet.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_file.beam \ @@ -580,6 +579,17 @@ $(TARGET)/preload.c: $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ LANG=C $(PERL) utils/make_preload -old $^ > $@ endif +.PHONY : generate +ifdef VOID_EMULATOR +generate: + @echo $(VOID_EMULATOR)' - omitted target generate' +else +generate: $(TTF_DIR)/GENERATED $(PRELOAD_SRC) + +$(TTF_DIR)/GENERATED: $(GENERATE) + echo $? >$(TTF_DIR)/GENERATED +endif + # ---------------------------------------------------------------------- # Pattern rules # @@ -656,7 +666,7 @@ endif # CS_SRC = sys/$(ERLANG_OSTYPE)/erl_child_setup.c -$(BINDIR)/$(CS_EXECUTABLE): $(CS_SRC) +$(BINDIR)/$(CS_EXECUTABLE): $(TTF_DIR)/GENERATED $(PRELOAD_SRC) $(CS_SRC) $(ERTS_LIB) $(CS_PURIFY) $(CC) $(CS_LDFLAGS) -o $(BINDIR)/$(CS_EXECUTABLE) \ $(CS_CFLAGS) $(COMMON_INCLUDES) $(CS_SRC) $(CS_LIBS) @@ -685,16 +695,7 @@ $(ERL_TOP)/lib/%.beam: # Object files # -# On windows the preloaded objects are in a resource object. - -ifeq ($(TARGET),win32) -PRELOAD = $(OBJDIR)/beams.$(RES_EXT) -else -PRELOAD = $(OBJDIR)/preload.o -endif - - -INIT_OBJS = $(OBJDIR)/erl_main.o $(PRELOAD) +INIT_OBJS = $(OBJDIR)/erl_main.o $(PRELOAD_OBJ) EMU_OBJS = \ $(OBJDIR)/beam_emu.o $(OBJDIR)/beam_opcodes.o \ @@ -834,6 +835,9 @@ BASE_OBJS = $(RUN_OBJS) $(EMU_OBJS) $(OS_OBJS) $(EXTRA_BASE_OBJS) OBJS = $(BASE_OBJS) $(DRV_OBJS) +$(INIT_OBJS): $(TTF_DIR)/GENERATED +$(OBJS): $(TTF_DIR)/GENERATED + ######################################## # HiPE section @@ -857,30 +861,49 @@ $(OBJDIR)/%.o: hipe/%.c $(BINDIR)/hipe_mkliterals$(TF_MARKER): $(OBJDIR)/hipe_mkliterals.o $(CC) $(CFLAGS) $(INCLUDES) -o $@ $< -$(OBJDIR)/hipe_mkliterals.o: $(TTF_DIR)/hipe_x86_asm.h $(TTF_DIR)/hipe_ppc_asm.h $(TTF_DIR)/beam_opcodes.h +$(OBJDIR)/hipe_mkliterals.o: $(HIPE_ASM) $(TTF_DIR)/erl_alloc_types.h \ + $(TTF_DIR)/OPCODES-GENERATED $(TARGET)/TABLES-GENERATED $(TTF_DIR)/hipe_literals.h: $(BINDIR)/hipe_mkliterals$(TF_MARKER) $(BINDIR)/hipe_mkliterals$(TF_MARKER) -c > $@ -$(OBJDIR)/hipe_x86_glue.o: hipe/hipe_x86_glue.S $(TTF_DIR)/hipe_x86_asm.h $(TTF_DIR)/hipe_literals.h hipe/hipe_mode_switch.h -$(TTF_DIR)/hipe_x86_bifs.S: hipe/hipe_x86_bifs.m4 hipe/hipe_x86_asm.m4 hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h -$(OBJDIR)/hipe_x86_bifs.o: $(TTF_DIR)/hipe_x86_bifs.S $(TTF_DIR)/hipe_literals.h - -$(OBJDIR)/hipe_amd64_glue.o: hipe/hipe_amd64_glue.S $(TTF_DIR)/hipe_amd64_asm.h $(TTF_DIR)/hipe_literals.h hipe/hipe_mode_switch.h -$(TTF_DIR)/hipe_amd64_bifs.S: hipe/hipe_amd64_bifs.m4 hipe/hipe_amd64_asm.m4 hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h -$(OBJDIR)/hipe_amd64_bifs.o: $(TTF_DIR)/hipe_amd64_bifs.S $(TTF_DIR)/hipe_literals.h - -$(OBJDIR)/hipe_sparc_glue.o: hipe/hipe_sparc_glue.S $(TTF_DIR)/hipe_sparc_asm.h hipe/hipe_mode_switch.h $(TTF_DIR)/hipe_literals.h -$(TTF_DIR)/hipe_sparc_bifs.S: hipe/hipe_sparc_bifs.m4 hipe/hipe_sparc_asm.m4 hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h -$(OBJDIR)/hipe_sparc_bifs.o: $(TTF_DIR)/hipe_sparc_bifs.S $(TTF_DIR)/hipe_literals.h - -$(OBJDIR)/hipe_ppc_glue.o: hipe/hipe_ppc_glue.S $(TTF_DIR)/hipe_ppc_asm.h hipe/hipe_mode_switch.h $(TTF_DIR)/hipe_literals.h -$(TTF_DIR)/hipe_ppc_bifs.S: hipe/hipe_ppc_bifs.m4 hipe/hipe_ppc_asm.m4 hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h -$(OBJDIR)/hipe_ppc_bifs.o: $(TTF_DIR)/hipe_ppc_bifs.S $(TTF_DIR)/hipe_literals.h - -$(OBJDIR)/hipe_arm_glue.o: hipe/hipe_arm_glue.S $(TTF_DIR)/hipe_arm_asm.h hipe/hipe_mode_switch.h $(TTF_DIR)/hipe_literals.h -$(TTF_DIR)/hipe_arm_bifs.S: hipe/hipe_arm_bifs.m4 hipe/hipe_arm_asm.m4 hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h -$(OBJDIR)/hipe_arm_bifs.o: $(TTF_DIR)/hipe_arm_bifs.S $(TTF_DIR)/hipe_literals.h +$(OBJDIR)/hipe_x86_glue.o: hipe/hipe_x86_glue.S \ + $(TTF_DIR)/hipe_x86_asm.h $(TTF_DIR)/hipe_literals.h \ + hipe/hipe_mode_switch.h +$(TTF_DIR)/hipe_x86_bifs.S: hipe/hipe_x86_bifs.m4 hipe/hipe_x86_asm.m4 \ + hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h +$(OBJDIR)/hipe_x86_bifs.o: $(TTF_DIR)/hipe_x86_bifs.S \ + $(TTF_DIR)/hipe_literals.h + +$(OBJDIR)/hipe_amd64_glue.o: hipe/hipe_amd64_glue.S \ + $(TTF_DIR)/hipe_amd64_asm.h $(TTF_DIR)/hipe_literals.h \ + hipe/hipe_mode_switch.h +$(TTF_DIR)/hipe_amd64_bifs.S: hipe/hipe_amd64_bifs.m4 hipe/hipe_amd64_asm.m4 \ + hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h +$(OBJDIR)/hipe_amd64_bifs.o: $(TTF_DIR)/hipe_amd64_bifs.S \ + $(TTF_DIR)/hipe_literals.h + +$(OBJDIR)/hipe_sparc_glue.o: hipe/hipe_sparc_glue.S \ + $(TTF_DIR)/hipe_sparc_asm.h hipe/hipe_mode_switch.h \ + $(TTF_DIR)/hipe_literals.h +$(TTF_DIR)/hipe_sparc_bifs.S: hipe/hipe_sparc_bifs.m4 hipe/hipe_sparc_asm.m4 \ + hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h +$(OBJDIR)/hipe_sparc_bifs.o: $(TTF_DIR)/hipe_sparc_bifs.S \ + $(TTF_DIR)/hipe_literals.h + +$(OBJDIR)/hipe_ppc_glue.o: hipe/hipe_ppc_glue.S $(TTF_DIR)/hipe_ppc_asm.h \ + hipe/hipe_mode_switch.h $(TTF_DIR)/hipe_literals.h +$(TTF_DIR)/hipe_ppc_bifs.S: hipe/hipe_ppc_bifs.m4 hipe/hipe_ppc_asm.m4 \ + hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h +$(OBJDIR)/hipe_ppc_bifs.o: $(TTF_DIR)/hipe_ppc_bifs.S \ + $(TTF_DIR)/hipe_literals.h + +$(OBJDIR)/hipe_arm_glue.o: hipe/hipe_arm_glue.S $(TTF_DIR)/hipe_arm_asm.h \ + hipe/hipe_mode_switch.h $(TTF_DIR)/hipe_literals.h +$(TTF_DIR)/hipe_arm_bifs.S: hipe/hipe_arm_bifs.m4 hipe/hipe_arm_asm.m4 \ + hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h +$(OBJDIR)/hipe_arm_bifs.o: $(TTF_DIR)/hipe_arm_bifs.S \ + $(TTF_DIR)/hipe_literals.h # end of HiPE section ######################################## @@ -927,13 +950,6 @@ $(BINDIR)/$(EMULATOR_EXECUTABLE): $(INIT_OBJS) $(OBJS) $(DEPLIBS) endif -# -# Create directories -# - -$(CREATE_DIRS): - $(MKDIR) -p $@ - # ---------------------------------------------------------------------- # Dependencies # @@ -945,6 +961,7 @@ $(TARGET)/Makefile: Makefile.in #SED_REPL_WIN_DRIVE=s|\([ ]\)\([A-Za-z]\):|\1/cygdrive/\2|g;s|^\([A-Za-z]\):|/cygdrive/\1|g SED_REPL_O=s|^\([^:]*:\)|$$(OBJDIR)/\1|g +SED_REPL_O_ZLIB=s|^\([^:]*:\)|$$(ZLIB_OBJDIR)/\1|g SED_REPL_TTF_DIR=s|$(TTF_DIR)/|$$(TTF_DIR)/|g SED_REPL_ERL_TOP=s|\([ ]\)$(ERL_TOP)/|\1$$(ERL_TOP)/|g;s|^$(ERL_TOP)/|$$(ERL_TOP)/|g SED_REPL_POLL=s|$$(OBJDIR)/erl_poll.o|$$(OBJDIR)/erl_poll.kp.o $$(OBJDIR)/erl_poll.nkp.o|g @@ -964,6 +981,7 @@ SED_SUFFIX= endif SED_DEPEND=sed '$(SED_PREFIX)$(SED_REPL_O);$(SED_REPL_TTF_DIR);$(SED_REPL_ERL_TOP)$(SED_SUFFIX)' +SED_DEPEND_ZLIB=sed '$(SED_PREFIX)$(SED_REPL_O_ZLIB)' ifdef HIPE_ENABLED HIPE_SRC=$(wildcard hipe/*.c) @@ -1008,22 +1026,36 @@ DEP_FLAGS=-MM $(MG_FLAG) $(CFLAGS) $(INCLUDES) -Idrivers/common -Idrivers/$(ERLA SYS_SRC=$(ALL_SYS_SRC) endif +.PHONY: depend +ifdef VOID_EMULATOR depend: + @echo $(VOID_EMULATOR)' - omitted target depend' +else +depend: $(TTF_DIR)/depend.mk +$(TTF_DIR)/depend.mk: $(TTF_DIR)/GENERATED $(PRELOAD_SRC) $(DEP_CC) $(DEP_FLAGS) $(BEAM_SRC) \ - | $(SED_DEPEND) > $(TARGET)/depend.mk + | $(SED_DEPEND) > $(TTF_DIR)/depend.mk $(DEP_CC) $(DEP_FLAGS) -DLIBSCTP=$(LIBSCTP) $(DRV_COMMON_SRC) \ - | $(SED_DEPEND) >> $(TARGET)/depend.mk + | $(SED_DEPEND) >> $(TTF_DIR)/depend.mk $(DEP_CC) $(DEP_FLAGS) -I../etc/$(ERLANG_OSTYPE) $(DRV_OSTYPE_SRC) \ - | $(SED_DEPEND) >> $(TARGET)/depend.mk + | $(SED_DEPEND) >> $(TTF_DIR)/depend.mk $(DEP_CC) $(DEP_FLAGS) $(SYS_SRC) \ - | $(SED_DEPEND) >> $(TARGET)/depend.mk + | $(SED_DEPEND) >> $(TTF_DIR)/depend.mk $(DEP_CC) $(DEP_FLAGS) $(TARGET_SRC) \ - | $(SED_DEPEND) >> $(TARGET)/depend.mk + | $(SED_DEPEND) >> $(TTF_DIR)/depend.mk + $(DEP_CC) $(DEP_FLAGS) $(ZLIB_SRC) \ + | $(SED_DEPEND_ZLIB) >> $(TTF_DIR)/depend.mk ifdef HIPE_ENABLED $(DEP_CC) $(DEP_FLAGS) $(HIPE_SRC) \ - | $(SED_DEPEND) >> $(TARGET)/depend.mk + | $(SED_DEPEND) >> $(TTF_DIR)/depend.mk +endif + cd $(ERTS_LIB_DIR) && $(MAKE) depend endif --include $(TARGET)/depend.mk - - +ifneq ($(MAKECMDGOALS),clean) +ifneq ($(MAKECMDGOALS),generate) +ifndef VOID_EMULATOR +-include $(TTF_DIR)/depend.mk +endif +endif +endif diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 71454b3e57..7be40976f6 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -95,6 +95,7 @@ atom atom atom atom_used atom attributes atom await_proc_exit +atom await_sched_wall_time_modifications atom awaiting_load atom awaiting_unload atom backtrace backtrace_depth @@ -239,6 +240,7 @@ atom generational atom get_seq_token atom get_tcw atom getenv +atom gather_sched_wall_time_result atom getting_linked atom getting_unlinked atom global @@ -554,6 +556,7 @@ atom waiting atom wall_clock atom warning atom warning_msg +atom scheduler_wall_time atom wordsize atom write_concurrency atom xor diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 692fa61fe8..dd13cd179a 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -1329,10 +1329,14 @@ static BpData *get_break(Process *p, BeamInstr *pc, BeamInstr break_op) { } static BpData *is_break(BeamInstr *pc, BeamInstr break_op) { - BpData **rs = (BpData **) pc[-4]; + BpData **rs; BpData *bd = NULL, *ebd = NULL; ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + if (erts_is_native_break(pc)) { + return NULL; + } + rs = (BpData **) pc[-4]; if (! rs) { return NULL; } diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 55f4798892..f8305944a4 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -43,6 +43,9 @@ static Export* set_cpu_topology_trap = NULL; static Export* await_proc_exit_trap = NULL; Export* erts_format_cpu_topology_trap = NULL; +static Export *await_sched_wall_time_mod_trap; +static erts_smp_atomic32_t sched_wall_time; + #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) /* @@ -4160,6 +4163,18 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(am_true); + } else if (BIF_ARG_1 == am_scheduler_wall_time) { + if (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false) { + erts_aint32_t new = BIF_ARG_2 == am_true ? 1 : 0; + erts_aint32_t old = erts_smp_atomic32_xchg_nob(&sched_wall_time, + new); + Eterm ref = erts_sched_wall_time_request(BIF_P, 1, new); + ASSERT(is_value(ref)); + BIF_TRAP2(await_sched_wall_time_mod_trap, + BIF_P, + ref, + old ? am_true : am_false); + } } else if (ERTS_IS_ATOM_STR("scheduling_statistics", BIF_ARG_1)) { int what; if (ERTS_IS_ATOM_STR("disable", BIF_ARG_2)) @@ -4457,6 +4472,9 @@ void erts_init_bif(void) am_format_cpu_topology, 1); await_proc_exit_trap = erts_export_put(am_erlang,am_await_proc_exit,3); + await_sched_wall_time_mod_trap + = erts_export_put(am_erlang, am_await_sched_wall_time_modifications, 2); + erts_smp_atomic32_init_nob(&sched_wall_time, 0); } #ifdef HARDDEBUG diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index 976f05c990..25ac790d81 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -1844,6 +1844,7 @@ dsize_t big_bytes(Eterm x) /* ** Load a bignum from bytes ** xsz is the number of bytes in xp +** *r is untouched if number fits in small */ Eterm bytes_to_big(byte *xp, dsize_t xsz, int xsgn, Eterm *r) { @@ -1852,7 +1853,7 @@ Eterm bytes_to_big(byte *xp, dsize_t xsz, int xsgn, Eterm *r) ErtsDigit d; int i; - while(xsz >= sizeof(ErtsDigit)) { + while(xsz > sizeof(ErtsDigit)) { d = 0; for(i = sizeof(ErtsDigit); --i >= 0;) d = (d << 8) | xp[i]; @@ -1867,11 +1868,20 @@ Eterm bytes_to_big(byte *xp, dsize_t xsz, int xsgn, Eterm *r) d = 0; for(i = xsz; --i >= 0;) d = (d << 8) | xp[i]; + if (++rsz == 1 && IS_USMALL(xsgn,d)) { + if (xsgn) d = -d; + return make_small(d); + } *rwp = d; rwp++; - rsz++; } - return big_norm(r, rsz, (short) xsgn); + if (xsgn) { + *r = make_neg_bignum_header(rsz); + } + else { + *r = make_pos_bignum_header(rsz); + } + return make_big(r); } /* diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 0d3b6a4dba..6f5020dc14 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -646,6 +646,9 @@ bin_check(void) void erl_crash_dump_v(char *file, int line, char* fmt, va_list args) { +#ifdef ERTS_SMP + ErtsThrPrgrData tpd_buf; /* in case we aren't a managed thread... */ +#endif int fd; time_t now; size_t dumpnamebufsize = MAXPATHLEN; @@ -663,7 +666,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) * We do not release system again. We expect an exit() or abort() after * dump has been written. */ - erts_thr_progress_fatal_error_block(60000); + erts_thr_progress_fatal_error_block(60000, &tpd_buf); /* Either worked or not... */ /* Allow us to pass certain places without locking... */ diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 4d02a67d54..df27186680 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2011. All Rights Reserved. + * Copyright Ericsson AB 2002-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -1577,9 +1577,11 @@ erts_alloc_register_scheduler(void *vesdp) } } +#ifdef ERTS_SMP void erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp, int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, int *more_work) { ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp; @@ -1599,10 +1601,12 @@ erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp, erts_alcu_check_delayed_dealloc(allctr, 1, need_thr_progress, + thr_prgr_p, more_work); } } } +#endif erts_aint32_t erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs) diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index 991061c48e..e475f9d8a2 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2011. All Rights Reserved. + * Copyright Ericsson AB 2002-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -21,6 +21,10 @@ #define ERL_ALLOC_H__ #include "erl_alloc_types.h" +#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY +#define ERL_THR_PROGRESS_TSD_TYPE_ONLY +#include "erl_thr_progress.h" +#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY #include "erl_alloc_util.h" #ifdef USE_THREADS #include "erl_threads.h" @@ -132,9 +136,12 @@ typedef struct { extern ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1]; void erts_alloc_register_scheduler(void *vesdp); +#ifdef ERTS_SMP void erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp, int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, int *more_work); +#endif erts_aint32_t erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs); __decl_noreturn void erts_alloc_enomem(ErtsAlcType_t,Uint) diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 962db8b831..90a6c0cbee 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -367,6 +367,7 @@ type EXPORT LONG_LIVED_LOW CODE export_entry type MONITOR_SH STANDARD_LOW PROCESSES monitor_sh type NLINK_SH STANDARD_LOW PROCESSES nlink_sh type AINFO_REQ STANDARD_LOW SYSTEM alloc_info_request +type SCHED_WTIME_REQ STANDARD_LOW SYSTEM sched_wall_time_request +else # "fullword" @@ -383,6 +384,7 @@ type EXPORT LONG_LIVED CODE export_entry type MONITOR_SH FIXED_SIZE PROCESSES monitor_sh type NLINK_SH FIXED_SIZE PROCESSES nlink_sh type AINFO_REQ SHORT_LIVED SYSTEM alloc_info_request +type SCHED_WTIME_REQ SHORT_LIVED SYSTEM sched_wall_time_request +endif diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index c32938bdff..e0d525bdde 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2011. All Rights Reserved. + * Copyright Ericsson AB 2002-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -949,7 +949,6 @@ ddq_check_incoming(ErtsAllctrDDQueue_t *ddq) ERTS_THR_MEMORY_BARRIER; else { ddq->head.next.unref_end = (ErtsAllctrDDBlock_t *) ilast; - ERTS_THR_MEMORY_BARRIER; ddq->head.next.thr_progress = erts_thr_progress_later(); erts_atomic32_set_relb(&ddq->tail.data.um_refc_ix, um_refc_ix); @@ -961,12 +960,24 @@ ddq_check_incoming(ErtsAllctrDDQueue_t *ddq) return 1; } +static ERTS_INLINE void +store_earliest_thr_prgr(ErtsThrPrgrVal *prev_val, ErtsAllctrDDQueue_t *ddq) +{ + if (!ddq->head.next.thr_progress_reached + && (*prev_val == ERTS_THR_PRGR_INVALID + || erts_thr_progress_cmp(ddq->head.next.thr_progress, + *prev_val) < 0)) { + *prev_val = ddq->head.next.thr_progress; + } +} + static ERTS_INLINE int handle_delayed_dealloc(Allctr_t *allctr, int allctr_locked, int use_limit, int ops_limit, int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, int *need_more_work) { int need_thr_prgr = 0; @@ -1008,8 +1019,12 @@ handle_delayed_dealloc(Allctr_t *allctr, if (have_checked_incoming) break; need_thr_prgr = ddq_check_incoming(ddq); - if (need_thr_progress) + if (need_thr_progress) { *need_thr_progress |= need_thr_prgr; + if (need_thr_prgr) + store_earliest_thr_prgr(thr_prgr_p, ddq); + + } have_checked_incoming = 1; goto dequeue; } @@ -1067,6 +1082,8 @@ handle_delayed_dealloc(Allctr_t *allctr, if (need_thr_progress && !(need_thr_prgr | need_mr_wrk)) { need_thr_prgr = ddq_check_incoming(ddq); *need_thr_progress |= need_thr_prgr; + if (need_thr_prgr) + store_earliest_thr_prgr(thr_prgr_p, ddq); } if (allctr->thread_safe && !allctr_locked) @@ -1086,25 +1103,27 @@ enqueue_dealloc_other_instance(ErtsAlcType_t type, Allctr_t *allctr, void *ptr) #endif +#ifdef ERTS_SMP void erts_alcu_check_delayed_dealloc(Allctr_t *allctr, int limit, int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, int *more_work) { -#ifdef ERTS_SMP handle_delayed_dealloc(allctr, 0, limit, ERTS_ALCU_DD_OPS_LIM_HIGH, need_thr_progress, + thr_prgr_p, more_work); -#endif } +#endif #define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \ handle_delayed_dealloc((Allctr), (Locked), 1, \ - ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL) + ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL, NULL) /* Multi block carrier alloc/realloc/free ... */ @@ -3956,7 +3975,8 @@ realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size, if (used_allctr->thread_safe && (!force_move || used_allctr != pref_allctr)) erts_mtx_lock(&used_allctr->mutex); - ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&used_allctr->mutex)); + ERTS_SMP_LC_ASSERT(!used_allctr->thread_safe || + erts_lc_mtx_is_locked(&used_allctr->mutex)); cpy_size = BLK_SZ(blk); if (used_allctr->thread_safe && (!force_move || used_allctr != pref_allctr)) diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index fc1eddb116..cedf4ccf85 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2011. All Rights Reserved. + * Copyright Ericsson AB 2002-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -170,7 +170,9 @@ Eterm erts_alcu_info(Allctr_t *, int, int *, void *, Uint **, Uint *); void erts_alcu_init(AlcUInit_t *); void erts_alcu_current_size(Allctr_t *, AllctrSize_t *, ErtsAlcUFixInfo_t *, int); -void erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, int *); +#ifdef ERTS_SMP +void erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, ErtsThrPrgrVal *, int *); +#endif erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); #endif diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 5a806777fe..ebd475f73a 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -57,6 +57,8 @@ static Export* alloc_info_trap = NULL; static Export* alloc_sizes_trap = NULL; +static Export *gather_sched_wall_time_res_trap; + #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) /* Keep erts_system_version as a global variable for easy access from a core */ @@ -3180,7 +3182,12 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) Eterm res; Eterm* hp; - if (BIF_ARG_1 == am_context_switches) { + if (BIF_ARG_1 == am_scheduler_wall_time) { + res = erts_sched_wall_time_request(BIF_P, 0, 0); + if (is_non_value(res)) + BIF_RET(am_undefined); + BIF_TRAP1(gather_sched_wall_time_res_trap, BIF_P, res); + } else if (BIF_ARG_1 == am_context_switches) { Eterm cs = erts_make_integer(erts_get_total_context_switches(), BIF_P); hp = HAlloc(BIF_P, 3); res = TUPLE2(hp, cs, SMALL_ZERO); @@ -4160,6 +4167,8 @@ erts_bif_info_init(void) alloc_info_trap = erts_export_put(am_erlang, am_alloc_info, 1); alloc_sizes_trap = erts_export_put(am_erlang, am_alloc_sizes, 1); + gather_sched_wall_time_res_trap + = erts_export_put(am_erlang, am_gather_sched_wall_time_result, 1); process_info_init(); os_info_init(); } diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index eb89baf1c9..51bdf53823 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -251,7 +251,7 @@ free_dbtable(DbTable* tb) ASSERT(is_immed(tb->common.heir_data)); erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable)); ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable)); - ERTS_THR_MEMORY_BARRIER; + ERTS_SMP_MEMORY_BARRIER; } #ifdef ERTS_SMP @@ -3816,7 +3816,7 @@ void db_info(int to, void *to_arg, int show) /* Called by break handler */ Uint erts_get_ets_misc_mem_size(void) { - ERTS_THR_MEMORY_BARRIER; + ERTS_SMP_MEMORY_BARRIER; /* Memory not allocated in ets_alloc */ return (Uint) erts_smp_atomic_read_nob(&erts_ets_misc_mem_size); } diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 038a667b06..c726be5fb4 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2011. All Rights Reserved. + * Copyright Ericsson AB 1998-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -105,11 +105,7 @@ #define NSEG_2 256 /* Size of second segment table */ #define NSEG_INC 128 /* Number of segments to grow after that */ -#ifdef ETHR_ORDERED_READ_DEPEND -#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_nob(&(tb)->segtab)) -#else -#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_rb(&(tb)->segtab)) -#endif +#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_ddrb(&(tb)->segtab)) #define NACTIVE(tb) ((int)erts_smp_atomic_read_nob(&(tb)->nactive)) #define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems)) diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index e80eae0b86..7510f6b724 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -371,11 +371,17 @@ typedef struct erl_drv_entry { #ifndef ERL_DRIVER_TYPES_ONLY #if defined(VXWORKS) -# define DRIVER_INIT(DRIVER_NAME) ErlDrvEntry* DRIVER_NAME ## _init(void) +# define DRIVER_INIT(DRIVER_NAME) \ + ErlDrvEntry* DRIVER_NAME ## _init(void); \ + ErlDrvEntry* DRIVER_NAME ## _init(void) #elif defined(__WIN32__) -# define DRIVER_INIT(DRIVER_NAME) __declspec(dllexport) ErlDrvEntry* driver_init(void) +# define DRIVER_INIT(DRIVER_NAME) \ + __declspec(dllexport) ErlDrvEntry* driver_init(void); \ + __declspec(dllexport) ErlDrvEntry* driver_init(void) #else -# define DRIVER_INIT(DRIVER_NAME) ErlDrvEntry* driver_init(void) +# define DRIVER_INIT(DRIVER_NAME) \ + ErlDrvEntry* driver_init(void); \ + ErlDrvEntry* driver_init(void) #endif /* diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index eb2b945877..bde87b8346 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -357,10 +357,11 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); p->gcstatus = p->status; p->status = P_GARBING; + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + if (erts_system_monitor_long_gc != 0) { get_now(&ms1, &s1, &us1); } - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); ERTS_CHK_OFFHEAP(p); @@ -910,7 +911,18 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) * XXX: WARNING: If HiPE starts storing other non-Erlang values on the * nstack, such as floats, then this will have to be changed. */ -#define offset_nstack(p,offs,area,area_size) offset_heap_ptr(hipe_nstack_start((p)),hipe_nstack_used((p)),(offs),(area),(area_size)) +static ERTS_INLINE void offset_nstack(Process* p, Sint offs, + char* area, Uint area_size) +{ + if (p->hipe.nstack) { + ASSERT(p->hipe.nsp && p->hipe.nstend); + offset_heap_ptr(hipe_nstack_start(p), hipe_nstack_used(p), + offs, area, area_size); + } + else { + ASSERT(!p->hipe.nsp && !p->hipe.nstend); + } +} #else /* !HIPE */ diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 2b5e65b11a..a2b08fcf56 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -731,7 +731,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) int reds = ERTS_PORT_REDS_EXECUTE; erts_aint_t io_tasks_executed = 0; int fpe_was_unmasked; - ErtsPortTaskExeBlockData blk_data = {runq, NULL}; ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); @@ -965,8 +964,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) #endif done: - blk_data.resp = &res; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); ERTS_PORT_REDUCTIONS_EXECUTED(runq, reds); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 5469a59d8c..138acfeb2c 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -228,7 +228,6 @@ static Uint last_reductions; static Uint last_exact_reductions; Uint erts_default_process_flags; Eterm erts_system_monitor; -Eterm erts_system_monitor_msg_queue_len; Eterm erts_system_monitor_long_gc; Eterm erts_system_monitor_large_heap; struct erts_system_monitor_flags_t erts_system_monitor_flags; @@ -348,40 +347,25 @@ dbg_chk_aux_work_val(erts_aint32_t value) { erts_aint32_t valid = 0; -#ifdef ERTS_SSI_AUX_WORK_SET_TMO valid |= ERTS_SSI_AUX_WORK_SET_TMO; -#endif -#ifdef ERTS_SSI_AUX_WORK_CHECK_CHILDREN - valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN; -#endif -#ifdef ERTS_SSI_AUX_WORK_MISC valid |= ERTS_SSI_AUX_WORK_MISC; -#endif -#ifdef ERTS_SSI_AUX_WORK_MISC_THR_PRGR - valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR; -#endif -#ifdef ERTS_SSI_AUX_WORK_ASYNC_READY - valid |= ERTS_SSI_AUX_WORK_ASYNC_READY; -#endif -#ifdef ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN - valid |= ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; -#endif - -#ifdef ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM; -#endif -#ifdef ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC; +#if ERTS_USE_ASYNC_READY_Q + valid |= ERTS_SSI_AUX_WORK_ASYNC_READY; + valid |= ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; #endif -#ifdef ERTS_SSI_AUX_WORK_DD +#ifdef ERTS_SMP + valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR; valid |= ERTS_SSI_AUX_WORK_DD; -#endif -#ifdef ERTS_SSI_AUX_WORK_DD valid |= ERTS_SSI_AUX_WORK_DD_THR_PRGR; #endif -#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK +#if HAVE_ERTS_MSEG valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK; #endif +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN + valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN; +#endif if (~valid & value) erl_exit(ERTS_ABORT_EXIT, @@ -537,6 +521,209 @@ erts_late_init_process(void) } +static void +init_sched_wall_time(ErtsSchedWallTime *swtp) +{ + swtp->enabled = 0; + swtp->start = 0; + swtp->working.total = 0; + swtp->working.start = 0; + swtp->working.currently = 0; +} + +static ERTS_INLINE Uint64 +sched_wall_time_ts(void) +{ +#ifdef HAVE_GETHRTIME + return (Uint64) sys_gethrtime(); +#else + Uint64 res; + SysTimeval tv; + sys_gettimeofday(&tv); + res = (Uint64) tv.tv_sec*1000000; + res += (Uint64) tv.tv_usec; + return res; +#endif +} + +static ERTS_INLINE void +sched_wall_time_change(ErtsSchedulerData *esdp, int working) +{ + if (esdp->sched_wall_time.enabled) { + Uint64 ts = sched_wall_time_ts(); + if (working) { +#ifdef DEBUG + ASSERT(!esdp->sched_wall_time.working.currently); + esdp->sched_wall_time.working.currently = 1; +#endif + ts -= esdp->sched_wall_time.start; + esdp->sched_wall_time.working.start = ts; + } + else { +#ifdef DEBUG + ASSERT(esdp->sched_wall_time.working.currently); + esdp->sched_wall_time.working.currently = 0; +#endif + ts -= esdp->sched_wall_time.start; + ts -= esdp->sched_wall_time.working.start; + esdp->sched_wall_time.working.total += ts; + } + } +} + +typedef struct { + int set; + int enable; + Process *proc; + Eterm ref; + Eterm ref_heap[REF_THING_SIZE]; + Uint req_sched; + erts_smp_atomic32_t refc; +} ErtsSchedWallTimeReq; + +#if !HALFWORD_HEAP +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(swtreq, + ErtsSchedWallTimeReq, + 5, + ERTS_ALC_T_SCHED_WTIME_REQ) +#else +static ERTS_INLINE ErtsSchedWallTimeReq * +swtreq_alloc(void) +{ + return erts_alloc(ERTS_ALC_T_SCHED_WTIME_REQ, + sizeof(ErtsSchedWallTimeReq)); +} + +static ERTS_INLINE void +swtreq_free(ErtsSchedWallTimeReq *ptr) +{ + erts_free(ERTS_ALC_T_SCHED_WTIME_REQ, ptr); +} +#endif + +static void +reply_sched_wall_time(void *vswtrp) +{ + Uint64 working = 0, total = 0; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsSchedWallTimeReq *swtrp = (ErtsSchedWallTimeReq *) vswtrp; + ErtsProcLocks rp_locks = (swtrp->req_sched == esdp->no + ? ERTS_PROC_LOCK_MAIN + : 0); + Process *rp = swtrp->proc; + Eterm ref_copy = NIL, msg; + Eterm *hp = NULL; + Eterm **hpp; + Uint sz, *szp; + ErlOffHeap *ohp = NULL; + ErlHeapFragment *bp = NULL; + + ASSERT(esdp); + + if (swtrp->set) { + if (!swtrp->enable && esdp->sched_wall_time.enabled) + esdp->sched_wall_time.enabled = 0; + else if (swtrp->enable && !esdp->sched_wall_time.enabled) { + Uint64 ts = sched_wall_time_ts(); + esdp->sched_wall_time.enabled = 1; + esdp->sched_wall_time.start = ts; + esdp->sched_wall_time.working.total = 0; + esdp->sched_wall_time.working.start = 0; + esdp->sched_wall_time.working.currently = 1; + } + } + + if (esdp->sched_wall_time.enabled) { + Uint64 ts = sched_wall_time_ts(); + ASSERT(esdp->sched_wall_time.working.currently); + ts -= esdp->sched_wall_time.start; + total = ts; + ts -= esdp->sched_wall_time.working.start; + working = esdp->sched_wall_time.working.total + ts; + } + + sz = 0; + hpp = NULL; + szp = &sz; + + while (1) { + if (hpp) + ref_copy = STORE_NC(hpp, ohp, swtrp->ref); + else + *szp += REF_THING_SIZE; + + if (swtrp->set) + msg = ref_copy; + else { + msg = (!esdp->sched_wall_time.enabled + ? am_notsup + : erts_bld_tuple(hpp, szp, 3, + make_small(esdp->no), + erts_bld_uint64(hpp, szp, working), + erts_bld_uint64(hpp, szp, total))); + + msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg); + } + if (hpp) + break; + + hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks); + szp = NULL; + hpp = &hp; + } + + erts_queue_message(rp, &rp_locks, bp, msg, NIL); + + if (swtrp->req_sched == esdp->no) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + + erts_smp_proc_dec_refc(rp); + + if (erts_smp_atomic32_dec_read_nob(&swtrp->refc) == 0) + swtreq_free(vswtrp); +} + +Eterm +erts_sched_wall_time_request(Process *c_p, int set, int enable) +{ + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + Eterm ref; + ErtsSchedWallTimeReq *swtrp; + Eterm *hp; + + if (!set && !esdp->sched_wall_time.enabled) + return THE_NON_VALUE; + + swtrp = swtreq_alloc(); + ref = erts_make_ref(c_p); + hp = &swtrp->ref_heap[0]; + + swtrp->set = set; + swtrp->enable = enable; + swtrp->proc = c_p; + swtrp->ref = STORE_NC(&hp, NULL, ref); + swtrp->req_sched = esdp->no; + erts_smp_atomic32_init_nob(&swtrp->refc, + (erts_aint32_t) erts_no_schedulers); + + erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers); + +#ifdef ERTS_SMP + if (erts_no_schedulers > 1) + erts_schedule_multi_misc_aux_work(1, + erts_no_schedulers, + reply_sched_wall_time, + (void *) swtrp); +#endif + + reply_sched_wall_time((void *) swtrp); + + return ref; +} + static ERTS_INLINE ErtsProcList * proclist_create(Process *p) { @@ -709,6 +896,27 @@ unset_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs) return erts_atomic32_read_band_nob(&ssi->aux_work, ~flgs); } +#ifdef ERTS_SMP + +static ERTS_INLINE void +thr_prgr_current_reset(ErtsAuxWorkData *awdp) +{ + awdp->current_thr_prgr = ERTS_THR_PRGR_INVALID; +} + +static ERTS_INLINE ErtsThrPrgrVal +thr_prgr_current(ErtsAuxWorkData *awdp) +{ + ErtsThrPrgrVal current = awdp->current_thr_prgr; + if (current == ERTS_THR_PRGR_INVALID) { + current = erts_thr_progress_current(); + awdp->current_thr_prgr = current; + } + return current; +} + +#endif + typedef struct erts_misc_aux_work_t_ erts_misc_aux_work_t; struct erts_misc_aux_work_t_ { void (*func)(void *); @@ -781,7 +989,7 @@ misc_aux_work_clean(ErtsThrQ_t *q, return aux_work; } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_misc_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { @@ -801,12 +1009,13 @@ handle_misc_aux_work(ErtsAuxWorkData *awdp, #ifdef ERTS_SMP -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { - if (!erts_thr_progress_has_reached(awdp->misc.thr_prgr)) - return aux_work; + if (!erts_thr_progress_has_reached_this(thr_prgr_current(awdp), + awdp->misc.thr_prgr)) + return aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR; unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR); @@ -879,7 +1088,7 @@ erts_notify_check_async_ready_queue(void *vno) ERTS_SSI_AUX_WORK_ASYNC_READY); } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_async_ready(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { @@ -901,7 +1110,7 @@ handle_async_ready(ErtsAuxWorkData *awdp, | ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN); } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_async_ready_clean(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { @@ -909,7 +1118,8 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp, #ifdef ERTS_SMP if (awdp->async_ready.need_thr_prgr - && !erts_thr_progress_has_reached(awdp->async_ready.thr_prgr)) { + && !erts_thr_progress_has_reached_this(thr_prgr_current(awdp), + awdp->async_ready.thr_prgr)) { return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; } @@ -937,7 +1147,7 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp, #endif -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { ErtsSchedulerSleepInfo *ssi = awdp->ssi; @@ -965,16 +1175,18 @@ erts_alloc_notify_delayed_dealloc(int ix) ERTS_SSI_AUX_WORK_DD); } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { ErtsSchedulerSleepInfo *ssi = awdp->ssi; int need_thr_progress = 0; + ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; int more_work = 0; unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD); erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp, &need_thr_progress, + &wakeup, &more_work); if (more_work) { if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD) @@ -986,9 +1198,12 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) } if (need_thr_progress) { + if (wakeup == ERTS_THR_PRGR_INVALID) + wakeup = erts_thr_progress_later_than(thr_prgr_current(awdp)); + awdp->dd.thr_prgr = wakeup; set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); - awdp->dd.thr_prgr = erts_thr_progress_later(); - erts_thr_progress_wakeup(awdp->esdp, awdp->dd.thr_prgr); + awdp->dd.thr_prgr = wakeup; + erts_thr_progress_wakeup(awdp->esdp, wakeup); } else if (awdp->dd.completed_callback) { awdp->dd.completed_callback(awdp->dd.completed_arg); @@ -998,14 +1213,16 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) return aux_work & ~ERTS_SSI_AUX_WORK_DD; } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { ErtsSchedulerSleepInfo *ssi; int need_thr_progress; int more_work; + ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; + ErtsThrPrgrVal current = thr_prgr_current(awdp); - if (!erts_thr_progress_has_reached(awdp->dd.thr_prgr)) + if (!erts_thr_progress_has_reached_this(current, awdp->dd.thr_prgr)) return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR; ssi = awdp->ssi; @@ -1014,6 +1231,7 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp, &need_thr_progress, + &wakeup, &more_work); if (more_work) { set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD); @@ -1023,8 +1241,10 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) } if (need_thr_progress) { - awdp->dd.thr_prgr = erts_thr_progress_later(); - erts_thr_progress_wakeup(awdp->esdp, awdp->dd.thr_prgr); + if (wakeup == ERTS_THR_PRGR_INVALID) + wakeup = erts_thr_progress_later_than(current); + awdp->dd.thr_prgr = wakeup; + erts_thr_progress_wakeup(awdp->esdp, wakeup); } else { unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); @@ -1121,7 +1341,7 @@ erts_smp_notify_check_children_needed(void) ERTS_SSI_AUX_WORK_CHECK_CHILDREN); } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_CHECK_CHILDREN); @@ -1131,9 +1351,9 @@ handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) #endif -#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK +#if HAVE_ERTS_MSEG -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK); @@ -1143,7 +1363,7 @@ handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) #endif -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_SET_TMO); @@ -1151,69 +1371,92 @@ handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) return aux_work & ~ERTS_SSI_AUX_WORK_SET_TMO; } -static ERTS_INLINE erts_aint32_t -handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +static erts_aint32_t +handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work) { +#undef HANDLE_AUX_WORK +#define HANDLE_AUX_WORK(FLG, HNDLR) \ + ignore |= FLG; \ + if (aux_work & FLG) { \ + aux_work = HNDLR(awdp, aux_work); \ + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \ + if (!(aux_work & ~ignore)) { \ + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \ + return aux_work; \ + } \ + } + + erts_aint32_t aux_work = orig_aux_work; + erts_aint32_t ignore = 0; + +#ifdef ERTS_SMP + thr_prgr_current_reset(awdp); +#endif + + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + ASSERT(aux_work); + /* * Handlers are *only* allowed to modify flags in return value * and ssi flags that are explicity handled by the handler. * Handlers are, e.g., not allowed to read the ssi flag field and * then unconditionally return that value. + * + * Flag field returned should only contain flags for work that + * can continue immediately. + */ + + /* + * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative + * eachother. Most frequent first. */ - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - if (aux_work & ERTS_SSI_AUX_WORK_SET_TMO) { - aux_work = handle_setup_aux_work_timer(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } #ifdef ERTS_SMP - if (aux_work & ERTS_SSI_AUX_WORK_MISC_THR_PRGR) { - aux_work = handle_misc_aux_work_thr_prgr(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD, + handle_delayed_dealloc); + /* DD must be before DD_THR_PRGR */ + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD_THR_PRGR, + handle_delayed_dealloc_thr_prgr); #endif - if (aux_work & ERTS_SSI_AUX_WORK_MISC) { - aux_work = handle_misc_aux_work(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + + HANDLE_AUX_WORK((ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), + handle_fix_alloc); + #if ERTS_USE_ASYNC_READY_Q - if (aux_work & ERTS_SSI_AUX_WORK_ASYNC_READY) { - aux_work = handle_async_ready(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } - if (aux_work & ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN) { - aux_work = handle_async_ready_clean(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY, + handle_async_ready); + /* ASYNC_READY must be before ASYNC_READY_CLEAN */ + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN, + handle_async_ready_clean); #endif -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) { - aux_work = handle_check_children(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } -#endif - if (aux_work & (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM - | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC)) { - aux_work = handle_fix_alloc(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + #ifdef ERTS_SMP - if (aux_work & ERTS_SSI_AUX_WORK_DD) { - aux_work = handle_delayed_dealloc(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } - if (aux_work & ERTS_SSI_AUX_WORK_DD_THR_PRGR) { - aux_work = handle_delayed_dealloc_thr_prgr(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC_THR_PRGR, + handle_misc_aux_work_thr_prgr); #endif -#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK - if (aux_work & ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK) { - aux_work = handle_mseg_cache_check(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + /* MISC_THR_PRGR must be before MISC */ + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC, + handle_misc_aux_work); + +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CHECK_CHILDREN, + handle_check_children); #endif + + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_SET_TMO, + handle_setup_aux_work_timer); + +#if HAVE_ERTS_MSEG + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK, + handle_mseg_cache_check); +#endif + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + return aux_work; + +#undef HANDLE_AUX_WORK + } typedef struct { @@ -1707,6 +1950,7 @@ aux_thread(void *unused) static void scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) { + int working = 1; ErtsSchedulerSleepInfo *ssi = esdp->ssi; int spincount; erts_aint32_t aux_work = 0; @@ -1737,12 +1981,17 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) tse_wait: + if (thr_prgr_active != working) + sched_wall_time_change(esdp, thr_prgr_active); + while (1) { aux_work = erts_atomic32_read_acqb(&ssi->aux_work); if (aux_work) { - if (!thr_prgr_active) + if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } aux_work = handle_aux_work(&esdp->aux_work_data, aux_work); if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); @@ -1751,8 +2000,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) if (aux_work) flgs = erts_smp_atomic32_read_acqb(&ssi->flags); else { - if (thr_prgr_active) + if (thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 0); + sched_wall_time_change(esdp, 0); + } erts_thr_progress_prepare_wait(esdp); flgs = sched_spin_wait(ssi, spincount); @@ -1789,8 +2040,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) if (flgs & ~ERTS_SSI_FLG_SUSPENDED) erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); - if (!thr_prgr_active) + if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } erts_smp_runq_lock(rq); sched_active(esdp->no, rq); @@ -1806,14 +2059,21 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) sched_waiting_sys(esdp->no, rq); + erts_smp_runq_unlock(rq); + ASSERT(working); + sched_wall_time_change(esdp, working = 0); + spincount = ERTS_SCHED_SYS_SLEEP_SPINCOUNT; while (spincount-- > 0) { sys_poll_aux_work: + if (working) + sched_wall_time_change(esdp, working = 0); + ASSERT(!erts_port_task_have_outstanding_io_tasks()); erl_sys_schedule(1); /* Might give us something to do */ @@ -1828,6 +2088,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) aux_work = erts_atomic32_read_acqb(&ssi->aux_work); if (aux_work) { + if (!working) + sched_wall_time_change(esdp, working = 1); #ifdef ERTS_SMP if (!thr_prgr_active) erts_thr_progress_active(esdp, thr_prgr_active = 1); @@ -1920,6 +2182,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_smp_runq_unlock(rq); + if (working) + sched_wall_time_change(esdp, working = 0); + #ifdef ERTS_SMP if (thr_prgr_active) erts_thr_progress_active(esdp, thr_prgr_active = 0); @@ -1955,6 +2220,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) if (flgs & ~ERTS_SSI_FLG_SUSPENDED) erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); #endif + if (!working) + sched_wall_time_change(esdp, working = 1); sched_active_sys(esdp->no, rq); } @@ -3430,9 +3697,14 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) esdp->run_queue->scheduler = esdp; init_aux_work_data(&esdp->aux_work_data, esdp); + init_sched_wall_time(&esdp->sched_wall_time); } init_misc_aux_work(); +#if !HALFWORD_HEAP + init_swtreq_alloc(); +#endif + #ifdef ERTS_SMP @@ -3755,6 +4027,8 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_inactive); + sched_wall_time_change(esdp, 0); + erts_smp_mtx_lock(&schdlr_sspnd.mtx); flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); @@ -3813,16 +4087,20 @@ suspend_scheduler(ErtsSchedulerData *esdp) aux_work = erts_atomic32_read_acqb(&ssi->aux_work); if (aux_work) { - if (!thr_prgr_active) + if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } aux_work = handle_aux_work(&esdp->aux_work_data, aux_work); if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); } if (!aux_work) { - if (thr_prgr_active) + if (thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 0); + sched_wall_time_change(esdp, 0); + } erts_thr_progress_prepare_wait(esdp); flgs = sched_spin_suspended(ssi, ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); @@ -3877,8 +4155,10 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_active); - if (!thr_prgr_active) + if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index a51b380bb0..c23810f15a 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -248,25 +248,22 @@ typedef enum { | ERTS_SSI_FLG_WAITING \ | ERTS_SSI_FLG_SUSPENDED) -#define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 0) -#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 1) -#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 2) -#ifdef ERTS_SMP -#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR (((erts_aint32_t) 1) << 3) -#endif -#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM (((erts_aint32_t) 1) << 4) -#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC (((erts_aint32_t) 1) << 5) -#define ERTS_SSI_AUX_WORK_ASYNC_READY (((erts_aint32_t) 1) << 6) -#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN (((erts_aint32_t) 1) << 7) -#ifdef ERTS_SMP -#define ERTS_SSI_AUX_WORK_DD (((erts_aint32_t) 1) << 8) -#define ERTS_SSI_AUX_WORK_DD_THR_PRGR (((erts_aint32_t) 1) << 9) -#endif -#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 10) +/* + * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative + * eachother. Most frequent - lowest bit number. + */ -#if !HAVE_ERTS_MSEG -# undef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK -#endif +#define ERTS_SSI_AUX_WORK_DD (((erts_aint32_t) 1) << 0) +#define ERTS_SSI_AUX_WORK_DD_THR_PRGR (((erts_aint32_t) 1) << 1) +#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC (((erts_aint32_t) 1) << 2) +#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM (((erts_aint32_t) 1) << 3) +#define ERTS_SSI_AUX_WORK_ASYNC_READY (((erts_aint32_t) 1) << 4) +#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN (((erts_aint32_t) 1) << 5) +#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR (((erts_aint32_t) 1) << 6) +#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 7) +#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 8) +#define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 9) +#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 10) typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo; @@ -394,9 +391,22 @@ do { \ } while (0) typedef struct { + int enabled; + Uint64 start; + struct { + Uint64 total; + Uint64 start; + int currently; + } working; +} ErtsSchedWallTime; + +typedef struct { int sched_id; ErtsSchedulerData *esdp; ErtsSchedulerSleepInfo *ssi; +#ifdef ERTS_SMP + ErtsThrPrgrVal current_thr_prgr; +#endif struct { int ix; #ifdef ERTS_SMP @@ -457,6 +467,8 @@ struct ErtsSchedulerData_ { ErtsSchedAllocData alloc_data; + ErtsSchedWallTime sched_wall_time; + #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC erts_alloc_verify_func_t verify_unused_temp_alloc; Allctr_t *verify_unused_temp_alloc_data; @@ -1064,6 +1076,8 @@ void erts_late_init_process(void); void erts_early_init_scheduling(int); void erts_init_scheduling(int, int); +Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable); + ErtsProcList *erts_proclist_create(Process *); void erts_proclist_destroy(ErtsProcList *); int erts_proclist_same(ErtsProcList *, Process *); diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c index a7ccea7403..bff9d246a3 100644 --- a/erts/emulator/beam/erl_sched_spec_pre_alloc.c +++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011. All Rights Reserved. + * Copyright Ericsson AB 2011-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -227,7 +227,6 @@ fetch_remote(erts_sspa_chunk_header_t *chdr, int max) ERTS_THR_MEMORY_BARRIER; else { chdr->head.next.unref_end = (erts_sspa_blk_t *) ilast; - ERTS_THR_MEMORY_BARRIER; chdr->head.next.thr_progress = erts_thr_progress_later(); erts_atomic32_set_relb(&chdr->tail.data.um_refc_ix, um_refc_ix); diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index 63179dfad4..a32e9d9d7c 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2011. All Rights Reserved. + * Copyright Ericsson AB 2005-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -61,6 +61,11 @@ typedef erts_spinlock_t erts_smp_spinlock_t; typedef erts_rwlock_t erts_smp_rwlock_t; void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ +#define ERTS_SMP_MEMORY_BARRIER ERTS_THR_MEMORY_BARRIER +#define ERTS_SMP_WRITE_MEMORY_BARRIER ERTS_THR_WRITE_MEMORY_BARRIER +#define ERTS_SMP_READ_MEMORY_BARRIER ERTS_THR_READ_MEMORY_BARRIER +#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER + #else /* #ifdef ERTS_SMP */ #define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0} @@ -95,6 +100,11 @@ typedef struct { int gcc_is_buggy; } erts_smp_spinlock_t; typedef struct { int gcc_is_buggy; } erts_smp_rwlock_t; #endif +#define ERTS_SMP_MEMORY_BARRIER +#define ERTS_SMP_WRITE_MEMORY_BARRIER +#define ERTS_SMP_READ_MEMORY_BARRIER +#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER + #endif /* #ifdef ERTS_SMP */ ERTS_GLB_INLINE void erts_smp_thr_init(erts_smp_thr_init_data_t *id); @@ -206,13 +216,8 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */ /* - * Functions implementing atomic operations with with no (nob), - * full (mb), acquire (acqb), release (relb), read (rb), and - * write (wb) memory barriers. - * - * If SMP support has been disabled, they are mapped to functions - * that performs the same operation, but aren't atomic and don't - * imply memory barriers. + * See "Documentation of atomics and memory barriers" at the top + * of erl_threads.h for info on atomics. */ #ifdef ERTS_SMP @@ -239,6 +244,11 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_dw_atomic_read_relb erts_dw_atomic_read_relb #define erts_smp_dw_atomic_cmpxchg_relb erts_dw_atomic_cmpxchg_relb +#define erts_smp_dw_atomic_init_ddrb erts_dw_atomic_init_ddrb +#define erts_smp_dw_atomic_set_ddrb erts_dw_atomic_set_ddrb +#define erts_smp_dw_atomic_read_ddrb erts_dw_atomic_read_ddrb +#define erts_smp_dw_atomic_cmpxchg_ddrb erts_dw_atomic_cmpxchg_ddrb + #define erts_smp_dw_atomic_init_rb erts_dw_atomic_init_rb #define erts_smp_dw_atomic_set_rb erts_dw_atomic_set_rb #define erts_smp_dw_atomic_read_rb erts_dw_atomic_read_rb @@ -307,6 +317,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb #define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb +#define erts_smp_atomic_init_ddrb erts_atomic_init_ddrb +#define erts_smp_atomic_set_ddrb erts_atomic_set_ddrb +#define erts_smp_atomic_read_ddrb erts_atomic_read_ddrb +#define erts_smp_atomic_inc_read_ddrb erts_atomic_inc_read_ddrb +#define erts_smp_atomic_dec_read_ddrb erts_atomic_dec_read_ddrb +#define erts_smp_atomic_inc_ddrb erts_atomic_inc_ddrb +#define erts_smp_atomic_dec_ddrb erts_atomic_dec_ddrb +#define erts_smp_atomic_add_read_ddrb erts_atomic_add_read_ddrb +#define erts_smp_atomic_add_ddrb erts_atomic_add_ddrb +#define erts_smp_atomic_read_bor_ddrb erts_atomic_read_bor_ddrb +#define erts_smp_atomic_read_band_ddrb erts_atomic_read_band_ddrb +#define erts_smp_atomic_xchg_ddrb erts_atomic_xchg_ddrb +#define erts_smp_atomic_cmpxchg_ddrb erts_atomic_cmpxchg_ddrb + #define erts_smp_atomic_init_rb erts_atomic_init_rb #define erts_smp_atomic_set_rb erts_atomic_set_rb #define erts_smp_atomic_read_rb erts_atomic_read_rb @@ -393,6 +417,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb #define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb +#define erts_smp_atomic32_init_ddrb erts_atomic32_init_ddrb +#define erts_smp_atomic32_set_ddrb erts_atomic32_set_ddrb +#define erts_smp_atomic32_read_ddrb erts_atomic32_read_ddrb +#define erts_smp_atomic32_inc_read_ddrb erts_atomic32_inc_read_ddrb +#define erts_smp_atomic32_dec_read_ddrb erts_atomic32_dec_read_ddrb +#define erts_smp_atomic32_inc_ddrb erts_atomic32_inc_ddrb +#define erts_smp_atomic32_dec_ddrb erts_atomic32_dec_ddrb +#define erts_smp_atomic32_add_read_ddrb erts_atomic32_add_read_ddrb +#define erts_smp_atomic32_add_ddrb erts_atomic32_add_ddrb +#define erts_smp_atomic32_read_bor_ddrb erts_atomic32_read_bor_ddrb +#define erts_smp_atomic32_read_band_ddrb erts_atomic32_read_band_ddrb +#define erts_smp_atomic32_xchg_ddrb erts_atomic32_xchg_ddrb +#define erts_smp_atomic32_cmpxchg_ddrb erts_atomic32_cmpxchg_ddrb + #define erts_smp_atomic32_init_rb erts_atomic32_init_rb #define erts_smp_atomic32_set_rb erts_atomic32_set_rb #define erts_smp_atomic32_read_rb erts_atomic32_read_rb @@ -445,6 +483,11 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_dw_atomic_read_relb erts_no_dw_atomic_read #define erts_smp_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg +#define erts_smp_dw_atomic_init_ddrb erts_no_dw_atomic_init +#define erts_smp_dw_atomic_set_ddrb erts_no_dw_atomic_set +#define erts_smp_dw_atomic_read_ddrb erts_no_dw_atomic_read +#define erts_smp_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg + #define erts_smp_dw_atomic_init_rb erts_no_dw_atomic_init #define erts_smp_dw_atomic_set_rb erts_no_dw_atomic_set #define erts_smp_dw_atomic_read_rb erts_no_dw_atomic_read @@ -513,6 +556,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_xchg_relb erts_no_atomic_xchg #define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg +#define erts_smp_atomic_init_ddrb erts_no_atomic_set +#define erts_smp_atomic_set_ddrb erts_no_atomic_set +#define erts_smp_atomic_read_ddrb erts_no_atomic_read +#define erts_smp_atomic_inc_read_ddrb erts_no_atomic_inc_read +#define erts_smp_atomic_dec_read_ddrb erts_no_atomic_dec_read +#define erts_smp_atomic_inc_ddrb erts_no_atomic_inc +#define erts_smp_atomic_dec_ddrb erts_no_atomic_dec +#define erts_smp_atomic_add_read_ddrb erts_no_atomic_add_read +#define erts_smp_atomic_add_ddrb erts_no_atomic_add +#define erts_smp_atomic_read_bor_ddrb erts_no_atomic_read_bor +#define erts_smp_atomic_read_band_ddrb erts_no_atomic_read_band +#define erts_smp_atomic_xchg_ddrb erts_no_atomic_xchg +#define erts_smp_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg + #define erts_smp_atomic_init_rb erts_no_atomic_set #define erts_smp_atomic_set_rb erts_no_atomic_set #define erts_smp_atomic_read_rb erts_no_atomic_read @@ -599,6 +656,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg #define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg +#define erts_smp_atomic32_init_ddrb erts_no_atomic32_set +#define erts_smp_atomic32_set_ddrb erts_no_atomic32_set +#define erts_smp_atomic32_read_ddrb erts_no_atomic32_read +#define erts_smp_atomic32_inc_read_ddrb erts_no_atomic32_inc_read +#define erts_smp_atomic32_dec_read_ddrb erts_no_atomic32_dec_read +#define erts_smp_atomic32_inc_ddrb erts_no_atomic32_inc +#define erts_smp_atomic32_dec_ddrb erts_no_atomic32_dec +#define erts_smp_atomic32_add_read_ddrb erts_no_atomic32_add_read +#define erts_smp_atomic32_add_ddrb erts_no_atomic32_add +#define erts_smp_atomic32_read_bor_ddrb erts_no_atomic32_read_bor +#define erts_smp_atomic32_read_band_ddrb erts_no_atomic32_read_band +#define erts_smp_atomic32_xchg_ddrb erts_no_atomic32_xchg +#define erts_smp_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg + #define erts_smp_atomic32_init_rb erts_no_atomic32_set #define erts_smp_atomic32_set_rb erts_no_atomic32_set #define erts_smp_atomic32_read_rb erts_no_atomic32_read diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c index 75f8209c3b..9ef83746c5 100644 --- a/erts/emulator/beam/erl_thr_progress.c +++ b/erts/emulator/beam/erl_thr_progress.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011. All Rights Reserved. + * Copyright Ericsson AB 2011-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -33,10 +33,11 @@ * This module keeps track of the progress of a set of managed threads. Only * threads that behave well can be allowed to be managed. A managed thread * should update its thread progress frequently. Currently only scheduler - * threads and the aux_thread are managed threads. We typically do not want - * any async threads as managed threads since they cannot guarantee a - * frequent update of thread progress, since they execute user implemented - * driver code. + * threads, the system-message-dispatcher threads, and the aux-thread are + * managed threads. We typically do not want any async threads as managed + * threads since they cannot guarantee a frequent update of thread progress, + * since they execute user implemented driver code that is assumed to be + * time consuming. * * erts_thr_progress_current() returns the global current thread progress * value of managed threads. I.e., the latest progress value that all @@ -112,8 +113,10 @@ * * On 32-bit systems we therefore need a double word atomic. */ - +#undef read_acqb #define read_acqb erts_thr_prgr_read_acqb__ +#undef read_nob +#define read_nob erts_thr_prgr_read_nob__ #ifdef ARCH_64 @@ -129,12 +132,6 @@ set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) erts_atomic_set_nob(atmc, val); } -static ERTS_INLINE ErtsThrPrgrVal -read_nob(ERTS_THR_PRGR_ATOMIC *atmc) -{ - return (ErtsThrPrgrVal) erts_atomic_read_nob(atmc); -} - static ERTS_INLINE void init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) { @@ -143,52 +140,44 @@ init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) #else -#undef dw_sint_to_val -#define dw_sint_to_val erts_thr_prgr_dw_sint_to_val__ +#undef dw_aint_to_val +#define dw_aint_to_val erts_thr_prgr_dw_aint_to_val__ static void -val_to_dw_sint(ethr_dw_sint_t *dw_sint, ErtsThrPrgrVal val) +val_to_dw_aint(erts_dw_aint_t *dw_aint, ErtsThrPrgrVal val) { #ifdef ETHR_SU_DW_NAINT_T__ - dw_sint->dw_sint = (ETHR_SU_DW_NAINT_T__) val; + dw_aint->dw_sint = (ETHR_SU_DW_NAINT_T__) val; #else - dw_sint->sint[ETHR_DW_SINT_LOW_WORD] - = (ethr_sint_t) (val & 0xffffffff); - dw_sint->sint[ETHR_DW_SINT_HIGH_WORD] - = (ethr_sint_t) ((val >> 32) & 0xffffffff); + dw_aint->sint[ERTS_DW_AINT_LOW_WORD] + = (erts_aint_t) (val & 0xffffffff); + dw_aint->sint[ERTS_DW_AINT_HIGH_WORD] + = (erts_aint_t) ((val >> 32) & 0xffffffff); #endif } static ERTS_INLINE void set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) { - ethr_dw_sint_t dw_sint; - val_to_dw_sint(&dw_sint, val); - erts_dw_atomic_set_mb(atmc, &dw_sint); + erts_dw_aint_t dw_aint; + val_to_dw_aint(&dw_aint, val); + erts_dw_atomic_set_mb(atmc, &dw_aint); } static ERTS_INLINE void set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) { - ethr_dw_sint_t dw_sint; - val_to_dw_sint(&dw_sint, val); - erts_dw_atomic_set_nob(atmc, &dw_sint); -} - -static ERTS_INLINE ErtsThrPrgrVal -read_nob(ERTS_THR_PRGR_ATOMIC *atmc) -{ - ethr_dw_sint_t dw_sint; - erts_dw_atomic_read_nob(atmc, &dw_sint); - return erts_thr_prgr_dw_sint_to_val__(&dw_sint); + erts_dw_aint_t dw_aint; + val_to_dw_aint(&dw_aint, val); + erts_dw_atomic_set_nob(atmc, &dw_aint); } static ERTS_INLINE void init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) { - ethr_dw_sint_t dw_sint; - val_to_dw_sint(&dw_sint, val); - erts_dw_atomic_init_nob(atmc, &dw_sint); + erts_dw_aint_t dw_aint; + val_to_dw_aint(&dw_aint, val); + erts_dw_atomic_init_nob(atmc, &dw_aint); } #endif @@ -1222,9 +1211,9 @@ erts_thr_progress_block(void) } void -erts_thr_progress_fatal_error_block(SWord timeout) +erts_thr_progress_fatal_error_block(SWord timeout, + ErtsThrPrgrData *tmp_tpd_bufp) { - ErtsThrPrgrData tpd_buf; ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL); erts_aint32_t bc; SWord time_left = timeout; @@ -1248,7 +1237,7 @@ erts_thr_progress_fatal_error_block(SWord timeout) * since we never complete an unblock after a fatal error * block. */ - tpd = &tpd_buf; + tpd = tmp_tpd_bufp; init_tmp_thr_prgr_data(tpd); } diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h index 68d14174b9..a71724b813 100644 --- a/erts/emulator/beam/erl_thr_progress.h +++ b/erts/emulator/beam/erl_thr_progress.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011. All Rights Reserved. + * Copyright Ericsson AB 2011-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -44,7 +44,6 @@ #define erts_smp_thr_progress_unblock erts_thr_progress_unblock #define erts_smp_thr_progress_is_blocking erts_thr_progress_is_blocking -void erts_thr_progress_fatal_error_block(SWord timeout); void erts_thr_progress_block(void); void erts_thr_progress_unblock(void); int erts_thr_progress_is_blocking(void); @@ -73,6 +72,10 @@ typedef struct { ErtsThrPrgrVal current; } previous; } ErtsThrPrgrData; + +void erts_thr_progress_fatal_error_block(SWord timeout, + ErtsThrPrgrData *tmp_tpd_bufp); + #endif /* ERTS_SMP */ #endif @@ -86,6 +89,7 @@ typedef struct { #ifdef ERTS_SMP #define ERTS_THR_PRGR_VAL_WAITING (~((ErtsThrPrgrVal) 0)) +#define ERTS_THR_PRGR_INVALID (~((ErtsThrPrgrVal) 0)) extern erts_tsd_key_t erts_thr_prgr_data_key__; @@ -127,14 +131,20 @@ void erts_thr_progress_dbg_print_state(void); #ifdef ARCH_32 #define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t -ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_dw_sint_to_val__(ethr_dw_sint_t *dw_sint); +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint); #endif +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc); ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc); +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc); ERTS_GLB_INLINE int erts_thr_progress_is_managed_thread(void); +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current_to_later__(ErtsThrPrgrVal val); +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later_than(ErtsThrPrgrVal val); ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later(void); ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current(void); ERTS_GLB_INLINE int erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2); +ERTS_GLB_INLINE int erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val); +ERTS_GLB_INLINE int erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2); ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -142,33 +152,61 @@ ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val); #ifdef ARCH_64 ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc) +{ + return (ErtsThrPrgrVal) erts_atomic_read_nob(atmc); +} + +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc) { return (ErtsThrPrgrVal) erts_atomic_read_acqb(atmc); } +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc) +{ + return (ErtsThrPrgrVal) erts_atomic_read_mb(atmc); +} + #else /* ARCH_32 */ ERTS_GLB_INLINE ErtsThrPrgrVal -erts_thr_prgr_dw_sint_to_val__(ethr_dw_sint_t *dw_sint) +erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint) { #ifdef ETHR_SU_DW_NAINT_T__ - return (ErtsThrPrgrVal) dw_sint->dw_sint; + return (ErtsThrPrgrVal) dw_aint->dw_sint; #else ErtsThrPrgrVal res; - res = (ErtsThrPrgrVal) ((Uint32) dw_sint->sint[ETHR_DW_SINT_HIGH_WORD]); + res = (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_HIGH_WORD]); res <<= 32; - res |= (ErtsThrPrgrVal) ((Uint32) dw_sint->sint[ETHR_DW_SINT_LOW_WORD]); + res |= (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_LOW_WORD]); return res; #endif } ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc) +{ + erts_dw_aint_t dw_aint; + erts_dw_atomic_read_nob(atmc, &dw_aint); + return erts_thr_prgr_dw_aint_to_val__(&dw_aint); +} + +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc) { - ethr_dw_sint_t dw_sint; - erts_dw_atomic_read_acqb(atmc, &dw_sint); - return erts_thr_prgr_dw_sint_to_val__(&dw_sint); + erts_dw_aint_t dw_aint; + erts_dw_atomic_read_acqb(atmc, &dw_aint); + return erts_thr_prgr_dw_aint_to_val__(&dw_aint); +} + +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc) +{ + erts_dw_aint_t dw_aint; + erts_dw_atomic_read_mb(atmc, &dw_aint); + return erts_thr_prgr_dw_aint_to_val__(&dw_aint); } #endif @@ -181,9 +219,8 @@ erts_thr_progress_is_managed_thread(void) } ERTS_GLB_INLINE ErtsThrPrgrVal -erts_thr_progress_later(void) +erts_thr_progress_current_to_later__(ErtsThrPrgrVal val) { - ErtsThrPrgrVal val = erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current); if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)2))) return ((ErtsThrPrgrVal) 0); else if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)1))) @@ -193,9 +230,26 @@ erts_thr_progress_later(void) } ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_progress_later_than(ErtsThrPrgrVal val) +{ + ERTS_THR_MEMORY_BARRIER; + return erts_thr_progress_current_to_later__(val); +} + +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_progress_later(void) +{ + ErtsThrPrgrVal val = erts_thr_prgr_read_mb__(&erts_thr_prgr__.current); + return erts_thr_progress_current_to_later__(val); +} + +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current(void) { - return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current); + if (erts_thr_progress_is_managed_thread()) + return erts_thr_prgr_read_nob__(&erts_thr_prgr__.current); + else + return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current); } ERTS_GLB_INLINE int @@ -217,13 +271,29 @@ erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val0) } ERTS_GLB_INLINE int -erts_thr_progress_has_reached(ErtsThrPrgrVal val) +erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val) { - ErtsThrPrgrVal current; - current = erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current); - if (current == val) + if (this == val) return 1; - return erts_thr_progress_has_passed__(current, val); + return erts_thr_progress_has_passed__(this, val); +} + +ERTS_GLB_INLINE int +erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2) +{ + if (val1 == val2) + return 0; + if (erts_thr_progress_has_passed__(val1, val2)) + return 1; + else + return -1; +} + +ERTS_GLB_INLINE int +erts_thr_progress_has_reached(ErtsThrPrgrVal val) +{ + ErtsThrPrgrVal current = erts_thr_progress_current(); + return erts_thr_progress_has_reached_this(current, val); } #endif diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c index efb8c635d7..70949ece76 100644 --- a/erts/emulator/beam/erl_thr_queue.c +++ b/erts/emulator/beam/erl_thr_queue.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011. All Rights Reserved. + * Copyright Ericsson AB 2011-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -418,10 +418,9 @@ clean(ErtsThrQ_t *q, int max_ops, int do_notify) } if (q->head.unref_end == (ErtsThrQElement_t *) ilast) - ERTS_THR_MEMORY_BARRIER; + ERTS_SMP_MEMORY_BARRIER; else { q->head.next.unref_end = (ErtsThrQElement_t *) ilast; - ERTS_THR_MEMORY_BARRIER; #ifdef ERTS_SMP q->head.next.thr_progress = erts_thr_progress_later(); #endif diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index 065e7077c0..ee47c98009 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2011. All Rights Reserved. + * Copyright Ericsson AB 2001-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -25,6 +25,235 @@ #ifndef ERL_THREAD_H__ #define ERL_THREAD_H__ +/* + * --- Documentation of atomics and memory barriers -------------------------- + * + * The following explicit memory barriers exist: + * + * - ERTS_THR_MEMORY_BARRIER + * Full memory barrier. Orders both loads, and stores. No + * load or store is allowed to be reordered over the + * barrier. + * - ERTS_THR_WRITE_MEMORY_BARRIER + * Write barrier. Orders *only* stores. These are not + * allowed to be reordered over the barrier. + * - ERTS_THR_READ_MEMORY_BARRIER + * Read barrier. Orders *only* loads. These are not + * allowed to be reordered over the barrier. + * - ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER + * Data dependency read barrier. Orders *only* loads + * according to data dependency across the barrier. + * + * If thread support has been disabled, these barriers will become no-ops. + * + * If the prefix ERTS_THR_ is replaced with ERTS_SMP_, the barriers will + * be enabled only in the SMP enabled runtime system. + * + * --- Atomic operations --- + * + * Atomics operations exist for 32-bit, word size, and double word size + * integers. Function prototypes are listed below. + * + * Each function implementing an atomic operation exist with the following + * implied memory barrier semantics. Not all combinations are useful, but + * all of them exist for simplicity. <B> is suffix in function name: + * + * - <B> - Description + * + * - mb - Full memory barrier. Orders both loads, and + * stores before, and after the atomic operation. + * No load or store is allowed to be reordered + * over the atomic operation. + * - relb - Release barrier. Orders both loads, and + * stores appearing *before* the atomic + * operation. These are not allowed to be + * reordered over the atomic operation. + * - acqb - Acquire barrier. Orders both loads, and stores + * appearing *after* the atomic operation. These + * are not allowed to be reordered over the + * atomic operation. + * - wb - Write barrier. Orders *only* stores. These are + * not allowed to be reordered over the barrier. + * Store in atomic operation is ordered *after* + * the barrier. + * - rb - Read barrier. Orders *only* loads. These are + * not allowed to be reordered over the barrier. + * Load in atomic operation is ordered *before* + * the barrier. + * - ddrb - Data dependency read barrier. Orders *only* + * loads according to data dependency across the + * barrier. Load in atomic operation is ordered + * before the barrier. + * + * If thread support has been disabled, these functions are mapped to + * functions that performs the same operation, but aren't atomic + * and don't imply any memory barriers. + * + * If the atomic operations are prefixed with erts_smp_ instead of only + * erts_ the atomic operations will only be atomic in the SMP enabled + * runtime system, and will be mapped to non-atomic operations without + * memory barriers in the runtime system without SMP support. Atomic + * operations with erts_smp_ prefix should use the atomic types + * erts_smp_atomic32_t, erts_smp_atomic_t, and erts_smp_dw_atomic_t + * instead of erts_atomic32_t, erts_atomic_t, and erts_dw_atomic_t. The + * integer data types erts_aint32_t, erts_aint_t, and erts_dw_atomic_t + * are the same. + * + * --- 32-bit atomic operations --- + * + * The following 32-bit atomic operations exist. <B> should be + * replaced with a supported memory barrier (see above). Note + * that sizeof(erts_atomic32_t) might be larger than 4! + * + * + * Initialize (not necessarily the same as the set operation): + * void erts_atomic32_init_<B>(erts_atomic32_t *atmc, + * erts_aint32_t val); + * + * Set value: + * void erts_atomic32_set_<B>(erts_atomic32_t *atmc, + * erts_aint32_t val); + * + * Read; returns current value: + * erts_aint32_t erts_atomic32_read_<B>(erts_atomic32_t *atmc); + * + * Increment; returns resulting value: + * erts_aint32_t erts_atomic32_inc_read_<B>(erts_atomic32_t *atmc); + * + * Decrement; returns resulting value: + * erts_aint32_t erts_atomic32_dec_read_<B>(erts_atomic32_t *atmc); + * + * Increment: + * void erts_atomic32_inc_<B>(erts_atomic32_t *atmc); + * + * Decrement: + * void erts_atomic32_dec_<B>(erts_atomic32_t *atmc); + * + * Add value; returns resulting value: + * erts_aint32_t erts_atomic32_add_read_<B>(erts_atomic32_t *atmc, + * erts_aint32_t val); + * + * Add value: + * void erts_atomic32_add_<B>(erts_atomic32_t *atmc, + * erts_aint32_t val); + * + * Bitwise-or; returns previous value: + * erts_aint32_t erts_atomic32_read_bor_<B>(erts_atomic32_t *atmc, + * erts_aint32_t val); + * + * Bitwise-and; returns previous value: + * erts_aint32_t erts_atomic32_read_band_<B>(erts_atomic32_t *atmc, + * erts_aint32_t val); + * + * Exchange; returns previous value: + * erts_aint32_t erts_atomic32_xchg_<B>(erts_atomic32_t *atmc, + * erts_aint32_t val); + * + * Compare and exchange; returns previous or current value. If + * returned value equals 'exp' the value was changed to 'new'; + * otherwise not: + * erts_aint32_t erts_atomic32_cmpxchg_<B>(erts_atomic32_t *a, + * erts_aint32_t new, + * erts_aint32_t exp); + * + * --- Word size atomic operations --- + * + * The following word size (same size as sizeof(void *)) atomic + * operations exist. <B> should be replaced with a supported + * memory barrier (see above). Note that sizeof(erts_atomic_t) + * might be larger than sizeof(void *)! + * + * Initialize (not necessarily the same as the set operation): + * void erts_atomic_init_<B>(erts_atomic_t *atmc, + * erts_aint_t val); + * + * Set value; + * void erts_atomic_set_<B>(erts_atomic_t *atmc, + * erts_aint_t val); + * + * Read; returns current value: + * erts_aint_t erts_atomic_read_<B>(erts_atomic_t *atmc); + * + * Increment; returns resulting value: + * erts_aint_t erts_atomic_inc_read_<B>(erts_atomic_t *atmc); + * + * Decrement; returns resulting value: + * erts_aint_t erts_atomic_dec_read_<B>(erts_atomic_t *atmc); + * + * Increment: + * void erts_atomic_inc_<B>(erts_atomic_t *atmc); + * + * Decrement: + * void erts_atomic_dec_<B>(erts_atomic_t *atmc); + * + * Add value; returns resulting value: + * erts_aint_t erts_atomic_add_read_<B>(erts_atomic_t *atmc, + * erts_aint_t val); + * + * Add value: + * void erts_atomic_add_<B>(erts_atomic_t *atmc, + * erts_aint_t val); + * + * Bitwise-or; returns previous value: + * erts_aint_t erts_atomic_read_bor_<B>(erts_atomic_t *atmc, + * erts_aint_t val); + * + * Bitwise-and; returns previous value: + * erts_aint_t erts_atomic_read_band_<B>(erts_atomic_t *atmc, + * erts_aint_t val); + * + * Exchange; returns previous value: + * erts_aint_t erts_atomic_xchg_<B>(erts_atomic_t *atmc, + * erts_aint_t val); + * + * Compare and exchange; returns previous or current value. If + * returned value equals 'exp' the value was changed to 'new'; + * otherwise not: + * erts_aint_t erts_atomic_cmpxchg_<B>(erts_atomic_t *a, + * erts_aint_t new, + * erts_aint_t exp); + * + * --- Double word size atomic operations --- + * + * The following double word atomic operations exist. <B> should be + * replaced with a supported memory barrier (see above). + * + * Note that sizeof(erts_dw_atomic_t) usually is larger than + * 2*sizeof(void *)! + * + * The erts_dw_aint_t data type should be accessed as if it was defined + * like this: + * + * typedef struct { + * erts_aint_t sint[2]; + * } erts_dw_aint_t; + * + * Most significant word is 'sint[ERTS_DW_AINT_HIGH_WORD]' and least + * significant word is 'sint[ERTS_DW_AINT_LOW_WORD]'. + * + * + * Initialize (not necessarily the same as the set operation): + * void erts_dw_atomic_init_<B>(erts_dw_atomic_t *var, + * erts_dw_aint_t *val); + * + * Set; value is written into 'val': + * void erts_dw_atomic_set_<B>(erts_dw_atomic_t *var, + * erts_dw_aint_t *val); + * + * Read; value is written into 'val': + * void erts_dw_atomic_read_<B>(erts_dw_atomic_t *var, + * erts_dw_aint_t *val); + * + * Compare and exchange; returns a value != 0 if exchange was + * made; otherwise 0. 'new_val' contains new value to set. If 'exp_act' + * contains the same value as in memory when the function is called, + * 'new' is written to memory; otherwise, not. If exchange was not + * made, 'exp_act' contains the actual value in memory: + * int erts_dw_atomic_cmpxchg_<B>(erts_dw_atomic_t *var, + * erts_dw_aint_t *new, + * erts_dw_aint_t *exp_act); + */ + #define ERTS_SPIN_BODY ETHR_SPIN_BODY #include "sys.h" @@ -52,6 +281,9 @@ typedef Sint32 erts_no_atomic32_t; #endif #define ERTS_THR_MEMORY_BARRIER ETHR_MEMORY_BARRIER +#define ERTS_THR_WRITE_MEMORY_BARRIER ETHR_WRITE_MEMORY_BARRIER +#define ERTS_THR_READ_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER +#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER ETHR_READ_DEPEND_MEMORY_BARRIER #ifdef ERTS_ENABLE_LOCK_COUNT #define erts_mtx_lock(L) erts_mtx_lock_x(L, __FILE__, __LINE__) @@ -113,6 +345,9 @@ typedef ethr_ts_event erts_tse_t; #define erts_aint32_t ethr_sint32_t #define erts_atomic32_t ethr_atomic32_t +#define ERTS_DW_AINT_HIGH_WORD ETHR_DW_SINT_HIGH_WORD +#define ERTS_DW_AINT_LOW_WORD ETHR_DW_SINT_LOW_WORD + /* spinlock */ typedef struct { ethr_spinlock_t slck; @@ -149,6 +384,9 @@ __decl_noreturn void __noreturn erts_thr_fatal_error(int, char *); #else /* #ifdef USE_THREADS */ #define ERTS_THR_MEMORY_BARRIER +#define ERTS_THR_WRITE_MEMORY_BARRIER +#define ERTS_THR_READ_MEMORY_BARRIER +#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER #define ERTS_THR_OPTS_DEFAULT_INITER 0 typedef int erts_thr_opts_t; @@ -361,18 +599,13 @@ ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set, ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #endif /* #ifdef HAVE_ETHR_SIG_FUNCS */ +#ifdef USE_THREADS + /* - * Functions implementing atomic operations with with no (nob), - * full (mb), acquire (acqb), release (relb), read (rb), and - * write (wb) memory barriers. - * - * If thread support has been disabled, they are mapped to - * functions that performs the same operation, but aren't atomic - * and don't imply memory barriers. + * See "Documentation of atomics and memory barriers" at the top + * of this file for info on atomics. */ -#ifdef USE_THREADS - /* Double word size atomics */ #define erts_dw_atomic_init_nob ethr_dw_atomic_init @@ -395,6 +628,11 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_dw_atomic_read_relb ethr_dw_atomic_read_relb #define erts_dw_atomic_cmpxchg_relb ethr_dw_atomic_cmpxchg_relb +#define erts_dw_atomic_init_ddrb ethr_dw_atomic_init_ddrb +#define erts_dw_atomic_set_ddrb ethr_dw_atomic_set_ddrb +#define erts_dw_atomic_read_ddrb ethr_dw_atomic_read_ddrb +#define erts_dw_atomic_cmpxchg_ddrb ethr_dw_atomic_cmpxchg_ddrb + #define erts_dw_atomic_init_rb ethr_dw_atomic_init_rb #define erts_dw_atomic_set_rb ethr_dw_atomic_set_rb #define erts_dw_atomic_read_rb ethr_dw_atomic_read_rb @@ -463,6 +701,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_xchg_relb ethr_atomic_xchg_relb #define erts_atomic_cmpxchg_relb ethr_atomic_cmpxchg_relb +#define erts_atomic_init_ddrb ethr_atomic_init_ddrb +#define erts_atomic_set_ddrb ethr_atomic_set_ddrb +#define erts_atomic_read_ddrb ethr_atomic_read_ddrb +#define erts_atomic_inc_read_ddrb ethr_atomic_inc_read_ddrb +#define erts_atomic_dec_read_ddrb ethr_atomic_dec_read_ddrb +#define erts_atomic_inc_ddrb ethr_atomic_inc_ddrb +#define erts_atomic_dec_ddrb ethr_atomic_dec_ddrb +#define erts_atomic_add_read_ddrb ethr_atomic_add_read_ddrb +#define erts_atomic_add_ddrb ethr_atomic_add_ddrb +#define erts_atomic_read_bor_ddrb ethr_atomic_read_bor_ddrb +#define erts_atomic_read_band_ddrb ethr_atomic_read_band_ddrb +#define erts_atomic_xchg_ddrb ethr_atomic_xchg_ddrb +#define erts_atomic_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb + #define erts_atomic_init_rb ethr_atomic_init_rb #define erts_atomic_set_rb ethr_atomic_set_rb #define erts_atomic_read_rb ethr_atomic_read_rb @@ -549,6 +801,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_xchg_relb ethr_atomic32_xchg_relb #define erts_atomic32_cmpxchg_relb ethr_atomic32_cmpxchg_relb +#define erts_atomic32_init_ddrb ethr_atomic32_init_ddrb +#define erts_atomic32_set_ddrb ethr_atomic32_set_ddrb +#define erts_atomic32_read_ddrb ethr_atomic32_read_ddrb +#define erts_atomic32_inc_read_ddrb ethr_atomic32_inc_read_ddrb +#define erts_atomic32_dec_read_ddrb ethr_atomic32_dec_read_ddrb +#define erts_atomic32_inc_ddrb ethr_atomic32_inc_ddrb +#define erts_atomic32_dec_ddrb ethr_atomic32_dec_ddrb +#define erts_atomic32_add_read_ddrb ethr_atomic32_add_read_ddrb +#define erts_atomic32_add_ddrb ethr_atomic32_add_ddrb +#define erts_atomic32_read_bor_ddrb ethr_atomic32_read_bor_ddrb +#define erts_atomic32_read_band_ddrb ethr_atomic32_read_band_ddrb +#define erts_atomic32_xchg_ddrb ethr_atomic32_xchg_ddrb +#define erts_atomic32_cmpxchg_ddrb ethr_atomic32_cmpxchg_ddrb + #define erts_atomic32_init_rb ethr_atomic32_init_rb #define erts_atomic32_set_rb ethr_atomic32_set_rb #define erts_atomic32_read_rb ethr_atomic32_read_rb @@ -601,6 +867,11 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_dw_atomic_read_relb erts_no_dw_atomic_read #define erts_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg +#define erts_dw_atomic_init_ddrb erts_no_dw_atomic_init +#define erts_dw_atomic_set_ddrb erts_no_dw_atomic_set +#define erts_dw_atomic_read_ddrb erts_no_dw_atomic_read +#define erts_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg + #define erts_dw_atomic_init_rb erts_no_dw_atomic_init #define erts_dw_atomic_set_rb erts_no_dw_atomic_set #define erts_dw_atomic_read_rb erts_no_dw_atomic_read @@ -669,6 +940,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_xchg_relb erts_no_atomic_xchg #define erts_atomic_cmpxchg_relb erts_no_atomic_cmpxchg +#define erts_atomic_init_ddrb erts_no_atomic_set +#define erts_atomic_set_ddrb erts_no_atomic_set +#define erts_atomic_read_ddrb erts_no_atomic_read +#define erts_atomic_inc_read_ddrb erts_no_atomic_inc_read +#define erts_atomic_dec_read_ddrb erts_no_atomic_dec_read +#define erts_atomic_inc_ddrb erts_no_atomic_inc +#define erts_atomic_dec_ddrb erts_no_atomic_dec +#define erts_atomic_add_read_ddrb erts_no_atomic_add_read +#define erts_atomic_add_ddrb erts_no_atomic_add +#define erts_atomic_read_bor_ddrb erts_no_atomic_read_bor +#define erts_atomic_read_band_ddrb erts_no_atomic_read_band +#define erts_atomic_xchg_ddrb erts_no_atomic_xchg +#define erts_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg + #define erts_atomic_init_rb erts_no_atomic_set #define erts_atomic_set_rb erts_no_atomic_set #define erts_atomic_read_rb erts_no_atomic_read @@ -755,6 +1040,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_xchg_relb erts_no_atomic32_xchg #define erts_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg +#define erts_atomic32_init_ddrb erts_no_atomic32_set +#define erts_atomic32_set_ddrb erts_no_atomic32_set +#define erts_atomic32_read_ddrb erts_no_atomic32_read +#define erts_atomic32_inc_read_ddrb erts_no_atomic32_inc_read +#define erts_atomic32_dec_read_ddrb erts_no_atomic32_dec_read +#define erts_atomic32_inc_ddrb erts_no_atomic32_inc +#define erts_atomic32_dec_ddrb erts_no_atomic32_dec +#define erts_atomic32_add_read_ddrb erts_no_atomic32_add_read +#define erts_atomic32_add_ddrb erts_no_atomic32_add +#define erts_atomic32_read_bor_ddrb erts_no_atomic32_read_bor +#define erts_atomic32_read_band_ddrb erts_no_atomic32_read_band +#define erts_atomic32_xchg_ddrb erts_no_atomic32_xchg +#define erts_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg + #define erts_atomic32_init_rb erts_no_atomic32_set #define erts_atomic32_set_rb erts_no_atomic32_set #define erts_atomic32_read_rb erts_no_atomic32_read diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index b487dbf054..b1d1e1d9b0 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -544,7 +544,7 @@ send_to_port(Process *c_p, Eterm message, */ static void -profile_send(Eterm message) { +profile_send(Eterm from, Eterm message) { Uint sz = 0; ErlHeapFragment *bp = NULL; Uint *hp = NULL; @@ -554,6 +554,9 @@ profile_send(Eterm message) { Eterm profiler = erts_get_system_profile(); + /* do not profile profiler pid */ + if (from == profiler) return; + if (is_internal_port(profiler)) { Port *profiler_port = NULL; @@ -2617,7 +2620,7 @@ profile_scheduler(Eterm scheduler_id, Eterm state) { make_small(active_sched), timestamp); hp += 7; #ifndef ERTS_SMP - profile_send(msg); + profile_send(NIL, msg); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE #else @@ -2652,7 +2655,7 @@ profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint M timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4; msg = TUPLE6(hp, am_profile, am_scheduler, scheduler_id, state, no_schedulers, timestamp); hp += 7; #ifndef ERTS_SMP - profile_send(msg); + profile_send(NIL, msg); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE #else @@ -2919,11 +2922,11 @@ profile_runnable_port(Port *p, Eterm status) { msg = TUPLE5(hp, am_profile, p->id, status, count, timestamp); hp += 6; #ifndef ERTS_SMP - profile_send(msg); + profile_send(p->id, msg); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE #else - enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp); + enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, p->id, NIL, msg, bp); #endif erts_smp_mtx_unlock(&smq_mtx); } @@ -2972,11 +2975,11 @@ profile_runnable_proc(Process *p, Eterm status){ timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4; msg = TUPLE5(hp, am_profile, p->id, status, where, timestamp); hp += 6; #ifndef ERTS_SMP - profile_send(msg); + profile_send(p->id, msg); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE #else - enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp); + enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, p->id, NIL, msg, bp); #endif erts_smp_mtx_unlock(&smq_mtx); } diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 152dbcf085..9d52ed4e98 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -3118,6 +3118,9 @@ decoded_size(byte *ep, byte* endp, int internal_tags) case LARGE_BIG_EXT: CHKSIZE(4); n = get_int32(ep); + if (n > BIG_ARITY_MAX*sizeof(ErtsDigit)) { + return -1; + } SKIP2(n,4+1); /* skip, size,sign,digits */ heap_size += 1+1+(n+sizeof(Eterm)-1)/sizeof(Eterm); /* XXX: 1 too much? */ break; diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index f5a58f9dac..fe1a7ba345 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -818,6 +818,11 @@ erts_smp_xports_unlock(Port *prt) #define SET_VEC(iov, bv, bin, ptr, len, vlen) do { \ (iov)->iov_base = (ptr); \ (iov)->iov_len = (len); \ + if (sizeof((iov)->iov_len) < sizeof(len) \ + /* Check if (len) overflowed (iov)->iov_len */ \ + && ((len) >> (sizeof((iov)->iov_len)*CHAR_BIT)) != 0) { \ + goto L_overflow; \ + } \ *(bv)++ = (bin); \ (iov)++; \ (vlen)++; \ @@ -1146,11 +1151,21 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) ivp[0].iov_len = 0; bvp[0] = NULL; ev.vsize = io_list_to_vec(list, ivp+1, bvp+1, cbin, blimit); + if (ev.vsize < 0) { + if (ivp != iv) { + erts_free(ERTS_ALC_T_TMP, (void *) ivp); + } + if (bvp != bv) { + erts_free(ERTS_ALC_T_TMP, (void *) bvp); + } + driver_free_binary(cbin); + goto bad_value; + } ev.vsize++; #if 0 /* This assertion may say something useful, but it can be falsified during the emulator test suites. */ - ASSERT((ev.vsize >= 0) && (ev.vsize == vsize)); + ASSERT(ev.vsize == vsize); #endif ev.size = size; /* total size */ ev.iov = ivp; @@ -3912,7 +3927,7 @@ int driver_pushqv(ErlDrvPort ix, ErlIOVec* vec, ErlDrvSizeT skip) ErlDrvSizeT driver_deq(ErlDrvPort ix, ErlDrvSizeT size) { ErlIOQueue* q = drvport2ioq(ix); - int len; + ErlDrvSizeT len; if ((q == NULL) || (q->size < size)) return -1; diff --git a/erts/emulator/beam/register.h b/erts/emulator/beam/register.h index 97bab3ab71..bf15e63554 100644 --- a/erts/emulator/beam/register.h +++ b/erts/emulator/beam/register.h @@ -41,7 +41,7 @@ struct port; typedef struct reg_proc { HashBucket bucket; /* MUST BE LOCATED AT TOP OF STRUCT!!! */ - Process *p; /* The process registerd (only one of this and + Process *p; /* The process registered (only one of this and 'pt' is non-NULL */ struct port *pt; /* The port registered */ Eterm name; /* Atom name */ diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c index b132991a3b..36ed108b76 100644 --- a/erts/emulator/drivers/common/efile_drv.c +++ b/erts/emulator/drivers/common/efile_drv.c @@ -1385,7 +1385,11 @@ static void invoke_writev(void *data) { size = d->c.writev.size; } - /* Copy the io vector to avoid locking the port que while writing */ + /* Copy the io vector to avoid locking the port que while writing, + * also, both we and efile_writev might/will change the SysIOVec + * when segmenting or due to partial write and we do not want to + * tamper with the actual queue that we get from driver_peekq + */ MUTEX_LOCK(d->c.writev.q_mtx); /* Lock before accessing the port queue */ iov0 = driver_peekq(d->c.writev.port, &iovlen); @@ -1424,7 +1428,7 @@ static void invoke_writev(void *data) { } else { d->result_ok = efile_writev(&d->errInfo, d->flags, (int) d->fd, - iov, iovcnt, size); + iov, iovcnt); } } else if (iovlen == 0) { d->result_ok = 1; diff --git a/erts/emulator/drivers/common/erl_efile.h b/erts/emulator/drivers/common/erl_efile.h index 3868b38137..69ad02633c 100644 --- a/erts/emulator/drivers/common/erl_efile.h +++ b/erts/emulator/drivers/common/erl_efile.h @@ -162,7 +162,7 @@ int efile_write_info(Efile_error* errInfo, Efile_info* pInfo, char *name); int efile_write(Efile_error* errInfo, int flags, int fd, char* buf, size_t count); int efile_writev(Efile_error* errInfo, int flags, int fd, - SysIOVec* iov, int iovcnt, size_t size); + SysIOVec* iov, int iovcnt); int efile_read(Efile_error* errInfo, int flags, int fd, char* buf, size_t count, size_t* pBytesRead); int efile_seek(Efile_error* errInfo, int fd, diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index 7cee4cb928..060d72199d 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -6200,6 +6200,7 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len) type = SCTP_DEFAULT_SEND_PARAM; arg_ptr = (char*) (&arg.sri); arg_sz = sizeof ( arg.sri); + VALGRIND_MAKE_MEM_DEFINED(arg_ptr, arg_sz); /*suppress "uninitialised bytes"*/ break; } case SCTP_OPT_EVENTS: @@ -10305,7 +10306,6 @@ static void packet_inet_command(ErlDrvData e, char* buf, ErlDrvSizeT len) cmsg.hdr.cmsg_level = IPPROTO_SCTP; cmsg.hdr.cmsg_type = SCTP_SNDRCV; cmsg.hdr.cmsg_len = CMSG_LEN(sizeof(*sri)); - VALGRIND_MAKE_MEM_DEFINED(&cmsg, (char*)sri - (char*)&cmsg); /*suppress padding as "uninitialised bytes"*/ data_len = (buf + len) - ptr; /* The whole msg. @@ -10319,6 +10319,7 @@ static void packet_inet_command(ErlDrvData e, char* buf, ErlDrvSizeT len) mhdr.msg_iovlen = 1; mhdr.msg_control = cmsg.ancd; /* For ancilary data */ mhdr.msg_controllen = cmsg.hdr.cmsg_len; + VALGRIND_MAKE_MEM_DEFINED(mhdr.msg_control, mhdr.msg_controllen); /*suppress "uninitialised bytes"*/ mhdr.msg_flags = 0; /* Not used with "sendmsg" */ /* Now do the actual sending. NB: "flags" in "sendmsg" itself are NOT diff --git a/erts/emulator/drivers/unix/unix_efile.c b/erts/emulator/drivers/unix/unix_efile.c index 7cf0a712ce..796843a735 100644 --- a/erts/emulator/drivers/unix/unix_efile.c +++ b/erts/emulator/drivers/unix/unix_efile.c @@ -1004,13 +1004,11 @@ efile_writev(Efile_error* errInfo, /* Where to return error codes */ * opened */ int fd, /* File descriptor to write to */ SysIOVec* iov, /* Vector of buffer structs. - * The structs are unchanged - * after the call */ - int iovcnt, /* Number of structs in vector */ - size_t size) /* Number of bytes to write */ + * The structs may be changed i.e. + * due to incomplete writes */ + int iovcnt) /* Number of structs in vector */ { int cnt = 0; /* Buffers so far written */ - int p = 0; /* Position in next buffer */ ASSERT(iovcnt >= 0); @@ -1021,66 +1019,47 @@ efile_writev(Efile_error* errInfo, /* Where to return error codes */ #endif while (cnt < iovcnt) { + if ((! iov[cnt].iov_base) || (iov[cnt].iov_len <= 0)) { + /* Empty buffer - skip */ + cnt++; + } else { /* Non-empty buffer */ + ssize_t w; /* Bytes written in this call */ #ifdef HAVE_WRITEV - int w; /* Bytes written in this call */ - int b = iovcnt - cnt; /* Buffers to write */ - if (b > MAXIOV) - b = MAXIOV; - if (iov[cnt].iov_base && iov[cnt].iov_len > 0) { - if (b == 1) { - /* Degenerated io vector */ - do { - w = write(fd, iov[cnt].iov_base + p, iov[cnt].iov_len - p); - } while (w < 0 && errno == EINTR); - } else { - /* Non-empty vector first. - * Adjust pos in first buffer in case of - * previous incomplete writev */ - iov[cnt].iov_base += p; - iov[cnt].iov_len -= p; + int b = iovcnt - cnt; /* Buffers to write */ + /* Use as many buffers as MAXIOV allows */ + if (b > MAXIOV) + b = MAXIOV; + if (b > 1) { do { w = writev(fd, &iov[cnt], b); } while (w < 0 && errno == EINTR); - iov[cnt].iov_base -= p; - iov[cnt].iov_len += p; - } - if (w < 0) - return check_error(-1, errInfo); - } else { - /* Empty vector first - skip */ - cnt++; - continue; - } - ASSERT(w >= 0); - /* Move forward to next vector to write */ - for (; cnt < iovcnt; cnt++) { - if (iov[cnt].iov_base && iov[cnt].iov_len > 0) { - if (w < iov[cnt].iov_len) - break; - else - w -= iov[cnt].iov_len; - } - } - ASSERT(w >= 0); - p = w > 0 ? w : 0; /* Skip p bytes next writev */ -#else /* #ifdef HAVE_WRITEV */ - if (iov[cnt].iov_base && iov[cnt].iov_len > 0) { - /* Non-empty vector */ - int w; /* Bytes written in this call */ - while (p < iov[cnt].iov_len) { - do { - w = write(fd, iov[cnt].iov_base + p, iov[cnt].iov_len - p); - } while (w < 0 && errno == EINTR); - if (w < 0) - return check_error(-1, errInfo); - p += w; + } else + /* Degenerated io vector - use regular write */ +#endif + { + do { + w = write(fd, iov[cnt].iov_base, iov[cnt].iov_len); + } while (w < 0 && errno == EINTR); + ASSERT(w <= iov[cnt].iov_len); + } + if (w < 0) return check_error(-1, errInfo); + /* Move forward to next buffer to write */ + for (; cnt < iovcnt && w > 0; cnt++) { + if (iov[cnt].iov_base && iov[cnt].iov_len > 0) { + if (w < iov[cnt].iov_len) { + /* Adjust the buffer for next write */ + iov[cnt].iov_len -= w; + iov[cnt].iov_base += w; + w = 0; + break; + } else { + w -= iov[cnt].iov_len; + } + } } - } - cnt++; - p = 0; -#endif /* #ifdef HAVE_WRITEV */ + ASSERT(w == 0); + } /* else Non-empty buffer */ } /* while (cnt< iovcnt) */ - size = 0; /* Avoid compiler warning */ return 1; } @@ -1427,10 +1406,9 @@ efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, } #ifdef HAVE_SENDFILE - // For some reason the maximum size_t cannot be used as the max size // 3GB seems to work on all platforms -#define SENDFILE_CHUNK_SIZE ((1 << 30) -1) +#define SENDFILE_CHUNK_SIZE ((1UL << 30) -1) /* * sendfile: The implementation of the sendfile system call varies @@ -1467,7 +1445,7 @@ efile_sendfile(Efile_error* errInfo, int in_fd, int out_fd, written += retval; *nbytes -= retval; } - } while (retval != -1 && retval == SENDFILE_CHUNK_SIZE); + } while (retval == SENDFILE_CHUNK_SIZE); *nbytes = written; return check_error(retval == -1 ? -1 : 0, errInfo); #elif defined(__sun) && defined(__SVR4) && defined(HAVE_SENDFILEV) diff --git a/erts/emulator/drivers/win32/win_efile.c b/erts/emulator/drivers/win32/win_efile.c index 0d3d334154..606fa1d7de 100644 --- a/erts/emulator/drivers/win32/win_efile.c +++ b/erts/emulator/drivers/win32/win_efile.c @@ -1115,8 +1115,7 @@ efile_writev(Efile_error* errInfo, /* Where to return error codes */ SysIOVec* iov, /* Vector of buffer structs. * The structs are unchanged * after the call */ - int iovcnt, /* Number of structs in vector */ - size_t size) /* Number of bytes to write */ + int iovcnt) /* Number of structs in vector */ { int cnt; /* Buffers so far written */ OVERLAPPED overlapped; diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index cec22b3836..28e4382835 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -174,8 +174,13 @@ static inline unsigned char *bytearray_lvalue(Eterm bin, Eterm idx) { Sint i; unsigned char *bytes; +#ifndef DEBUG + ERTS_DECLARE_DUMMY(Uint bitoffs); + ERTS_DECLARE_DUMMY(Uint bitsize); +#else Uint bitoffs; Uint bitsize; +#endif if (is_not_binary(bin) || is_not_small(idx) || @@ -235,9 +240,15 @@ BIF_RETTYPE hipe_bifs_bitarray_2(BIF_ALIST_2) BIF_RETTYPE hipe_bifs_bitarray_update_3(BIF_ALIST_3) { unsigned char *bytes, bytemask; - Uint bitoffs, bitsize; Uint bitnr, bytenr; int set; +#ifndef DEBUG + ERTS_DECLARE_DUMMY(Uint bitoffs); + ERTS_DECLARE_DUMMY(Uint bitsize); +#else + Uint bitoffs; + Uint bitsize; +#endif if (is_not_binary(BIF_ARG_1)) BIF_ERROR(BIF_P, BADARG); @@ -267,8 +278,15 @@ BIF_RETTYPE hipe_bifs_bitarray_update_3(BIF_ALIST_3) BIF_RETTYPE hipe_bifs_bitarray_sub_2(BIF_ALIST_2) { unsigned char *bytes, bytemask; - Uint bitoffs, bitsize; Uint bitnr, bytenr; +#ifndef DEBUG + ERTS_DECLARE_DUMMY(Uint bitoffs); + ERTS_DECLARE_DUMMY(Uint bitsize); +#else + Uint bitoffs; + Uint bitsize; +#endif + if (is_not_binary(BIF_ARG_1)) BIF_ERROR(BIF_P, BADARG); @@ -397,10 +415,15 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) Uint nrbytes; void *bytes; void *address; - Uint bitoffs; - Uint bitsize; Eterm trampolines; Eterm *hp; +#ifndef DEBUG + ERTS_DECLARE_DUMMY(Uint bitoffs); + ERTS_DECLARE_DUMMY(Uint bitsize); +#else + Uint bitoffs; + Uint bitsize; +#endif if (is_not_binary(BIF_ARG_1)) BIF_ERROR(BIF_P, BADARG); diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c index 0199dea99e..e0575c35ff 100644 --- a/erts/emulator/hipe/hipe_gc.c +++ b/erts/emulator/hipe/hipe_gc.c @@ -46,9 +46,14 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) char *src, *oh; Uint src_size, oh_size; + if (!p->hipe.nstack) { + ASSERT(!p->hipe.nsp && !p->hipe.nstend); + return n_htop; + } if (!nstack_walk_init_check(p)) return n_htop; + ASSERT(p->hipe.nsp && p->hipe.nstend); nsp = nstack_walk_nsp_begin(p); nsp_end = p->hipe.nstgraylim; if (nsp_end) @@ -136,9 +141,14 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) char *heap; Uint heap_size, mature_size; + if (!p->hipe.nstack) { + ASSERT(!p->hipe.nsp && !p->hipe.nstend); + return; + } if (!nstack_walk_init_check(p)) return; + ASSERT(p->hipe.nsp && p->hipe.nstend); nsp = nstack_walk_nsp_begin(p); nsp_end = p->hipe.nstgraylim; if (nsp_end) { diff --git a/erts/emulator/pcre/pcre.mk b/erts/emulator/pcre/pcre.mk index b752c11459..352137b341 100644 --- a/erts/emulator/pcre/pcre.mk +++ b/erts/emulator/pcre/pcre.mk @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2011. All Rights Reserved. +# Copyright Ericsson AB 2012. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -17,8 +17,6 @@ # %CopyrightEnd% # -ARFLAGS = rc - PCRE_O = \ pcre_latin_1_table.o \ pcre_compile.o \ @@ -43,7 +41,7 @@ pcre_xclass.o PCRE_OBJS = $(PCRE_O:%=$(PCRE_OBJDIR)/%) -GENINC = pcre/pcre_exec_loop_break_cases.inc +PCRE_GENINC = $(ERL_TOP)/erts/emulator/pcre/pcre_exec_loop_break_cases.inc PCRE_OBJDIR = $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE) @@ -61,12 +59,12 @@ endif $(PCRE_OBJDIR)/%.o: pcre/%.c $(CC) -c $(PCRE_CFLAGS) -o $@ $< -$(GENINC): pcre/pcre_exec.c +$(PCRE_GENINC): pcre/pcre_exec.c for x in `grep -n COST_CHK pcre/pcre_exec.c | grep -v 'COST_CHK(N)' | awk -F: '{print $$1}'`; \ do \ N=`expr $$x + 100`; \ echo "case $$N: goto L_LOOP_COUNT_$${x};"; \ - done > $(GENINC) + done > $(PCRE_GENINC) # Dependencies. @@ -79,7 +77,7 @@ $(PCRE_OBJDIR)/pcre_config.o: pcre/pcre_config.c pcre/pcre_internal.h \ $(PCRE_OBJDIR)/pcre_dfa_exec.o: pcre/pcre_dfa_exec.c pcre/pcre_internal.h \ pcre/local_config.h pcre/pcre.h pcre/ucp.h $(PCRE_OBJDIR)/pcre_exec.o: pcre/pcre_exec.c pcre/pcre_internal.h \ - pcre/local_config.h pcre/pcre.h pcre/ucp.h $(GENINC) + pcre/local_config.h pcre/pcre.h pcre/ucp.h $(PCRE_GENINC) $(PCRE_OBJDIR)/pcre_fullinfo.o: pcre/pcre_fullinfo.c pcre/pcre_internal.h \ pcre/local_config.h pcre/pcre.h pcre/ucp.h $(PCRE_OBJDIR)/pcre_get.o: pcre/pcre_get.c pcre/pcre_internal.h \ diff --git a/erts/emulator/test/call_trace_SUITE.erl b/erts/emulator/test/call_trace_SUITE.erl index 3e2bee06d1..7030ebed3f 100644 --- a/erts/emulator/test/call_trace_SUITE.erl +++ b/erts/emulator/test/call_trace_SUITE.erl @@ -165,10 +165,14 @@ worker_loop() -> worker_foo(_Arg) -> ok. -basic(doc) -> - "Basic test of the call tracing (we trace one process)."; -basic(suite) -> []; -basic(Config) when is_list(Config) -> +%% Basic test of the call tracing (we trace one process). +basic(_Config) -> + case test_server:is_native(lists) of + true -> {skip,"lists is native"}; + false -> basic() + end. + +basic() -> ?line start_tracer(), ?line trace_info(self(), flags), ?line trace_info(self(), tracer), @@ -263,9 +267,15 @@ foo() -> foo0. foo(X) -> X+1. foo(X, Y) -> X+Y. -flags(doc) -> "Test flags (arity, timestamp) for call_trace/3. " - "Also, test the '{tracer,Pid}' option."; -flags(Config) when is_list(Config) -> +%% Test flags (arity, timestamp) for call_trace/3. +%% Also, test the '{tracer,Pid}' option. +flags(_Config) -> + case test_server:is_native(filename) of + true -> {skip,"filename is native"}; + false -> flags() + end. + +flags() -> ?line Tracer = start_tracer_loop(), ?line trace_pid(self(), true, [call,{tracer,Tracer}]), @@ -428,9 +438,14 @@ pam_foo(A, B) -> {ok,A,B}. -change_pam(doc) -> "Test changing PAM programs for a function."; -change_pam(suite) -> []; -change_pam(Config) when is_list(Config) -> +%% Test changing PAM programs for a function. +change_pam(_Config) -> + case test_server:is_native(lists) of + true -> {skip,"lists is native"}; + false -> change_pam() + end. + +change_pam() -> ?line start_tracer(), ?line Self = self(), @@ -468,10 +483,11 @@ change_pam_trace(Prog) -> {match_spec,Prog} = trace_info({erlang,process_info,2}, match_spec), ok. -return_trace(doc) -> "Test the new return trace."; -return_trace(suite) -> []; -return_trace(Config) when is_list(Config) -> - return_trace(). +return_trace(_Config) -> + case test_server:is_native(lists) of + true -> {skip,"lists is native"}; + false -> return_trace() + end. return_trace() -> X = {save,me}, @@ -521,7 +537,7 @@ return_trace() -> ?line {match_spec,Prog2} = trace_info({erlang,atom_to_list,1}, match_spec), ?line lists:seq(2, 7), - ?line atom_to_list(non_literal(nisse)), + ?line _ = atom_to_list(non_literal(nisse)), ?line expect({trace,Self,return_from,{lists,seq,2},[2,3,4,5,6,7]}), ?line expect({trace,Self,return_from,{erlang,atom_to_list,1},"nisse"}), @@ -539,10 +555,11 @@ return_trace() -> nasty() -> exit(good_bye). -exception_trace(doc) -> "Test the new exception trace."; -exception_trace(suite) -> []; -exception_trace(Config) when is_list(Config) -> - exception_trace(). +exception_trace(_Config) -> + case test_server:is_native(lists) of + true -> {skip,"lists is native"}; + false -> exception_trace() + end. exception_trace() -> X = {save,me}, @@ -600,7 +617,7 @@ exception_trace() -> trace_info({erlang,atom_to_list,1}, match_spec), ?line lists:seq(2, 7), - ?line atom_to_list(non_literal(nisse)), + ?line _ = atom_to_list(non_literal(nisse)), ?line expect({trace,Self,return_from,{lists,seq,2},[2,3,4,5,6,7]}), ?line expect({trace,Self,return_from,{erlang,atom_to_list,1},"nisse"}), diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index 370363bf9e..6bd7361612 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -859,7 +859,13 @@ resource_holder(Pid,Reply,List) -> threading(doc) -> ["Test the threading API functions (reuse tests from driver API)"]; -threading(Config) when is_list(Config) -> +threading(Config) when is_list(Config) -> + case erlang:system_info(threads) of + true -> threading_do(Config); + false -> {skipped,"No thread support"} + end. + +threading_do(Config) -> ?line Data = ?config(data_dir, Config), ?line File = filename:join(Data, "tester"), ?line {ok,tester,ModBin} = compile:file(File, [binary,return_errors]), diff --git a/erts/emulator/test/nif_SUITE_data/tester.c b/erts/emulator/test/nif_SUITE_data/tester.c index 08466d0f18..257b116322 100644 --- a/erts/emulator/test/nif_SUITE_data/tester.c +++ b/erts/emulator/test/nif_SUITE_data/tester.c @@ -61,6 +61,7 @@ static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) static ERL_NIF_TERM run(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { testcase_run(NULL); + testcase_cleanup(NULL); return enif_make_atom(env, "ok"); } diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl index 0392312a6f..a93dd309c1 100644 --- a/erts/emulator/test/statistics_SUITE.erl +++ b/erts/emulator/test/statistics_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2011. All Rights Reserved. +%% Copyright Ericsson AB 1997-2012. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,6 +29,7 @@ runtime_zero_diff/1, runtime_update/1, runtime_diff/1, run_queue_one/1, + scheduler_wall_time/1, reductions/1, reductions_big/1, garbage_collection/1, io/1, badarg/1]). @@ -51,8 +52,8 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [{group, wall_clock}, {group, runtime}, reductions, - reductions_big, {group, run_queue}, garbage_collection, - io, badarg]. + reductions_big, {group, run_queue}, scheduler_wall_time, + garbage_collection, io, badarg]. groups() -> [{wall_clock, [], @@ -266,11 +267,10 @@ run_queue_one(Config) when is_list(Config) -> run_queue_one_test(Config) when is_list(Config) -> - ?line Hog = spawn_link(?MODULE, hog, [self()]), + ?line _Hog = spawn_link(?MODULE, hog, [self()]), ?line receive - hog_started -> - Hog ! go - end, + hog_started -> ok + end, ?line receive after 100 -> ok end, % Give hog a head start. ?line case statistics(run_queue) of N when N >= 1 -> ok; @@ -280,18 +280,88 @@ run_queue_one_test(Config) when is_list(Config) -> %% CPU-bound process, going at low priority. It will always be ready %% to run. - + hog(Pid) -> ?line process_flag(priority, low), ?line Pid ! hog_started, - ?line receive - go -> hog_iter(0) + ?line Mon = erlang:monitor(process, Pid), + ?line hog_iter(0, Mon). + +hog_iter(N, Mon) when N > 0 -> + receive + {'DOWN', Mon, _, _, _} -> ok + after 0 -> + ?line hog_iter(N-1, Mon) + end; +hog_iter(0, Mon) -> + ?line hog_iter(10000, Mon). + +%%% Tests of statistics(scheduler_wall_time). + +scheduler_wall_time(doc) -> + "Tests that statistics(scheduler_wall_time) works as intended"; +scheduler_wall_time(Config) when is_list(Config) -> + %% Should return undefined if system_flag is not turned on yet + undefined = statistics(scheduler_wall_time), + %% Turn on statistics + false = erlang:system_flag(scheduler_wall_time, true), + try + Schedulers = erlang:system_info(schedulers_online), + %% Let testserver and everyone else finish their work + timer:sleep(500), + %% Empty load + EmptyLoad = get_load(), + {false, _} = {lists:any(fun(Load) -> Load > 50 end, EmptyLoad),EmptyLoad}, + MeMySelfAndI = self(), + StartHog = fun() -> + Pid = spawn(?MODULE, hog, [self()]), + receive hog_started -> MeMySelfAndI ! go end, + Pid + end, + P1 = StartHog(), + %% Max on one, the other schedulers empty (hopefully) + %% Be generous the process can jump between schedulers + %% which is ok and we don't want the test to fail for wrong reasons + _L1 = [S1Load|EmptyScheds1] = get_load(), + {true,_} = {S1Load > 50,S1Load}, + {false,_} = {lists:any(fun(Load) -> Load > 50 end, EmptyScheds1),EmptyScheds1}, + {true,_} = {lists:sum(EmptyScheds1) < 60,EmptyScheds1}, + + %% 50% load + HalfHogs = [StartHog() || _ <- lists:seq(1, (Schedulers-1) div 2)], + HalfLoad = lists:sum(get_load()) div Schedulers, + if Schedulers < 2, HalfLoad > 80 -> ok; %% Ok only one scheduler online and one hog + %% We want roughly 50% load + HalfLoad > 40, HalfLoad < 60 -> ok; + true -> exit({halfload, HalfLoad}) + end, + + %% 100% load + LastHogs = [StartHog() || _ <- lists:seq(1, Schedulers div 2)], + FullScheds = get_load(), + {false,_} = {lists:any(fun(Load) -> Load < 80 end, FullScheds),FullScheds}, + FullLoad = lists:sum(FullScheds) div Schedulers, + if FullLoad > 90 -> ok; + true -> exit({fullload, FullLoad}) + end, + + [exit(Pid, kill) || Pid <- [P1|HalfHogs++LastHogs]], + AfterLoad = get_load(), + {false,_} = {lists:any(fun(Load) -> Load > 5 end, AfterLoad),AfterLoad}, + true = erlang:system_flag(scheduler_wall_time, false) + after + erlang:system_flag(scheduler_wall_time, false) end. -hog_iter(N) when N > 0 -> - ?line hog_iter(N-1); -hog_iter(0) -> - ?line hog_iter(10000). +get_load() -> + Start = erlang:statistics(scheduler_wall_time), + timer:sleep(500), + End = erlang:statistics(scheduler_wall_time), + lists:reverse(lists:sort(load_percentage(lists:sort(Start),lists:sort(End)))). + +load_percentage([{Id, WN, TN}|Ss], [{Id, WP, TP}|Ps]) -> + [100*(WN-WP) div (TN-TP)|load_percentage(Ss, Ps)]; +load_percentage([], []) -> []. garbage_collection(doc) -> diff --git a/erts/emulator/test/system_profile_SUITE.erl b/erts/emulator/test/system_profile_SUITE.erl index 32089e8872..659e43f81d 100644 --- a/erts/emulator/test/system_profile_SUITE.erl +++ b/erts/emulator/test/system_profile_SUITE.erl @@ -27,6 +27,7 @@ system_profile_on_and_off/1, runnable_procs/1, runnable_ports/1, + dont_profile_profiler/1, scheduler/1 ]). @@ -40,7 +41,7 @@ -define(default_timeout, ?t:minutes(1)). init_per_testcase(_Case, Config) -> - ?line Dog=?t:timetrap(?default_timeout), + Dog=?t:timetrap(?default_timeout), [{watchdog, Dog}|Config]. end_per_testcase(_Case, Config) -> Dog=?config(watchdog, Config), @@ -51,7 +52,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [system_profile_on_and_off, runnable_procs, - runnable_ports, scheduler]. + runnable_ports, scheduler, dont_profile_profiler]. groups() -> []. @@ -77,31 +78,31 @@ system_profile_on_and_off(suite) -> system_profile_on_and_off(doc) -> ["Tests switching system_profiling on and off."]; system_profile_on_and_off(Config) when is_list(Config) -> - ?line Pid = start_profiler_process(), + Pid = start_profiler_process(), % Test runnable_ports on and off - ?line undefined = erlang:system_profile(Pid, [runnable_ports]), - ?line {Pid, [runnable_ports]} = erlang:system_profile(), - ?line {Pid, [runnable_ports]} = erlang:system_profile(undefined, []), + undefined = erlang:system_profile(Pid, [runnable_ports]), + {Pid, [runnable_ports]} = erlang:system_profile(), + {Pid, [runnable_ports]} = erlang:system_profile(undefined, []), % Test runnable_procs on and off - ?line undefined = erlang:system_profile(Pid, [runnable_procs]), - ?line {Pid, [runnable_procs]} = erlang:system_profile(), - ?line {Pid, [runnable_procs]} = erlang:system_profile(undefined, []), + undefined = erlang:system_profile(Pid, [runnable_procs]), + {Pid, [runnable_procs]} = erlang:system_profile(), + {Pid, [runnable_procs]} = erlang:system_profile(undefined, []), % Test scheduler on and off - ?line undefined = erlang:system_profile(Pid, [scheduler]), - ?line {Pid, [scheduler]} = erlang:system_profile(), - ?line {Pid, [scheduler]} = erlang:system_profile(undefined, []), + undefined = erlang:system_profile(Pid, [scheduler]), + {Pid, [scheduler]} = erlang:system_profile(), + {Pid, [scheduler]} = erlang:system_profile(undefined, []), % Test combined runnable_ports, runnable_procs, scheduler; on and off - ?line undefined = erlang:system_profile(Pid, [scheduler, runnable_procs, runnable_ports]), - ?line {Pid, [scheduler,runnable_procs,runnable_ports]} = erlang:system_profile(), - ?line {Pid, [scheduler,runnable_procs,runnable_ports]} = erlang:system_profile(undefined, []), + undefined = erlang:system_profile(Pid, [scheduler, runnable_procs, runnable_ports]), + {Pid, [scheduler,runnable_procs,runnable_ports]} = erlang:system_profile(), + {Pid, [scheduler,runnable_procs,runnable_ports]} = erlang:system_profile(undefined, []), % Test turned off and kill process - ?line undefined = erlang:system_profile(), - ?line exit(Pid,kill), + undefined = erlang:system_profile(), + exit(Pid,kill), ok. %% Test runnable_procs @@ -111,25 +112,25 @@ runnable_procs(suite) -> runnable_procs(doc) -> ["Tests system_profiling with runnable_procs."]; runnable_procs(Config) when is_list(Config) -> - ?line Pid = start_profiler_process(), + Pid = start_profiler_process(), % start a ring of processes % FIXME: Set #laps and #nodes in config file Nodes = 10, Laps = 10, - ?line Master = ring(Nodes), - ?line undefined = erlang:system_profile(Pid, [runnable_procs]), + Master = ring(Nodes), + undefined = erlang:system_profile(Pid, [runnable_procs]), % loop a message - ?line ok = ring_message(Master, message, Laps), - ?line Events = get_profiler_events(), - ?line kill_em_all = kill_ring(Master), - ?line erlang:system_profile(undefined, []), + ok = ring_message(Master, message, Laps), + Events = get_profiler_events(), + kill_em_all = kill_ring(Master), + erlang:system_profile(undefined, []), put(master, Master), put(laps, Laps), - ?line true = has_runnable_event(Events), + true = has_runnable_event(Events), Pids = sort_events_by_pid(Events), - ?line ok = check_events(Pids), + ok = check_events(Pids), erase(), - ?line exit(Pid,kill), + exit(Pid,kill), ok. runnable_ports(suite) -> @@ -137,21 +138,21 @@ runnable_ports(suite) -> runnable_ports(doc) -> ["Tests system_profiling with runnable_port."]; runnable_ports(Config) when is_list(Config) -> - ?line Pid = start_profiler_process(), - ?line undefined = erlang:system_profile(Pid, [runnable_ports]), - ?line EchoPid = echo(Config), + Pid = start_profiler_process(), + undefined = erlang:system_profile(Pid, [runnable_ports]), + EchoPid = echo(Config), % FIXME: Set config to number_of_echos Laps = 10, put(laps, Laps), - ?line ok = echo_message(EchoPid, Laps, message), - ?line Events = get_profiler_events(), - ?line kill_em_all = kill_echo(EchoPid), - ?line erlang:system_profile(undefined, []), - ?line true = has_runnable_event(Events), + ok = echo_message(EchoPid, Laps, message), + Events = get_profiler_events(), + kill_em_all = kill_echo(EchoPid), + erlang:system_profile(undefined, []), + true = has_runnable_event(Events), Pids = sort_events_by_pid(Events), - ?line ok = check_events(Pids), + ok = check_events(Pids), erase(), - ?line exit(Pid,kill), + exit(Pid,kill), ok. scheduler(suite) -> @@ -160,46 +161,68 @@ scheduler(doc) -> ["Tests system_profiling with scheduler."]; scheduler(Config) when is_list(Config) -> case {erlang:system_info(smp_support), erlang:system_info(schedulers_online)} of - {false,_} -> ?line {skipped, "No need for scheduler test when smp support is disabled."}; - {_, 1} -> ?line {skipped, "No need for scheduler test when only one scheduler online."}; + {false,_} -> {skipped, "No need for scheduler test when smp support is disabled."}; + {_, 1} -> {skipped, "No need for scheduler test when only one scheduler online."}; _ -> Nodes = 10, - ?line ok = check_block_system(Nodes), - ?line ok = check_multi_scheduling_block(Nodes), - ok + ok = check_block_system(Nodes), + ok = check_multi_scheduling_block(Nodes) end. +% the profiler pid should not be profiled +dont_profile_profiler(suite) -> + []; +dont_profile_profiler(doc) -> + ["Ensure system profiler process is not profiled."]; +dont_profile_profiler(Config) when is_list(Config) -> + Pid = start_profiler_process(), + + Nodes = 10, + Laps = 10, + Master = ring(Nodes), + undefined = erlang:system_profile(Pid, [runnable_procs]), + % loop a message + ok = ring_message(Master, message, Laps), + erlang:system_profile(undefined, []), + kill_em_all = kill_ring(Master), + Events = get_profiler_events(), + false = has_profiler_pid_event(Events, Pid), + + exit(Pid,kill), + ok. + + %%% Check scheduler profiling check_multi_scheduling_block(Nodes) -> - ?line Pid = start_profiler_process(), - ?line undefined = erlang:system_profile(Pid, [scheduler]), - ?line {ok, Supervisor} = start_load(Nodes), - ?line erlang:system_flag(multi_scheduling, block), - ?line erlang:system_flag(multi_scheduling, unblock), - ?line {Pid, [scheduler]} = erlang:system_profile(undefined, []), - ?line Events = get_profiler_events(), - ?line true = has_scheduler_event(Events), + Pid = start_profiler_process(), + undefined = erlang:system_profile(Pid, [scheduler]), + {ok, Supervisor} = start_load(Nodes), + erlang:system_flag(multi_scheduling, block), + erlang:system_flag(multi_scheduling, unblock), + {Pid, [scheduler]} = erlang:system_profile(undefined, []), + Events = get_profiler_events(), + true = has_scheduler_event(Events), stop_load(Supervisor), - ?line exit(Pid,kill), + exit(Pid,kill), erase(), ok. check_block_system(Nodes) -> - ?line Dummy = spawn(?MODULE, profiler_process, [[]]), - ?line Pid = start_profiler_process(), - ?line undefined = erlang:system_profile(Pid, [scheduler]), - ?line {ok, Supervisor} = start_load(Nodes), + Dummy = spawn(?MODULE, profiler_process, [[]]), + Pid = start_profiler_process(), + undefined = erlang:system_profile(Pid, [scheduler]), + {ok, Supervisor} = start_load(Nodes), % FIXME: remove wait !! wait(300), - ?line undefined = erlang:system_monitor(Dummy, [busy_port]), - ?line {Dummy, [busy_port]} = erlang:system_monitor(undefined, []), - ?line {Pid, [scheduler]} = erlang:system_profile(undefined, []), - ?line Events = get_profiler_events(), - ?line true = has_scheduler_event(Events), + undefined = erlang:system_monitor(Dummy, [busy_port]), + {Dummy, [busy_port]} = erlang:system_monitor(undefined, []), + {Pid, [scheduler]} = erlang:system_profile(undefined, []), + Events = get_profiler_events(), + true = has_scheduler_event(Events), stop_load(Supervisor), - ?line exit(Pid,kill), - ?line exit(Dummy,kill), + exit(Pid,kill), + exit(Dummy,kill), erase(), ok. @@ -211,17 +234,17 @@ check_events([Pid | Pids]) -> Laps = get(laps), CheckPids = get(pids), {Events, N} = get_pid_events(Pid), - ?line ok = check_event_flow(Events), - ?line ok = check_event_ts(Events), + ok = check_event_flow(Events), + ok = check_event_ts(Events), IsMember = lists:member(Pid, CheckPids), case Pid of Master -> io:format("Expected ~p and got ~p profile events from ~p: ok~n", [Laps*2+2, N, Pid]), - ?line N = Laps*2 + 2, + N = Laps*2 + 2, check_events(Pids); Pid when IsMember == true -> io:format("Expected ~p and got ~p profile events from ~p: ok~n", [Laps*2, N, Pid]), - ?line N = Laps*2, + N = Laps*2, check_events(Pids); Pid -> check_events(Pids) @@ -448,6 +471,12 @@ has_runnable_event(Events) -> end end, Events). +has_profiler_pid_event([], _) -> false; +has_profiler_pid_event([{profile, Pid, _Activity, _MFA, _TS}|Events], Pid) -> true; +has_profiler_pid_event([_|Events], Pid) -> + has_profiler_pid_event(Events, Pid). + + wait(Time) -> receive after Time -> ok end. %%% diff --git a/erts/emulator/test/trace_port_SUITE.erl b/erts/emulator/test/trace_port_SUITE.erl index 0026da4979..b0ce6f81db 100644 --- a/erts/emulator/test/trace_port_SUITE.erl +++ b/erts/emulator/test/trace_port_SUITE.erl @@ -77,7 +77,8 @@ end_per_testcase(_Func, Config) -> call_trace(doc) -> "Test sending call trace messages to a port."; call_trace(Config) when is_list(Config) -> - case test_server:is_native(?MODULE) of + case test_server:is_native(?MODULE) orelse + test_server:is_native(lists) of true -> {skip,"Native code"}; false -> @@ -128,7 +129,8 @@ bs_sum_c(<<>>, Acc) -> Acc. return_trace(doc) -> "Test the new return trace."; return_trace(Config) when is_list(Config) -> - case test_server:is_native(?MODULE) of + case test_server:is_native(?MODULE) orelse + test_server:is_native(lists) of true -> {skip,"Native code"}; false -> diff --git a/erts/emulator/zlib/Makefile b/erts/emulator/zlib/Makefile deleted file mode 100644 index def8e1aa47..0000000000 --- a/erts/emulator/zlib/Makefile +++ /dev/null @@ -1,23 +0,0 @@ -# -# %CopyrightBegin% -# -# Copyright Ericsson AB 1997-2009. All Rights Reserved. -# -# The contents of this file are subject to the Erlang Public License, -# Version 1.1, (the "License"); you may not use this file except in -# compliance with the License. You should have received a copy of the -# Erlang Public License along with this software. If not, it can be -# retrieved online at http://www.erlang.org/. -# -# Software distributed under the License is distributed on an "AS IS" -# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -# the License for the specific language governing rights and limitations -# under the License. -# -# %CopyrightEnd% -# -# -# Invoke with GNU make or clearmake -C gnu. -# - -include $(ERL_TOP)/make/run_make.mk diff --git a/erts/emulator/zlib/Makefile.in b/erts/emulator/zlib/Makefile.in deleted file mode 100644 index b44a87551d..0000000000 --- a/erts/emulator/zlib/Makefile.in +++ /dev/null @@ -1,102 +0,0 @@ -# Makefile for zlib -# Copyright (C) 1995-1996 Jean-loup Gailly. -# For conditions of distribution and use, see copyright notice in zlib.h - -# %ExternalCopyright% - -# To compile and test, type: -# ./configure; make test -# The call of configure is optional if you don't have special requirements - -# To install /usr/local/lib/libz.* and /usr/local/include/zlib.h, type: -# make install -# To install in $HOME instead of /usr/local, use: -# make install prefix=$HOME - -ARFLAGS = rc -CFLAGS = $(subst -O2, -O3, @CFLAGS@ @DEFS@ @EMU_THR_DEFS@) -#CFLAGS=-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7 -#CFLAGS=-g -DDEBUG -#CFLAGS=-O3 -Wall -Wwrite-strings -Wpointer-arith -Wconversion \ -# -Wstrict-prototypes -Wmissing-prototypes - -VER=1.0.4 - -O = adler32.o compress.o crc32.o uncompr.o deflate.o trees.o \ - zutil.o inflate.o inftrees.o inffast.o -OBJS = $(O:%=$(OBJDIR)/%) - - -#### Begin OTP targets - -include $(ERL_TOP)/make/target.mk - -ifeq ($(TYPE),gcov) -CFLAGS = -O0 -fprofile-arcs -ftest-coverage @DEBUG_CFLAGS@ @DEFS@ @EMU_THR_DEFS@ -else # gcov -ifeq ($(TYPE),debug) -CFLAGS = @DEBUG_CFLAGS@ @DEFS@ @EMU_THR_DEFS@ -endif # debug -endif # gcov - -# On windows we *need* a separate zlib during debug build -OBJDIR= $(ERL_TOP)/erts/emulator/zlib/obj/$(TARGET)/$(TYPE) - -include $(ERL_TOP)/make/$(TARGET)/otp.mk - -ifeq ($(TARGET), win32) -LIBRARY=$(OBJDIR)/z.lib -else -LIBRARY=$(OBJDIR)/libz.a -endif - -all: $(LIBRARY) - -# ---------------------------------------------------- -# Release Target -# ---------------------------------------------------- -include $(ERL_TOP)/make/otp_release_targets.mk - -release_spec: opt - -tests release_tests: - -docs release_docs release_docs_spec: - -clean: - rm -f $(OBJS) $(OBJDIR)/libz.a - -#### end OTP targets - -ifeq ($(TARGET), win32) -$(LIBRARY): $(OBJS) - $(AR) -out:$@ $(OBJS) -else -$(LIBRARY): $(OBJS) - $(AR) $(ARFLAGS) $@ $(OBJS) - -@ ($(RANLIB) $@ || true) 2>/dev/null -endif - -$(OBJDIR)/%.o: %.c - $(CC) -c $(CFLAGS) -o $@ $< - -# DO NOT DELETE THIS LINE -- make depend depends on it. - -adler32.o: zlib.h zconf.h -compress.o: zlib.h zconf.h -crc32.o: zlib.h zconf.h -deflate.o: deflate.h zutil.h zlib.h zconf.h -example.o: zlib.h zconf.h -gzio.o: zutil.h zlib.h zconf.h -infblock.o: infblock.h inftrees.h infcodes.h infutil.h zutil.h zlib.h zconf.h -infcodes.o: zutil.h zlib.h zconf.h -infcodes.o: inftrees.h infblock.h infcodes.h infutil.h inffast.h -inffast.o: zutil.h zlib.h zconf.h inftrees.h -inffast.o: infblock.h infcodes.h infutil.h inffast.h -inflate.o: zutil.h zlib.h zconf.h infblock.h -inftrees.o: zutil.h zlib.h zconf.h inftrees.h -infutil.o: zutil.h zlib.h zconf.h infblock.h inftrees.h infcodes.h infutil.h -minigzip.o: zlib.h zconf.h -trees.o: deflate.h zutil.h zlib.h zconf.h -uncompr.o: zlib.h zconf.h -zutil.o: zutil.h zlib.h zconf.h diff --git a/erts/emulator/zlib/zlib.mk b/erts/emulator/zlib/zlib.mk new file mode 100644 index 0000000000..fa1f159fae --- /dev/null +++ b/erts/emulator/zlib/zlib.mk @@ -0,0 +1,74 @@ +#-*-makefile-*- ; force emacs to enter makefile-mode +# ---------------------------------------------------- +# Make include file for zlib +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 2011-2012. All Rights Reserved. +# +# The contents of this file are subject to the Erlang Public License, +# Version 1.1, (the "License"); you may not use this file except in +# compliance with the License. You should have received a copy of the +# Erlang Public License along with this software. If not, it can be +# retrieved online at http://www.erlang.org/. +# +# Software distributed under the License is distributed on an "AS IS" +# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +# the License for the specific language governing rights and limitations +# under the License. +# +# %CopyrightEnd% +# +# ---------------------------------------------------- +# Copyright for zlib itself see copyright notice in zlib.h + +ZLIB_FILES = \ + adler32 \ + compress \ + crc32 \ + uncompr \ + deflate \ + trees \ + zutil \ + inflate \ + inftrees \ + inffast + +# On windows we *need* a separate zlib during debug build +ZLIB_OBJDIR = $(ERL_TOP)/erts/emulator/zlib/obj/$(TARGET)/$(TYPE) + +ZLIB_OBJS = $(ZLIB_FILES:%=$(ZLIB_OBJDIR)/%.o) +ZLIB_SRC = $(ZLIB_FILES:%=zlib/%.c) + +ifeq ($(TARGET), win32) +ZLIB_LIBRARY = $(ZLIB_OBJDIR)/z.lib +else +ZLIB_LIBRARY = $(ZLIB_OBJDIR)/libz.a +endif + + +ifeq ($(TYPE),gcov) +ZLIB_CFLAGS = -O0 -fprofile-arcs -ftest-coverage $(DEBUG_CFLAGS) $(DEFS) $(THR_DEFS) +else # gcov +ifeq ($(TYPE),debug) +ZLIB_CFLAGS = $(DEBUG_CFLAGS) $(DEFS) $(THR_DEFS) +else # debug +ZLIB_CFLAGS = $(subst -O2, -O3, $(CONFIGURE_CFLAGS) $(DEFS) $(THR_DEFS)) +#ZLIB_CFLAGS=-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7 +#ZLIB_CFLAGS=-g -DDEBUG +#ZLIB_CFLAGS=-O3 -Wall -Wwrite-strings -Wpointer-arith -Wconversion \ +# -Wstrict-prototypes -Wmissing-prototypes +endif # debug +endif # gcov + +ifeq ($(TARGET), win32) +$(ZLIB_LIBRARY): $(ZLIB_OBJS) + $(AR) -out:$@ $(ZLIB_OBJS) +else +$(ZLIB_LIBRARY): $(ZLIB_OBJS) + $(AR) $(ARFLAGS) $@ $(ZLIB_OBJS) + -@ ($(RANLIB) $@ || true) 2>/dev/null +endif + +$(ZLIB_OBJDIR)/%.o: zlib/%.c + $(CC) -c $(ZLIB_CFLAGS) -o $@ $< |