diff options
Diffstat (limited to 'erts/emulator')
168 files changed, 16288 insertions, 9830 deletions
diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index a81ab28847..ed0d1b3fa6 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -61,11 +61,7 @@ ifeq ($(TYPE),quantify) PURIFY = quantify $(QUANTIFY_BUILD_OPTIONS) TYPEMARKER = .quantify ENABLE_ALLOC_TYPE_VARS += quantify -ifeq ($(findstring ose,$(TARGET)),ose) - TYPE_FLAGS = @CFLAGS@ -DQUANTIFY -else TYPE_FLAGS = @CFLAGS@ -g -O2 -DQUANTIFY -DNO_JUMP_TABLE -endif else ifeq ($(TYPE),purecov) @@ -203,6 +199,14 @@ MKDIR = @MKDIR@ USING_MINGW=@MIXED_CYGWIN_MINGW@ +ifeq ($(TARGET),win32) +LIB_PREFIX= +LIB_SUFFIX=.lib +else +LIB_PREFIX=lib +LIB_SUFFIX=.a +endif + OMIT_OMIT_FP=no ifeq (@EMU_LOCK_CHECKING@,yes) @@ -283,16 +287,11 @@ endif ifeq ($(TARGET),win32) LIBS += -L$(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE) -lepcre -DEPLIBS += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/epcre.lib else -LIBS += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/libepcre.a -DEPLIBS += \ - $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/libepcre.a \ - $(ERL_TOP)/erts/lib/internal/$(TARGET)/liberts_internal.a -# rem liberts_internal.a +LIBS += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/$(LIB_PREFIX)epcre$(LIB_SUFFIX) endif -ELIB_FLAGS = -DENABLE_ELIB_MALLOC -DELIB_ALLOC_IS_CLIB -DELIB_HEAP_SBRK +DEPLIBS += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/$(LIB_PREFIX)epcre$(LIB_SUFFIX) PERFCTR_PATH=@PERFCTR_PATH@ USE_PERFCTR=@USE_PERFCTR@ @@ -309,7 +308,19 @@ LIBSCTP = @LIBSCTP@ ORG_THR_LIBS=@EMU_THR_LIBS@ THR_LIB_NAME=@EMU_THR_LIB_NAME@ -THR_LIBS=$(subst -l$(THR_LIB_NAME),-l$(THR_LIB_NAME)$(TYPEMARKER),$(ORG_THR_LIBS)) +ifneq ($(strip $(THR_LIB_NAME)),) +DEPLIBS += $(ERL_TOP)/erts/lib/internal/$(TARGET)/$(LIB_PREFIX)erts_internal_r$(TYPEMARKER)$(LIB_SUFFIX) \ + $(ERL_TOP)/erts/lib/internal/$(TARGET)/$(LIB_PREFIX)ethread$(TYPEMARKER)$(LIB_SUFFIX) +else +DEPLIBS += $(ERL_TOP)/erts/lib/internal/$(TARGET)/$(LIB_PREFIX)erts_internal$(TYPEMARKER)$(LIB_SUFFIX) +endif + +THR_LIBS=$(subst -l$(THR_LIB_NAME),-l$(THR_LIB_NAME)$(TYPEMARKER), \ + $(subst -lerts_internal_r,-lerts_internal_r$(TYPEMARKER),$(ORG_THR_LIBS))) + +LIBS += $(THR_LIBS) + +ifneq ($(findstring erts_internal_r, $(THR_LIBS)),erts_internal_r) ifeq ($(findstring vxworks,$(TARGET)),vxworks) ERTS_INTERNAL_LIB=erts_internal @@ -321,7 +332,9 @@ ERTS_INTERNAL_LIB=erts_internal endif endif -LIBS += $(THR_LIBS) -l$(ERTS_INTERNAL_LIB)$(TYPEMARKER) +LIBS += -l$(ERTS_INTERNAL_LIB)$(TYPEMARKER) + +endif # erts_internal_r LIBS += @LIBRT@ @@ -345,7 +358,6 @@ ERLANG_OSTYPE = @ERLANG_OSTYPE@ ENABLE_ALLOC_TYPE_VARS += @ERLANG_OSTYPE@ -EMULATOR_EXECUTABLE_SAE = beam_evm$(TF_MARKER) EMULATOR_EXECUTABLE_ELIB = beam.elib$(TF_MARKER) ifeq ($(TARGET), win32) EMULATOR_EXECUTABLE = beam$(TF_MARKER).dll @@ -410,7 +422,7 @@ endif @set -e ; cd zlib && $(MAKE) clean @set -e ; cd pcre && $(MAKE) clean -.PHONY: all sae zlib pcre clean +.PHONY: all zlib pcre clean docs: @@ -423,12 +435,6 @@ RELEASE_INCLUDES = beam/erl_driver.h sys/$(ERLANG_OSTYPE)/driver_int.h beam/erl_ ifeq ($(TARGET),win32) RELEASE_INCLUDES += sys/$(ERLANG_OSTYPE)/erl_win_dyn_driver.h endif -ifeq ($(findstring ose,$(TARGET)),ose) -RELEASE_INCLUDES += sys/$(ERLANG_OSTYPE)/erl_port_signals.sig \ - sys/$(ERLANG_OSTYPE)/ose_erl_port_prog.h \ - drivers/$(ERLANG_OSTYPE)/ose_erl_driver.h - -endif ifeq ($(TYPE)-@HAVE_VALGRIND@,valgrind-no) release_spec: @@ -452,8 +458,6 @@ release_spec: all ifeq ($(ERLANG_OSTYPE), unix) $(INSTALL_PROGRAM) $(BINDIR)/$(CS_EXECUTABLE) $(RELSYSDIR)/bin endif - $(INSTALL_DIR) $(RELEASE_PATH)/usr/include/obsolete - $(INSTALL_DATA) obsolete/driver.h $(RELEASE_PATH)/usr/include/obsolete endif endif @@ -480,8 +484,6 @@ endif ifeq ($(findstring vxworks,$(TARGET)),vxworks) else -ifeq ($(findstring ose,$(TARGET)),ose) -else ifdef HIPE_ENABLED GENERATE += $(TTF_DIR)/hipe_x86_asm.h \ $(TTF_DIR)/hipe_amd64_asm.h \ @@ -492,7 +494,6 @@ GENERATE += $(TTF_DIR)/hipe_x86_asm.h \ $(BINDIR)/hipe_mkliterals$(TF_MARKER) endif endif -endif ifeq ($(FLAVOR)-@ERTS_BUILD_SMP_EMU@,smp-no) GENERATE= @@ -592,7 +593,6 @@ ifeq ($(findstring vxworks,$(TARGET)),vxworks) INCLUDES += -I$(ERL_TOP)/erts/etc/vxworks endif -ifneq ($(findstring ose,$(TARGET)),ose) ifeq ($(TARGET),win32) # Usually the same as the default rule, but certain platforms (i.e. win32) mix # different compilers @@ -618,33 +618,6 @@ endif $(OBJDIR)/%.o: beam/%.c $(CC) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@ -else - -INCLUDES += -Idrivers/ose - -ifeq ($(TYPE),debug) -$(OBJDIR)/%.o: beam/%.c - $(CC) $(CFLAGS) -DNO_JUMP_TABLE $(INCLUDES) -c $< -o $@ -else - -VXCC=@VXCC@ -VXCFLAGS=@VXCFLAGS@ -CFLAGS_NOOPT=@CFLAGS_NOOPT@ @DEFS@ $(WFLAGS) $(THR_DEFS) $(ARCHCFLAGS) - -# we want to use jump table -$(OBJDIR)/beam_emu.o: beam/beam_emu.c - $(VXCC) $(VXCFLAGS) $(INCLUDES) -c $< -o $@ - -# erl_process does not work properly with DIAB's -XO option, -# we'll compile it with gcc instead -$(OBJDIR)/erl_process.o: beam/erl_process.c - $(VXCC) $(VXCFLAGS) $(INCLUDES) -c $< -o $@ - -$(OBJDIR)/%.o: beam/%.c - $(CC) $(CFLAGS) $(INCLUDES) -c $< -o $@ -endif -endif - $(OBJDIR)/%.o: $(TARGET)/%.c $(CC) $(CFLAGS) $(INCLUDES) -Idrivers/common -c $< -o $@ @@ -663,15 +636,11 @@ $(OBJDIR)/%.o: drivers/common/%.c $(OBJDIR)/%.o: drivers/$(ERLANG_OSTYPE)/%.c $(CC) $(CFLAGS) $(INCLUDES) -Idrivers/common -Idrivers/$(ERLANG_OSTYPE) -I../etc/$(ERLANG_OSTYPE) -c $< -o $@ -# VxWorks and OSE uses unix drivers too... +# VxWorks uses unix drivers too... ifeq ($(findstring vxworks,$(TARGET)),vxworks) $(OBJDIR)/%.o: drivers/unix/%.c $(CC) $(CFLAGS) $(INCLUDES) -Idrivers/common -c $< -o $@ endif -ifeq ($(findstring ose,$(TARGET)),ose) -$(OBJDIR)/%.o: drivers/unix/%.c - $(CC) $(CFLAGS) $(INCLUDES) -Idrivers/common -c $< -o $@ -endif # ---------------------------------------------------------------------- # Specials @@ -682,13 +651,6 @@ $(BINDIR)/$(CS_EXECUTABLE): $(CS_SRC) $(CS_PURIFY) $(CC) $(CS_LDFLAGS) -o $(BINDIR)/$(CS_EXECUTABLE) \ $(CS_CFLAGS) $(COMMON_INCLUDES) $(CS_SRC) $(CS_LIBS) -$(OBJDIR)/%.elib.o: beam/%.c - $(CC) $(ELIB_FLAGS) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@ - -# Disable vfork() for sae (then we don't need the child_setup program) -$(OBJDIR)/sys_sae.o: sys/$(ERLANG_OSTYPE)/sys.c - $(CC) -DDISABLE_VFORK=1 $(CFLAGS) $(INCLUDES) -c $< -o $@ - $(OBJDIR)/%.kp.o: sys/common/%.c $(CC) -DERTS_KERNEL_POLL_VERSION $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@ @@ -697,9 +659,6 @@ $(OBJDIR)/%.nkp.o: sys/common/%.c ifeq ($(GCC),yes) -$(OBJDIR)/erl_obsolete.o: beam/erl_obsolete.c - $(CC) $(subst -Wstrict-prototypes, , $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS))) $(INCLUDES) -c $< -o $@ - $(OBJDIR)/erl_goodfit_alloc.o: beam/erl_goodfit_alloc.c $(CC) $(subst -O2, $(GEN_OPT_FLGS) $(UNROLL_FLG), $(CFLAGS)) $(INCLUDES) -c $< -o $@ endif @@ -734,8 +693,6 @@ endif INIT_OBJS = $(OBJDIR)/erl_main.o $(PRELOAD) -INIT_OBJS_SAE = $(OBJDIR)/erl9_start.o - EMU_OBJS = \ $(OBJDIR)/beam_emu.o $(OBJDIR)/beam_opcodes.o \ $(OBJDIR)/beam_load.o $(OBJDIR)/beam_bif_load.o \ @@ -776,11 +733,12 @@ RUN_OBJS = \ $(OBJDIR)/erl_fun.o $(OBJDIR)/erl_bif_port.o \ $(OBJDIR)/erl_term.o $(OBJDIR)/erl_node_tables.o \ $(OBJDIR)/erl_monitors.o $(OBJDIR)/erl_process_dump.o \ - $(OBJDIR)/erl_obsolete.o $(OBJDIR)/erl_bif_timer.o \ + $(OBJDIR)/erl_bif_timer.o \ $(OBJDIR)/erl_drv_thread.o $(OBJDIR)/erl_bif_chksum.o \ $(OBJDIR)/erl_bif_re.o $(OBJDIR)/erl_unicode.o \ $(OBJDIR)/packet_parser.o $(OBJDIR)/safe_hash.o \ - $(OBJDIR)/erl_zlib.o $(OBJDIR)/erl_nif.o + $(OBJDIR)/erl_zlib.o $(OBJDIR)/erl_nif.o \ + $(OBJDIR)/erl_bif_binary.o ifeq ($(TARGET),win32) DRV_OBJS = \ @@ -799,29 +757,21 @@ OS_OBJS = \ $(OBJDIR)/sys_time.o \ $(OBJDIR)/sys_interrupt.o \ $(OBJDIR)/sys_env.o \ - $(OBJDIR)/dosmap.o \ - $(OBJDIR)/elib_malloc.o + $(OBJDIR)/dosmap.o else OS_OBJS = \ $(OBJDIR)/sys.o \ $(OBJDIR)/driver_tab.o \ $(OBJDIR)/unix_efile.o \ $(OBJDIR)/gzio.o \ - $(OBJDIR)/elib_malloc.o \ $(OBJDIR)/elib_memmove.o -ifeq ($(findstring ose,$(TARGET)),ose) - OS_OBJS += $(OBJDIR)/erl_port_init.o \ - $(OBJDIR)/ose_inet_sock_select.o \ - $(OBJDIR)/ose_sfp.o -else ifeq ($(findstring vxworks,$(TARGET)),vxworks) OS_OBJS += $(OBJDIR)/int64.o else OS_OBJS += $(OBJDIR)/sys_float.o \ $(OBJDIR)/sys_time.o endif -endif DRV_OBJS = \ $(OBJDIR)/efile_drv.o \ $(OBJDIR)/inet_drv.o \ @@ -830,11 +780,7 @@ DRV_OBJS = \ endif ifneq ($(findstring vxworks,$(TARGET)),vxworks) - ifeq ($(findstring ose,$(TARGET)),ose) - DRV_OBJS += $(OBJDIR)/ose_inet_drv.o - else DRV_OBJS += $(OBJDIR)/ttsl_drv.o - endif endif ifeq ($(ERTS_ENABLE_KERNEL_POLL),yes) @@ -880,17 +826,6 @@ endif BASE_OBJS = $(RUN_OBJS) $(EMU_OBJS) $(OS_OBJS) $(EXTRA_BASE_OBJS) OBJS = $(BASE_OBJS) $(DRV_OBJS) -OBJS_SAE = $(subst sys.o,sys_sae.o,$(OBJS)) - -ELIB_C_FILES = beam/elib_malloc.c \ - beam/elib_memmove.c \ - beam/erl_bif_info.c \ - beam/utils.c \ - beam/erl_alloc.c - -MOD_OBJS_ELIB = $(patsubst %.c,$(OBJDIR)/%.o,$(notdir $(ELIB_C_FILES))) -OBJS_ELIB = $(patsubst %.o,%.elib.o,$(MOD_OBJS_ELIB)) \ - $(filter-out $(MOD_OBJS_ELIB),$(OBJS)) ######################################## # HiPE section @@ -968,33 +903,6 @@ $(TARGET)/int64.c: endif -ifeq ($(findstring ose,$(TARGET)),ose) -# Extract soft float functions from libgcc.a (for beam_emu) -VXCC=@VXCC@ -VXCFLAGS=@VXCFLAGS@ -VXLD=@VXLD@ -VXLDFLAGS=@VXLDFLAGS@ -VXCCLIBFLAGS=@VXCCLIBFLAGS@ -STRIP=@STRIP@ -SYMPREFIX=@SYMPREFIX@ - -NEEDFUNCTIONS=__floatsidf __adddf3 __negdf2 __muldf3 __divdf3 __subdf3 -KEEPSYMS=$(NEEDFUNCTIONS:%=-K $(SYMPREFIX)%) - -$(OBJDIR)/ose_sfp.o: $(TARGET)/ose_sfp.c - $(VXCC) $(VXCFLAGS) -o $(OBJDIR)/ose_sfp_tmp.o -c $(TARGET)/ose_sfp.c - $(VXLD) -o $(OBJDIR)/ose_sfp.o $(OBJDIR)/ose_sfp_tmp.o $(VXLDFLAGS) $(VXCCLIBFLAGS) - $(STRIP) $(KEEPSYMS) $(OBJDIR)/ose_sfp.o - -$(TARGET)/ose_sfp.c: - echo 'void dummy(void); void dummy(void) {' > $(TARGET)/ose_sfp.c - for x in $(NEEDFUNCTIONS); do echo 'extern void '$$x'();' \ - >> $(TARGET)/ose_sfp.c; done - for x in $(NEEDFUNCTIONS); do echo $$x'();' >> $(TARGET)/ose_sfp.c; done - echo '}' >> $(TARGET)/ose_sfp.c - -endif - # ---------------------------------------------------------------------- # The emulator itself @@ -1010,14 +918,6 @@ $(BINDIR)/$(EMULATOR_EXECUTABLE): $(INIT_OBJS) $(OBJS) $(DEPLIBS) $(PURIFY) $(LD) -o $(BINDIR)/$(EMULATOR_EXECUTABLE) \ $(HIPEBEAMLDFLAGS) $(LDFLAGS) $(DEXPORT) $(INIT_OBJS) $(OBJS) $(LIBS) -$(BINDIR)/$(EMULATOR_EXECUTABLE_ELIB): $(INIT_OBJS) $(OBJS_ELIB) $(DEPLIBS) - $(PURIFY) $(LD) -o $(BINDIR)/$(EMULATOR_EXECUTABLE_ELIB) \ - $(LDFLAGS) $(DEXPORT) $(INIT_OBJS) $(OBJS_ELIB) $(LIBS) - -$(BINDIR)/$(EMULATOR_EXECUTABLE_SAE): $(INIT_OBJS_SAE) $(OBJS_SAE) $(DEPLIBS) - $(PURIFY) $(LD) -o $(BINDIR)/$(EMULATOR_EXECUTABLE_SAE) \ - $(LDFLAGS) $(DEXPORT) $(INIT_OBJS_SAE) $(OBJS_SAE) $(LIBS) - endif # @@ -1112,7 +1012,7 @@ depend: $(DEP_CC) $(DEP_FLAGS) $(TARGET_SRC) \ | $(SED_DEPEND) >> $(TARGET)/depend.mk ifneq ($(TARGET),win32) - $(DEP_CC) $(DEP_FLAGS) $(ELIB_FLAGS) $(ELIB_C_FILES) \ + $(DEP_CC) $(DEP_FLAGS) $(ELIB_C_FILES) \ | $(SED_ELIB_DEPEND) >> $(TARGET)/depend.mk endif ifdef HIPE_ENABLED diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index e2a79d6e4f..6b3c106a97 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -41,8 +41,7 @@ static erts_smp_rwmtx_t atom_table_lock; #define atom_read_unlock() erts_smp_rwmtx_runlock(&atom_table_lock) #define atom_write_lock() erts_smp_rwmtx_rwlock(&atom_table_lock) #define atom_write_unlock() erts_smp_rwmtx_rwunlock(&atom_table_lock) -#define atom_init_lock() erts_smp_rwmtx_init(&atom_table_lock, \ - "atom_tab") + #if 0 #define ERTS_ATOM_PUT_OPS_STAT #endif @@ -304,12 +303,17 @@ init_atom_table(void) HashFunctions f; int i; Atom a; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; #ifdef ERTS_ATOM_PUT_OPS_STAT erts_smp_atomic_init(&atom_put_ops, 0); #endif - atom_init_lock(); + erts_smp_rwmtx_init_opt(&atom_table_lock, &rwmtx_opt, "atom_tab"); + f.hash = (H_FUN) atom_hash; f.cmp = (HCMP_FUN) atom_cmp; f.alloc = (HALLOC_FUN) atom_alloc; diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 57c8b08223..0815cdbc7f 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -65,6 +65,7 @@ atom EXIT='EXIT' atom aborted atom abs_path atom absoluteURI +atom ac atom active atom all atom all_but_first @@ -100,8 +101,15 @@ atom band atom big atom bif_return_trap atom binary +atom binary_bin_to_list_trap +atom binary_copy_trap +atom binary_longest_prefix_trap +atom binary_longest_suffix_trap +atom binary_match_trap +atom binary_matches_trap atom block atom blocked +atom bm atom bnot atom bor atom bxor @@ -115,6 +123,7 @@ atom busy_dist_port atom busy_port atom call atom call_count +atom call_time atom caller atom capture atom case_clause @@ -256,6 +265,7 @@ atom info atom info_msg atom initial_call atom input +atom internal atom internal_error atom internal_status atom instruction_counts @@ -422,6 +432,7 @@ atom raw atom re atom re_pattern atom re_run_trap +atom read_concurrency atom ready_input atom ready_output atom ready_async @@ -453,6 +464,7 @@ atom scheduler atom scheduler_id atom schedulers_online atom scheme +atom scope atom sensitive atom sequential_tracer atom sequential_trace_token diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index b1feec7074..4fc271d41c 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -39,10 +39,10 @@ static Eterm check_process_code(Process* rp, Module* modp); static void delete_code(Process *c_p, ErtsProcLocks c_p_locks, Module* modp); static void delete_export_references(Eterm module); static int purge_module(int module); -static int is_native(Eterm* code); +static int is_native(BeamInstr* code); static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); -static void remove_from_address_table(Eterm* code); +static void remove_from_address_table(BeamInstr* code); Eterm load_module_2(BIF_ALIST_2) @@ -344,8 +344,8 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) modp->code[MI_ON_LOAD_FUNCTION_PTR] = 0; set_default_trace_pattern(BIF_ARG_1); } else if (BIF_ARG_2 == am_false) { - Eterm* code; - Eterm* end; + BeamInstr* code; + BeamInstr* end; /* * The on_load function failed. Remove the loaded code. @@ -354,7 +354,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) */ erts_total_code_size -= modp->code_length; code = modp->code; - end = (Eterm *)((char *)code + modp->code_length); + end = (BeamInstr *)((char *)code + modp->code_length); erts_cleanup_funs_on_purge(code, end); beam_catches_delmod(modp->catches, code, modp->code_length); erts_free(ERTS_ALC_T_CODE, (void *) code); @@ -368,7 +368,6 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) BIF_RET(am_true); } - static void set_default_trace_pattern(Eterm module) { @@ -397,13 +396,13 @@ set_default_trace_pattern(Eterm module) static Eterm check_process_code(Process* rp, Module* modp) { - Eterm* start; + BeamInstr* start; char* mod_start; Uint mod_size; - Eterm* end; + BeamInstr* end; Eterm* sp; #ifndef HYBRID /* FIND ME! */ - ErlFunThing* funp; + struct erl_off_heap_header* oh; int done_gc = 0; #endif @@ -418,7 +417,7 @@ check_process_code(Process* rp, Module* modp) * Pick up limits for the module. */ start = modp->old_code; - end = (Eterm *)((char *)start + modp->old_code_length); + end = (BeamInstr *)((char *)start + modp->old_code_length); mod_start = (char *) start; mod_size = modp->old_code_length; @@ -471,27 +470,30 @@ check_process_code(Process* rp, Module* modp) #ifndef HYBRID /* FIND ME! */ rescan: - for (funp = MSO(rp).funs; funp; funp = funp->next) { - Eterm* fun_code; - - fun_code = funp->fe->address; - - if (INSIDE((Eterm *) funp->fe->address)) { - if (done_gc) { - return am_true; - } else { - /* - * Try to get rid of this fun by garbage collecting. - * Clear both fvalue and ftrace to make sure they - * don't hold any funs. - */ - rp->freason = EXC_NULL; - rp->fvalue = NIL; - rp->ftrace = NIL; - done_gc = 1; - FLAGS(rp) |= F_NEED_FULLSWEEP; - (void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); - goto rescan; + for (oh = MSO(rp).first; oh; oh = oh->next) { + if (thing_subtag(oh->thing_word) == FUN_SUBTAG) { + ErlFunThing* funp = (ErlFunThing*) oh; + BeamInstr* fun_code; + + fun_code = funp->fe->address; + + if (INSIDE((BeamInstr *) funp->fe->address)) { + if (done_gc) { + return am_true; + } else { + /* + * Try to get rid of this fun by garbage collecting. + * Clear both fvalue and ftrace to make sure they + * don't hold any funs. + */ + rp->freason = EXC_NULL; + rp->fvalue = NIL; + rp->ftrace = NIL; + done_gc = 1; + FLAGS(rp) |= F_NEED_FULLSWEEP; + (void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); + goto rescan; + } } } } @@ -576,7 +578,7 @@ any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) switch (primary_tag(val)) { case TAG_PRIMARY_BOXED: case TAG_PRIMARY_LIST: - if (in_area(val, mod_start, mod_size)) { + if (in_area(EXPAND_POINTER(val), mod_start, mod_size)) { return 1; } break; @@ -596,7 +598,7 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) switch (primary_tag(val)) { case TAG_PRIMARY_BOXED: case TAG_PRIMARY_LIST: - if (in_area(val, mod_start, mod_size)) { + if (in_area(EXPAND_POINTER(val), mod_start, mod_size)) { return 1; } break; @@ -617,8 +619,8 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) static int purge_module(int module) { - Eterm* code; - Eterm* end; + BeamInstr* code; + BeamInstr* end; Module* modp; /* @@ -653,7 +655,7 @@ purge_module(int module) ASSERT(erts_total_code_size >= modp->old_code_length); erts_total_code_size -= modp->old_code_length; code = modp->old_code; - end = (Eterm *)((char *)code + modp->old_code_length); + end = (BeamInstr *)((char *)code + modp->old_code_length); erts_cleanup_funs_on_purge(code, end); beam_catches_delmod(modp->old_catches, code, modp->old_code_length); erts_free(ERTS_ALC_T_CODE, (void *) code); @@ -665,7 +667,7 @@ purge_module(int module) } static void -remove_from_address_table(Eterm* code) +remove_from_address_table(BeamInstr* code) { int i; @@ -738,11 +740,11 @@ delete_export_references(Eterm module) Export *ep = export_list(i); if (ep != NULL && (ep->code[0] == module)) { if (ep->address == ep->code+3 && - (ep->code[3] == (Eterm) em_apply_bif)) { + (ep->code[3] == (BeamInstr) em_apply_bif)) { continue; } ep->address = ep->code+3; - ep->code[3] = (Uint) em_call_error_handler; + ep->code[3] = (BeamInstr) em_call_error_handler; ep->code[4] = 0; MatchSetUnref(ep->match_prog_set); ep->match_prog_set = NULL; @@ -774,7 +776,7 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module) } static int -is_native(Eterm* code) +is_native(BeamInstr* code) { return ((Eterm *)code[MI_FUNCTIONS])[1] != 0; } diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 1abf1dc10c..682f31b83f 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2000-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -30,6 +30,7 @@ #include "error.h" #include "erl_binary.h" #include "beam_bp.h" +#include "erl_term.h" /* ************************************************************************* ** Macros @@ -100,6 +101,11 @@ do { \ (b)->prev = (a); \ } while (0) + +#define BREAK_IS_BIF (1) +#define BREAK_IS_ERL (0) + + /* ************************************************************************* ** Local prototypes */ @@ -109,24 +115,42 @@ do { \ */ static int set_break(Eterm mfa[3], int specified, - Binary *match_spec, Uint break_op, + Binary *match_spec, BeamInstr break_op, enum erts_break_op count_op, Eterm tracer_pid); static int set_module_break(Module *modp, Eterm mfa[3], int specified, - Binary *match_spec, Uint break_op, + Binary *match_spec, BeamInstr break_op, enum erts_break_op count_op, Eterm tracer_pid); -static int set_function_break(Module *modp, Uint *pc, - Binary *match_spec, Uint break_op, +static int set_function_break(Module *modp, BeamInstr *pc, int bif, + Binary *match_spec, BeamInstr break_op, enum erts_break_op count_op, Eterm tracer_pid); static int clear_break(Eterm mfa[3], int specified, - Uint break_op); + BeamInstr break_op); static int clear_module_break(Module *modp, Eterm mfa[3], int specified, - Uint break_op); -static int clear_function_break(Module *modp, Uint *pc, - Uint break_op); - -static BpData *is_break(Uint *pc, Uint break_op); - + BeamInstr break_op); +static int clear_function_break(Module *modp, BeamInstr *pc, int bif, + BeamInstr break_op); + +static BpData *is_break(BeamInstr *pc, BeamInstr break_op); +static BpData *get_break(Process *p, BeamInstr *pc, BeamInstr break_op); + +/* bp_hash */ +#define BP_TIME_ADD(pi0, pi1) \ + do { \ + Uint r; \ + (pi0)->count += (pi1)->count; \ + (pi0)->s_time += (pi1)->s_time; \ + (pi0)->us_time += (pi1)->us_time; \ + r = (pi0)->us_time / 1000000; \ + (pi0)->s_time += r; \ + (pi0)->us_time = (pi0)->us_time % 1000000; \ + } while(0) + +static void bp_hash_init(bp_time_hash_t *hash, Uint n); +static void bp_hash_rehash(bp_time_hash_t *hash, Uint n); +static ERTS_INLINE bp_data_time_item_t * bp_hash_get(bp_time_hash_t *hash, bp_data_time_item_t *sitem); +static ERTS_INLINE bp_data_time_item_t * bp_hash_put(bp_time_hash_t *hash, bp_data_time_item_t *sitem); +static void bp_hash_delete(bp_time_hash_t *hash); /* ************************************************************************* @@ -145,7 +169,7 @@ erts_set_trace_break(Eterm mfa[3], int specified, Binary *match_spec, Eterm tracer_pid) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); return set_break(mfa, specified, match_spec, - (Uint) BeamOp(op_i_trace_breakpoint), 0, tracer_pid); + (BeamInstr) BeamOp(op_i_trace_breakpoint), 0, tracer_pid); } int @@ -153,87 +177,84 @@ erts_set_mtrace_break(Eterm mfa[3], int specified, Binary *match_spec, Eterm tracer_pid) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); return set_break(mfa, specified, match_spec, - (Uint) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid); + (BeamInstr) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid); } +/* set breakpoint data for on exported bif entry */ + void -erts_set_mtrace_bif(Uint *pc, Binary *match_spec, Eterm tracer_pid) { - BpDataTrace *bdt; +erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, Eterm tracer_pid) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + set_function_break(NULL, pc, BREAK_IS_BIF, match_spec, (BeamInstr) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid); +} - bdt = (BpDataTrace *) pc[-4]; - if (bdt) { - MatchSetUnref(bdt->match_spec); - MatchSetRef(match_spec); - bdt->match_spec = match_spec; - bdt->tracer_pid = tracer_pid; - } else { - bdt = Alloc(sizeof(BpDataTrace)); - BpInit((BpData *) bdt, 0); - MatchSetRef(match_spec); - bdt->match_spec = match_spec; - bdt->tracer_pid = tracer_pid; - pc[-4] = (Uint) bdt; - } +void erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op count_op) { + set_function_break(NULL, pc, BREAK_IS_BIF, NULL, (BeamInstr) BeamOp(op_i_time_breakpoint), count_op, NIL); +} + +void erts_clear_time_trace_bif(BeamInstr *pc) { + clear_function_break(NULL, pc, BREAK_IS_BIF, (BeamInstr) BeamOp(op_i_time_breakpoint)); } int erts_set_debug_break(Eterm mfa[3], int specified) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); return set_break(mfa, specified, NULL, - (Uint) BeamOp(op_i_debug_breakpoint), 0, NIL); + (BeamInstr) BeamOp(op_i_debug_breakpoint), 0, NIL); } int erts_set_count_break(Eterm mfa[3], int specified, enum erts_break_op count_op) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); return set_break(mfa, specified, NULL, - (Uint) BeamOp(op_i_count_breakpoint), count_op, NIL); + (BeamInstr) BeamOp(op_i_count_breakpoint), count_op, NIL); } - +int +erts_set_time_break(Eterm mfa[3], int specified, enum erts_break_op count_op) { + ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + return set_break(mfa, specified, NULL, + (BeamInstr) BeamOp(op_i_time_breakpoint), count_op, NIL); +} int erts_clear_trace_break(Eterm mfa[3], int specified) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); return clear_break(mfa, specified, - (Uint) BeamOp(op_i_trace_breakpoint)); + (BeamInstr) BeamOp(op_i_trace_breakpoint)); } int erts_clear_mtrace_break(Eterm mfa[3], int specified) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); return clear_break(mfa, specified, - (Uint) BeamOp(op_i_mtrace_breakpoint)); + (BeamInstr) BeamOp(op_i_mtrace_breakpoint)); } void -erts_clear_mtrace_bif(Uint *pc) { - BpDataTrace *bdt; - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); - - bdt = (BpDataTrace *) pc[-4]; - if (bdt) { - if (bdt->match_spec) { - MatchSetUnref(bdt->match_spec); - } - Free(bdt); - } - pc[-4] = (Uint) NULL; +erts_clear_mtrace_bif(BeamInstr *pc) { + clear_function_break(NULL, pc, BREAK_IS_BIF, (BeamInstr) BeamOp(op_i_mtrace_breakpoint)); } int erts_clear_debug_break(Eterm mfa[3], int specified) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); return clear_break(mfa, specified, - (Uint) BeamOp(op_i_debug_breakpoint)); + (BeamInstr) BeamOp(op_i_debug_breakpoint)); } int erts_clear_count_break(Eterm mfa[3], int specified) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); return clear_break(mfa, specified, - (Uint) BeamOp(op_i_count_breakpoint)); + (BeamInstr) BeamOp(op_i_count_breakpoint)); +} + +int +erts_clear_time_break(Eterm mfa[3], int specified) { + ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + return clear_break(mfa, specified, + (BeamInstr) BeamOp(op_i_time_breakpoint)); } int @@ -250,10 +271,10 @@ erts_clear_module_break(Module *modp) { } int -erts_clear_function_break(Module *modp, Uint *pc) { +erts_clear_function_break(Module *modp, BeamInstr *pc) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); ASSERT(modp); - return clear_function_break(modp, pc, 0); + return clear_function_break(modp, pc, BREAK_IS_ERL, 0); } @@ -261,13 +282,16 @@ erts_clear_function_break(Module *modp, Uint *pc) { /* * SMP NOTE: Process p may have become exiting on return! */ -Uint -erts_trace_break(Process *p, Uint *pc, Eterm *args, +BeamInstr +erts_trace_break(Process *p, BeamInstr *pc, Eterm *args, Uint32 *ret_flags, Eterm *tracer_pid) { Eterm tpid1, tpid2; - BpDataTrace *bdt = (BpDataTrace *) pc[-4]; - - ASSERT(pc[-5] == (Uint) BeamOp(op_i_func_info_IaaI)); + BpData **bds = (BpData **) (pc)[-4]; + BpDataTrace *bdt = NULL; + + ASSERT(bds); + ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + bdt = (BpDataTrace *) bds[bp_sched2ix_proc(p)]; ASSERT(bdt); bdt = (BpDataTrace *) bdt->next; ASSERT(bdt); @@ -286,7 +310,7 @@ erts_trace_break(Process *p, Uint *pc, Eterm *args, bdt->tracer_pid = tpid2; ErtsSmpBPUnlock(bdt); } - pc[-4] = (Uint) bdt; + bds[bp_sched2ix_proc(p)] = (BpData *) bdt; return bdt->orig_instr; } @@ -296,14 +320,17 @@ erts_trace_break(Process *p, Uint *pc, Eterm *args, * SMP NOTE: Process p may have become exiting on return! */ Uint32 -erts_bif_mtrace(Process *p, Uint *pc, Eterm *args, int local, +erts_bif_mtrace(Process *p, BeamInstr *pc, Eterm *args, int local, Eterm *tracer_pid) { - BpDataTrace *bdt = (BpDataTrace *) pc[-4]; - + BpData **bds = (BpData **) (pc)[-4]; + BpDataTrace *bdt = NULL; + + ASSERT(tracer_pid); - if (bdt) { + if (bds) { Eterm tpid1, tpid2; Uint32 flags; + bdt = (BpDataTrace *)bds[bp_sched2ix_proc(p)]; ErtsSmpBPLock(bdt); tpid1 = tpid2 = bdt->tracer_pid; @@ -326,9 +353,9 @@ erts_bif_mtrace(Process *p, Uint *pc, Eterm *args, int local, int -erts_is_trace_break(Uint *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) { +erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) { BpDataTrace *bdt = - (BpDataTrace *) is_break(pc, (Uint) BeamOp(op_i_trace_breakpoint)); + (BpDataTrace *) is_break(pc, (BeamInstr) BeamOp(op_i_trace_breakpoint)); if (bdt) { if (match_spec_ret) { @@ -345,9 +372,9 @@ erts_is_trace_break(Uint *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) { } int -erts_is_mtrace_break(Uint *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) { +erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) { BpDataTrace *bdt = - (BpDataTrace *) is_break(pc, (Uint) BeamOp(op_i_mtrace_breakpoint)); + (BpDataTrace *) is_break(pc, (BeamInstr) BeamOp(op_i_mtrace_breakpoint)); if (bdt) { if (match_spec_ret) { @@ -364,74 +391,455 @@ erts_is_mtrace_break(Uint *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) { } int -erts_is_mtrace_bif(Uint *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) { - BpDataTrace *bdt = (BpDataTrace *) pc[-4]; - - if (bdt) { - if (match_spec_ret) { - *match_spec_ret = bdt->match_spec; - } - if (tracer_pid_ret) { - ErtsSmpBPLock(bdt); - *tracer_pid_ret = bdt->tracer_pid; - ErtsSmpBPUnlock(bdt); - } - return !0; - } - return 0; -} - -int -erts_is_native_break(Uint *pc) { +erts_is_native_break(BeamInstr *pc) { #ifdef HIPE - ASSERT(pc[-5] == (Uint) BeamOp(op_i_func_info_IaaI)); - return pc[0] == (Uint) BeamOp(op_hipe_trap_call) - || pc[0] == (Uint) BeamOp(op_hipe_trap_call_closure); + ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + return pc[0] == (BeamInstr) BeamOp(op_hipe_trap_call) + || pc[0] == (BeamInstr) BeamOp(op_hipe_trap_call_closure); #else return 0; #endif } int -erts_is_count_break(Uint *pc, Sint *count_ret) { +erts_is_count_break(BeamInstr *pc, Sint *count_ret) { BpDataCount *bdc = - (BpDataCount *) is_break(pc, (Uint) BeamOp(op_i_count_breakpoint)); + (BpDataCount *) is_break(pc, (BeamInstr) BeamOp(op_i_count_breakpoint)); if (bdc) { if (count_ret) { - ErtsSmpBPLock(bdc); - *count_ret = bdc->count; - ErtsSmpBPUnlock(bdc); + *count_ret = (Sint) erts_smp_atomic_read(&bdc->acount); } return !0; } return 0; } -Uint * +int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) { + Uint i, ix; + bp_time_hash_t hash; + Uint size; + Eterm *hp, t; + bp_data_time_item_t *item = NULL; + BpDataTime *bdt = (BpDataTime *) is_break(pc, (BeamInstr) BeamOp(op_i_time_breakpoint)); + + if (bdt) { + if (retval) { + /* collect all hashes to one hash */ + bp_hash_init(&hash, 64); + /* foreach threadspecific hash */ + for (i = 0; i < bdt->n; i++) { + bp_data_time_item_t *sitem; + + /* foreach hash bucket not NIL*/ + for(ix = 0; ix < bdt->hash[i].n; ix++) { + item = &(bdt->hash[i].item[ix]); + if (item->pid != NIL) { + sitem = bp_hash_get(&hash, item); + if (sitem) { + BP_TIME_ADD(sitem, item); + } else { + bp_hash_put(&hash, item); + } + } + } + } + /* *retval should be NIL or term from previous bif in export entry */ + + if (hash.used > 0) { + size = (5 + 2)*hash.used; + hp = HAlloc(p, size); + + for(ix = 0; ix < hash.n; ix++) { + item = &(hash.item[ix]); + if (item->pid != NIL) { + t = TUPLE4(hp, item->pid, + make_small(item->count), + make_small(item->s_time), + make_small(item->us_time)); + hp += 5; + *retval = CONS(hp, t, *retval); hp += 2; + } + } + } + bp_hash_delete(&hash); + } + return !0; + } + + return 0; +} + + +BeamInstr * erts_find_local_func(Eterm mfa[3]) { Module *modp; - Uint** code_base; - Uint* code_ptr; + BeamInstr** code_base; + BeamInstr* code_ptr; Uint i,n; if ((modp = erts_get_module(mfa[0])) == NULL) return NULL; - if ((code_base = (Uint **) modp->code) == NULL) + if ((code_base = (BeamInstr **) modp->code) == NULL) return NULL; - n = (Uint) code_base[MI_NUM_FUNCTIONS]; + n = (BeamInstr) code_base[MI_NUM_FUNCTIONS]; for (i = 0; i < n; ++i) { code_ptr = code_base[MI_FUNCTIONS+i]; - ASSERT(((Uint) BeamOp(op_i_func_info_IaaI)) == code_ptr[0]); + ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == code_ptr[0]); ASSERT(mfa[0] == ((Eterm) code_ptr[2])); if (mfa[1] == ((Eterm) code_ptr[3]) && - ((Uint) mfa[2]) == code_ptr[4]) { + ((BeamInstr) mfa[2]) == code_ptr[4]) { return code_ptr + 5; } } return NULL; } +/* bp_hash */ +ERTS_INLINE Uint bp_sched2ix() { +#ifdef ERTS_SMP + ErtsSchedulerData *esdp; + esdp = erts_get_scheduler_data(); + return esdp->no - 1; +#else + return 0; +#endif +} +static void bp_hash_init(bp_time_hash_t *hash, Uint n) { + Uint size = sizeof(bp_data_time_item_t)*n; + Uint i; + + hash->n = n; + hash->used = 0; + + hash->item = (bp_data_time_item_t *)Alloc(size); + sys_memzero(hash->item, size); + + for(i = 0; i < n; ++i) { + hash->item[i].pid = NIL; + } +} + +static void bp_hash_rehash(bp_time_hash_t *hash, Uint n) { + bp_data_time_item_t *item = NULL; + Uint size = sizeof(bp_data_time_item_t)*n; + Uint ix; + Uint hval; + + item = (bp_data_time_item_t *)Alloc(size); + sys_memzero(item, size); + + for( ix = 0; ix < n; ++ix) { + item[ix].pid = NIL; + } + + /* rehash, old hash -> new hash */ + + for( ix = 0; ix < hash->n; ix++) { + if (hash->item[ix].pid != NIL) { + + hval = ((hash->item[ix].pid) >> 4) % n; /* new n */ + + while (item[hval].pid != NIL) { + hval = (hval + 1) % n; + } + item[hval].pid = hash->item[ix].pid; + item[hval].count = hash->item[ix].count; + item[hval].s_time = hash->item[ix].s_time; + item[hval].us_time = hash->item[ix].us_time; + } + } + + Free(hash->item); + hash->n = n; + hash->item = item; +} +static ERTS_INLINE bp_data_time_item_t * bp_hash_get(bp_time_hash_t *hash, bp_data_time_item_t *sitem) { + Eterm pid = sitem->pid; + Uint hval = (pid >> 4) % hash->n; + bp_data_time_item_t *item = NULL; + + item = hash->item; + + while (item[hval].pid != pid) { + if (item[hval].pid == NIL) return NULL; + hval = (hval + 1) % hash->n; + } + + return &(item[hval]); +} + +static ERTS_INLINE bp_data_time_item_t * bp_hash_put(bp_time_hash_t *hash, bp_data_time_item_t* sitem) { + Uint hval; + float r = 0.0; + bp_data_time_item_t *item; + + /* make sure that the hash is not saturated */ + /* if saturated, rehash it */ + + r = hash->used / (float) hash->n; + + if (r > 0.7f) { + bp_hash_rehash(hash, hash->n * 2); + } + /* Do hval after rehash */ + hval = (sitem->pid >> 4) % hash->n; + + /* find free slot */ + item = hash->item; + + while (item[hval].pid != NIL) { + hval = (hval + 1) % hash->n; + } + item = &(hash->item[hval]); + + item->pid = sitem->pid; + item->s_time = sitem->s_time; + item->us_time = sitem->us_time; + item->count = sitem->count; + hash->used++; + + return item; +} + +static void bp_hash_delete(bp_time_hash_t *hash) { + hash->n = 0; + hash->used = 0; + Free(hash->item); + hash->item = NULL; +} + +static void bp_time_diff(bp_data_time_item_t *item, /* out */ + process_breakpoint_time_t *pbt, /* in */ + Uint ms, Uint s, Uint us) { + int dms,ds,dus; + + dms = ms - pbt->ms; + ds = s - pbt->s; + dus = us - pbt->us; + + /* get_sys_now may return zero difftime, + * this is ok. + */ + + ASSERT(dms >= 0 || ds >= 0 || dus >= 0); + + if (dus < 0) { + dus += 1000000; + ds -= 1; + } + if (ds < 0) { + ds += 1000000; + } + + item->s_time = ds; + item->us_time = dus; +} + +void erts_schedule_time_break(Process *p, Uint schedule) { + Uint ms, s, us; + process_breakpoint_time_t *pbt = NULL; + bp_data_time_item_t sitem, *item = NULL; + bp_time_hash_t *h = NULL; + BpDataTime *pbdt = NULL; + + ASSERT(p); + + pbt = ERTS_PROC_GET_CALL_TIME(p); + + if (pbt) { + + switch(schedule) { + case ERTS_BP_CALL_TIME_SCHEDULE_EXITING : + break; + case ERTS_BP_CALL_TIME_SCHEDULE_OUT : + /* When a process is scheduled _out_, + * timestamp it and add its delta to + * the previous breakpoint. + */ + + pbdt = (BpDataTime *) get_break(p, pbt->pc, (BeamInstr) BeamOp(op_i_time_breakpoint)); + if (pbdt) { + get_sys_now(&ms,&s,&us); + bp_time_diff(&sitem, pbt, ms, s, us); + sitem.pid = p->id; + sitem.count = 0; + + h = &(pbdt->hash[bp_sched2ix_proc(p)]); + + ASSERT(h); + ASSERT(h->item); + + item = bp_hash_get(h, &sitem); + if (!item) { + item = bp_hash_put(h, &sitem); + } else { + BP_TIME_ADD(item, &sitem); + } + } + break; + case ERTS_BP_CALL_TIME_SCHEDULE_IN : + /* When a process is scheduled _in_, + * timestamp it and remove the previous + * timestamp in the psd. + */ + get_sys_now(&ms,&s,&us); + pbt->ms = ms; + pbt->s = s; + pbt->us = us; + break; + default : + ASSERT(0); + /* will never happen */ + break; + } + } /* pbt */ +} + +/* call_time breakpoint + * Accumulated times are added to the previous bp, + * not the current one. The current one is saved + * for future reference. + * The previous breakpoint is stored in the process it self, the psd. + * We do not need to store in a stack frame. + * There is no need for locking, each thread has its own + * area in each bp to save data. + * Since we need to diffrentiate between processes for each bp, + * every bp has a hash (per thread) to process-bp statistics. + * - egil + */ + +void erts_trace_time_break(Process *p, BeamInstr *pc, BpDataTime *bdt, Uint type) { + Uint ms,s,us; + process_breakpoint_time_t *pbt = NULL; + bp_data_time_item_t sitem, *item = NULL; + bp_time_hash_t *h = NULL; + BpDataTime *pbdt = NULL; + + ASSERT(p); + ASSERT(p->status == P_RUNNING); + + /* get previous timestamp and breakpoint + * from the process psd */ + + pbt = ERTS_PROC_GET_CALL_TIME(p); + get_sys_now(&ms,&s,&us); + + switch(type) { + /* get pbt + * timestamp = t0 + * lookup bdt from code + * set ts0 to pbt + * add call count here? + */ + case ERTS_BP_CALL_TIME_CALL: + case ERTS_BP_CALL_TIME_TAIL_CALL: + + if (pbt) { + ASSERT(pbt->pc); + /* add time to previous code */ + bp_time_diff(&sitem, pbt, ms, s, us); + sitem.pid = p->id; + sitem.count = 0; + + /* previous breakpoint */ + pbdt = (BpDataTime *) get_break(p, pbt->pc, (BeamInstr) BeamOp(op_i_time_breakpoint)); + + /* if null then the breakpoint was removed */ + if (pbdt) { + h = &(pbdt->hash[bp_sched2ix_proc(p)]); + + ASSERT(h); + ASSERT(h->item); + + item = bp_hash_get(h, &sitem); + if (!item) { + item = bp_hash_put(h, &sitem); + } else { + BP_TIME_ADD(item, &sitem); + } + } + + } else { + /* first call of process to instrumented function */ + pbt = Alloc(sizeof(process_breakpoint_time_t)); + (void *) ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCK_MAIN, pbt); + } + /* add count to this code */ + sitem.pid = p->id; + sitem.count = 1; + sitem.s_time = 0; + sitem.us_time = 0; + + /* this breakpoint */ + ASSERT(bdt); + h = &(bdt->hash[bp_sched2ix_proc(p)]); + + ASSERT(h); + ASSERT(h->item); + + item = bp_hash_get(h, &sitem); + if (!item) { + item = bp_hash_put(h, &sitem); + } else { + BP_TIME_ADD(item, &sitem); + } + + pbt->pc = pc; + pbt->ms = ms; + pbt->s = s; + pbt->us = us; + break; + + case ERTS_BP_CALL_TIME_RETURN: + /* get pbt + * lookup bdt from code + * timestamp = t1 + * get ts0 from pbt + * get item from bdt->hash[bp_hash(p->id)] + * ack diff (t1, t0) to item + */ + + if(pbt) { + /* might have been removed due to + * trace_pattern(false) + */ + ASSERT(pbt->pc); + + bp_time_diff(&sitem, pbt, ms, s, us); + sitem.pid = p->id; + sitem.count = 0; + + /* previous breakpoint */ + pbdt = (BpDataTime *) get_break(p, pbt->pc, (BeamInstr) BeamOp(op_i_time_breakpoint)); + + /* beware, the trace_pattern might have been removed */ + if (pbdt) { + h = &(pbdt->hash[bp_sched2ix_proc(p)]); + + ASSERT(h); + ASSERT(h->item); + + item = bp_hash_get(h, &sitem); + if (!item) { + item = bp_hash_put(h, &sitem); + } else { + BP_TIME_ADD(item, &sitem); + } + } + + pbt->pc = pc; + pbt->ms = ms; + pbt->s = s; + pbt->us = us; + } + break; + default : + ASSERT(0); + /* will never happen */ + break; + } +} /* ************************************************************************* @@ -440,7 +848,7 @@ erts_find_local_func(Eterm mfa[3]) { static int set_break(Eterm mfa[3], int specified, - Binary *match_spec, Eterm break_op, + Binary *match_spec, BeamInstr break_op, enum erts_break_op count_op, Eterm tracer_pid) { Module *modp; @@ -470,45 +878,54 @@ static int set_break(Eterm mfa[3], int specified, } static int set_module_break(Module *modp, Eterm mfa[3], int specified, - Binary *match_spec, Uint break_op, + Binary *match_spec, BeamInstr break_op, enum erts_break_op count_op, Eterm tracer_pid) { - Uint** code_base; - Uint* code_ptr; + BeamInstr** code_base; + BeamInstr* code_ptr; int num_processed = 0; Uint i,n; ASSERT(break_op); ASSERT(modp); - code_base = (Uint **) modp->code; + code_base = (BeamInstr **) modp->code; if (code_base == NULL) { return 0; } - n = (Uint) code_base[MI_NUM_FUNCTIONS]; + n = (BeamInstr) code_base[MI_NUM_FUNCTIONS]; for (i = 0; i < n; ++i) { code_ptr = code_base[MI_FUNCTIONS+i]; - ASSERT(code_ptr[0] == (Uint) BeamOp(op_i_func_info_IaaI)); + ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); if ((specified < 2 || mfa[1] == ((Eterm) code_ptr[3])) && (specified < 3 || ((int) mfa[2]) == ((int) code_ptr[4]))) { - Uint *pc = code_ptr+5; + BeamInstr *pc = code_ptr+5; num_processed += - set_function_break(modp, pc, match_spec, + set_function_break(modp, pc, BREAK_IS_ERL, match_spec, break_op, count_op, tracer_pid); } } return num_processed; } -static int set_function_break(Module *modp, Uint *pc, - Binary *match_spec, Uint break_op, +static int set_function_break(Module *modp, BeamInstr *pc, int bif, + Binary *match_spec, BeamInstr break_op, enum erts_break_op count_op, Eterm tracer_pid) { - BpData *bd, **r; + + BeamInstr **code_base = NULL; + BpData *bd, **r, ***rs; size_t size; - Uint **code_base = (Uint **)modp->code; + Uint ix = 0; - ASSERT(code_base); - ASSERT(code_base <= (Uint **)pc); - ASSERT((Uint **)pc < code_base + (modp->code_length/sizeof(Uint *))); + if (bif == BREAK_IS_ERL) { + code_base = (BeamInstr **)modp->code; + ASSERT(code_base); + ASSERT(code_base <= (BeamInstr **)pc); + ASSERT((BeamInstr **)pc < code_base + (modp->code_length/sizeof(BeamInstr *))); + } else { + ASSERT(*pc == (BeamInstr) em_apply_bif); + ASSERT(modp == NULL); + } + /* * Currently no trace support for native code. */ @@ -517,8 +934,9 @@ static int set_function_break(Module *modp, Uint *pc, } /* Do not allow two breakpoints of the same kind */ if ( (bd = is_break(pc, break_op))) { - if (break_op == (Uint) BeamOp(op_i_trace_breakpoint) - || break_op == (Uint) BeamOp(op_i_mtrace_breakpoint)) { + if (break_op == (BeamInstr) BeamOp(op_i_trace_breakpoint) + || break_op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) { + BpDataTrace *bdt = (BpDataTrace *) bd; Binary *old_match_spec; @@ -531,71 +949,115 @@ static int set_function_break(Module *modp, Uint *pc, ErtsSmpBPUnlock(bdt); MatchSetUnref(old_match_spec); } else { + BpDataCount *bdc = (BpDataCount *) bd; + long count = 0; + long res = 0; + ASSERT(! match_spec); ASSERT(is_nil(tracer_pid)); - if (break_op == (Uint) BeamOp(op_i_count_breakpoint)) { - BpDataCount *bdc = (BpDataCount *) bd; - ErtsSmpBPLock(bdc); + if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) { if (count_op == erts_break_stop) { - if (bdc->count >= 0) { - bdc->count = -bdc->count-1; /* Stop call counter */ + count = erts_smp_atomic_read(&bdc->acount); + if (count >= 0) { + while(1) { + res = erts_smp_atomic_cmpxchg(&bdc->acount, -count - 1, count); + if ((res == count) || count < 0) break; + count = res; + } } } else { - bdc->count = 0; /* Reset call counter */ + /* Reset call counter */ + erts_smp_atomic_set(&bdc->acount, 0); } - ErtsSmpBPUnlock(bdc); + + } else if (break_op == (BeamInstr) BeamOp(op_i_time_breakpoint)) { + BpDataTime *bdt = (BpDataTime *) bd; + Uint i = 0; + + ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + + if (count_op == erts_break_stop) { + bdt->pause = 1; + } else { + bdt->pause = 0; + for (i = 0; i < bdt->n; i++) { + bp_hash_delete(&(bdt->hash[i])); + bp_hash_init(&(bdt->hash[i]), 32); + } + } + } else { ASSERT (! count_op); } } return 1; } - if (break_op == (Uint) BeamOp(op_i_trace_breakpoint) || - break_op == (Uint) BeamOp(op_i_mtrace_breakpoint)) { + if (break_op == (BeamInstr) BeamOp(op_i_trace_breakpoint) || + break_op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) { size = sizeof(BpDataTrace); } else { ASSERT(! match_spec); ASSERT(is_nil(tracer_pid)); - if (break_op == (Uint) BeamOp(op_i_count_breakpoint)) { - if (count_op == erts_break_reset - || count_op == erts_break_stop) { + if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) { + if (count_op == erts_break_reset || count_op == erts_break_stop) { /* Do not insert a new breakpoint */ return 1; } size = sizeof(BpDataCount); + } else if (break_op == (BeamInstr) BeamOp(op_i_time_breakpoint)) { + if (count_op == erts_break_reset || count_op == erts_break_stop) { + /* Do not insert a new breakpoint */ + return 1; + } + size = sizeof(BpDataTime); } else { ASSERT(! count_op); - ASSERT(break_op == (Uint) BeamOp(op_i_debug_breakpoint)); + ASSERT(break_op == (BeamInstr) BeamOp(op_i_debug_breakpoint)); size = sizeof(BpDataDebug); } } - r = (BpData **) (pc-4); + rs = (BpData ***) (pc-4); + if (! *rs) { + size_t ssize = sizeof(BeamInstr) * erts_no_schedulers; + *rs = (BpData **) Alloc(ssize); + sys_memzero(*rs, ssize); + } + + r = &((*rs)[0]); + if (! *r) { - ASSERT(*pc != (Uint) BeamOp(op_i_trace_breakpoint)); - ASSERT(*pc != (Uint) BeamOp(op_i_mtrace_breakpoint)); - ASSERT(*pc != (Uint) BeamOp(op_i_debug_breakpoint)); - ASSERT(*pc != (Uint) BeamOp(op_i_count_breakpoint)); + ASSERT(*pc != (BeamInstr) BeamOp(op_i_trace_breakpoint)); + ASSERT(*pc != (BeamInstr) BeamOp(op_i_mtrace_breakpoint)); + ASSERT(*pc != (BeamInstr) BeamOp(op_i_debug_breakpoint)); + ASSERT(*pc != (BeamInstr) BeamOp(op_i_count_breakpoint)); + ASSERT(*pc != (BeamInstr) BeamOp(op_i_time_breakpoint)); /* First breakpoint; create singleton ring */ bd = Alloc(size); BpInit(bd, *pc); - *pc = break_op; *r = bd; + if (bif == BREAK_IS_ERL) { + *pc = break_op; + } } else { - ASSERT(*pc == (Uint) BeamOp(op_i_trace_breakpoint) || - *pc == (Uint) BeamOp(op_i_mtrace_breakpoint) || - *pc == (Uint) BeamOp(op_i_debug_breakpoint) || - *pc == (Uint) BeamOp(op_i_count_breakpoint)); - if (*pc == (Uint) BeamOp(op_i_debug_breakpoint)) { + ASSERT(*pc == (BeamInstr) BeamOp(op_i_trace_breakpoint) || + *pc == (BeamInstr) BeamOp(op_i_mtrace_breakpoint) || + *pc == (BeamInstr) BeamOp(op_i_debug_breakpoint) || + *pc == (BeamInstr) BeamOp(op_i_time_breakpoint) || + *pc == (BeamInstr) BeamOp(op_i_count_breakpoint) || + *pc == (BeamInstr) em_apply_bif); + if (*pc == (BeamInstr) BeamOp(op_i_debug_breakpoint)) { /* Debug bp must be last, so if it is also first; * it must be singleton. */ - ASSERT(BpSingleton(*r)); + ASSERT(BpSingleton(*r)); /* Insert new bp first in the ring, i.e second to last. */ bd = Alloc(size); BpInitAndSpliceNext(bd, *pc, *r); - *pc = break_op; - } else if ((*r)->prev->orig_instr - == (Uint) BeamOp(op_i_debug_breakpoint)) { + if (bif == BREAK_IS_ERL) { + *pc = break_op; + } + } else if ((*r)->prev->orig_instr + == (BeamInstr) BeamOp(op_i_debug_breakpoint)) { /* Debug bp last in the ring; insert new second to last. */ bd = Alloc(size); BpInitAndSplicePrev(bd, (*r)->prev->orig_instr, *r); @@ -608,25 +1070,43 @@ static int set_function_break(Module *modp, Uint *pc, *r = bd; } } + for (ix = 1; ix < erts_no_schedulers; ++ix) { + (*rs)[ix] = (*rs)[0]; + } + + bd->this_instr = break_op; /* Init the bp type specific data */ - if (break_op == (Uint) BeamOp(op_i_trace_breakpoint) || - break_op == (Uint) BeamOp(op_i_mtrace_breakpoint)) { + if (break_op == (BeamInstr) BeamOp(op_i_trace_breakpoint) || + break_op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) { BpDataTrace *bdt = (BpDataTrace *) bd; MatchSetRef(match_spec); bdt->match_spec = match_spec; bdt->tracer_pid = tracer_pid; - } else if (break_op == (Uint) BeamOp(op_i_count_breakpoint)) { + } else if (break_op == (BeamInstr) BeamOp(op_i_time_breakpoint)) { + BpDataTime *bdt = (BpDataTime *) bd; + Uint i = 0; + + bdt->pause = 0; + bdt->n = erts_no_schedulers; + bdt->hash = Alloc(sizeof(bp_time_hash_t)*(bdt->n)); + + for (i = 0; i < bdt->n; i++) { + bp_hash_init(&(bdt->hash[i]), 32); + } + } else if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) { BpDataCount *bdc = (BpDataCount *) bd; + erts_smp_atomic_init(&bdc->acount, 0); + } - bdc->count = 0; + if (bif == BREAK_IS_ERL) { + ++(*(BeamInstr*)&code_base[MI_NUM_BREAKPOINTS]); } - ++(*(Uint*)&code_base[MI_NUM_BREAKPOINTS]); return 1; } -static int clear_break(Eterm mfa[3], int specified, Uint break_op) +static int clear_break(Eterm mfa[3], int specified, BeamInstr break_op) { int num_processed = 0; Module *modp; @@ -652,52 +1132,71 @@ static int clear_break(Eterm mfa[3], int specified, Uint break_op) } static int clear_module_break(Module *m, Eterm mfa[3], int specified, - Uint break_op) { - Uint** code_base; - Uint* code_ptr; + BeamInstr break_op) { + BeamInstr** code_base; + BeamInstr* code_ptr; int num_processed = 0; - Uint i,n; + Uint i; + BeamInstr n; ASSERT(m); - code_base = (Uint **) m->code; + code_base = (BeamInstr **) m->code; if (code_base == NULL) { return 0; } - n = (Uint) code_base[MI_NUM_FUNCTIONS]; + n = (BeamInstr) code_base[MI_NUM_FUNCTIONS]; for (i = 0; i < n; ++i) { code_ptr = code_base[MI_FUNCTIONS+i]; if ((specified < 2 || mfa[1] == ((Eterm) code_ptr[3])) && (specified < 3 || ((int) mfa[2]) == ((int) code_ptr[4]))) { - Uint *pc = code_ptr + 5; + BeamInstr *pc = code_ptr + 5; num_processed += - clear_function_break(m, pc, break_op); + clear_function_break(m, pc, BREAK_IS_ERL, break_op); } } return num_processed; } -static int clear_function_break(Module *m, Uint *pc, Uint break_op) { +static int clear_function_break(Module *m, BeamInstr *pc, int bif, BeamInstr break_op) { BpData *bd; - Uint **code_base = (Uint **)m->code; - - ASSERT(code_base); - ASSERT(code_base <= (Uint **)pc); - ASSERT((Uint **)pc < code_base + (m->code_length/sizeof(Uint *))); + Uint ix = 0; + BeamInstr **code_base = NULL; + + if (bif == BREAK_IS_ERL) { + code_base = (BeamInstr **)m->code; + ASSERT(code_base); + ASSERT(code_base <= (BeamInstr **)pc); + ASSERT((BeamInstr **)pc < code_base + (m->code_length/sizeof(BeamInstr *))); + } else { + ASSERT(*pc == (BeamInstr) em_apply_bif); + ASSERT(m == NULL); + } + /* * Currently no trace support for native code. */ if (erts_is_native_break(pc)) { return 0; } + while ( (bd = is_break(pc, break_op))) { /* Remove all breakpoints of this type. * There should be only one of each type, * but break_op may be 0 which matches any type. */ - Uint op; - BpData **r = (BpData **) (pc-4); + BeamInstr op; + BpData ***rs = (BpData ***) (pc - 4); + BpData **r = NULL; + +#ifdef DEBUG + for (ix = 1; ix < erts_no_schedulers; ++ix) { + ASSERT((*rs)[ix] == (*rs)[0]); + } +#endif + r = &((*rs)[0]); + ASSERT(*r); /* Find opcode for this breakpoint */ if (break_op) { @@ -713,8 +1212,11 @@ static int clear_function_break(Module *m, Uint *pc, Uint break_op) { if (BpSingleton(bd)) { ASSERT(*r == bd); /* Only one breakpoint to remove */ - *r = NULL; - *pc = bd->orig_instr; + if (bif == BREAK_IS_ERL) { + *pc = bd->orig_instr; + } + Free(*rs); + *rs = NULL; } else { BpData *bd_prev = bd->prev; @@ -726,22 +1228,64 @@ static int clear_function_break(Module *m, Uint *pc, Uint break_op) { bd_prev->orig_instr = bd->orig_instr; } else if (bd_prev == *r) { /* We removed the first breakpoint in the ring */ - *pc = bd->orig_instr; + if (bif == BREAK_IS_ERL) { + *pc = bd->orig_instr; + } } else { bd_prev->orig_instr = bd->orig_instr; } } - if (op == (Uint) BeamOp(op_i_trace_breakpoint) || - op == (Uint) BeamOp(op_i_mtrace_breakpoint)) { + if (op == (BeamInstr) BeamOp(op_i_trace_breakpoint) || + op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) { BpDataTrace *bdt = (BpDataTrace *) bd; - MatchSetUnref(bdt->match_spec); } + if (op == (BeamInstr) BeamOp(op_i_time_breakpoint)) { + BpDataTime *bdt = (BpDataTime *) bd; + Uint i = 0; + Uint j = 0; + Process *h_p = NULL; + bp_data_time_item_t *item = NULL; + process_breakpoint_time_t *pbt = NULL; + + /* remove all psd associated with the hash + * and then delete the hash. + * ... sigh ... + */ + + for( i = 0; i < bdt->n; ++i) { + if (bdt->hash[i].used) { + for (j = 0; j < bdt->hash[i].n; ++j) { + item = &(bdt->hash[i].item[j]); + if (item->pid != NIL) { + h_p = process_tab[internal_pid_index(item->pid)]; + if (h_p) { + pbt = ERTS_PROC_SET_CALL_TIME(h_p, ERTS_PROC_LOCK_MAIN, NULL); + if (pbt) { + Free(pbt); + } + } + } + } + } + bp_hash_delete(&(bdt->hash[i])); + } + Free(bdt->hash); + bdt->hash = NULL; + bdt->n = 0; + } Free(bd); - ASSERT(((Uint) code_base[MI_NUM_BREAKPOINTS]) > 0); - --(*(Uint*)&code_base[MI_NUM_BREAKPOINTS]); - } + if (bif == BREAK_IS_ERL) { + ASSERT(((BeamInstr) code_base[MI_NUM_BREAKPOINTS]) > 0); + --(*(BeamInstr*)&code_base[MI_NUM_BREAKPOINTS]); + } + if (*rs) { + for (ix = 1; ix < erts_no_schedulers; ++ix) { + (*rs)[ix] = (*rs)[0]; + } + } + } /* while bd != NULL */ return 1; } @@ -754,32 +1298,63 @@ static int clear_function_break(Module *m, Uint *pc, Uint break_op) { ** returned. The program counter must point to the first executable ** (breakpoint) instruction of the function. */ -static BpData *is_break(Uint *pc, Uint break_op) { - ASSERT(pc[-5] == (Uint) BeamOp(op_i_func_info_IaaI)); + +BpData *erts_get_time_break(Process *p, BeamInstr *pc) { + return get_break(p, pc, (BeamInstr) BeamOp(op_i_time_breakpoint)); +} + +static BpData *get_break(Process *p, BeamInstr *pc, BeamInstr break_op) { + ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); if (! erts_is_native_break(pc)) { - BpData *bd = (BpData *) pc[-4]; - - if (break_op == 0) { - return bd; - } - if (*pc == break_op) { - ASSERT(bd); - return bd->next; - } - if (! bd){ + BpData **rs = (BpData **) pc[-4]; + BpData *bd = NULL, *ebd = NULL; + + if (! rs) { return NULL; } + + bd = ebd = rs[bp_sched2ix_proc(p)]; + ASSERT(bd); + if (bd->this_instr == break_op) { + return bd; + } + bd = bd->next; - while (bd != (BpData *) pc[-4]) { + while (bd != ebd) { ASSERT(bd); - if (bd->orig_instr == break_op) { - bd = bd->next; + if (bd->this_instr == break_op) { ASSERT(bd); return bd; - } else { - bd = bd->next; } + bd = bd->next; } } return NULL; } + +static BpData *is_break(BeamInstr *pc, BeamInstr break_op) { + BpData **rs = (BpData **) pc[-4]; + BpData *bd = NULL, *ebd = NULL; + ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + + if (! rs) { + return NULL; + } + + bd = ebd = rs[bp_sched2ix()]; + ASSERT(bd); + if ( (break_op == 0) || (bd->this_instr == break_op)) { + return bd; + } + + bd = bd->next; + while (bd != ebd) { + ASSERT(bd); + if (bd->this_instr == break_op) { + ASSERT(bd); + return bd; + } + bd = bd->next; + } + return NULL; +} diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 44e6b294d8..ebc171078d 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2000-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -27,28 +27,46 @@ -/* -** Common struct to all bp_data_* -** -** Two gotchas: -** -** 1) The type of bp_data structure in the ring is deduced from the -** orig_instr field of the structure _before_ in the ring, except for -** the first structure in the ring that has its instruction in -** pc[0] of the code to execute. -** -** 2) pc[-4] points to the _last_ structure in the ring before the -** breakpoints are being executed. -** -** So, as an example, when a breakpointed function starts to execute, -** the first instruction that is a breakpoint instruction at pc[0] finds -** its data at ((BpData *) pc[-4])->next and has to cast that pointer -** to the correct bp_data type. +/* A couple of gotchas: + * + * The breakpoint structure from BeamInstr, + * In beam_emu where the instruction counter pointer, I (or pc), + * points to the *current* instruction. At that time, if the instruction + * is a breakpoint instruction the pc looks like the following, + * + * I[-5] | op_i_func_info_IaaI | scheduler specific entries + * I[-4] | BpData** bpa | --> | BpData * bdas1 | ... | BpData * bdasN | + * I[-3] | Tagged Module | | | + * I[-2] | Tagged Function | V V + * I[-1] | Arity | BpData -> BpData -> BpData -> BpData + * I[0] | The bp instruction | ^ * the bp wheel * | + * |------------------------------ + * + * Common struct to all bp_data_* + * + * 1) The type of bp_data structure in the ring is deduced from the + * orig_instr field of the structure _before_ in the ring, except for + * the first structure in the ring that has its instruction in + * pc[0] of the code to execute. + * This is valid as long as you don't search for the function while it is + * being executed by something else. Or is in the middle of its rotation for + * any other reason. + * A key, the bp beam instruction, is included for this reason. + * + * 2) pc[-4][sched_id - 1] points to the _last_ structure in the ring before the + * breakpoints are being executed. + * + * So, as an example, when a breakpointed function starts to execute, + * the first instruction that is a breakpoint instruction at pc[0] finds + * its data at ((BpData **) pc[-4][sched_id - 1])->next and has to cast that pointer + * to the correct bp_data type. */ + typedef struct bp_data { struct bp_data *next; /* Doubly linked ring pointers */ struct bp_data *prev; /* -"- */ - Uint orig_instr; /* The original instruction to execute */ + BeamInstr orig_instr; /* The original instruction to execute */ + BeamInstr this_instr; /* key */ } BpData; /* ** All the following bp_data_.. structs must begin the same way @@ -57,7 +75,8 @@ typedef struct bp_data { typedef struct bp_data_trace { struct bp_data *next; struct bp_data *prev; - Uint orig_instr; + BeamInstr orig_instr; + BeamInstr this_instr; /* key */ Binary *match_spec; Eterm tracer_pid; } BpDataTrace; @@ -65,18 +84,58 @@ typedef struct bp_data_trace { typedef struct bp_data_debug { struct bp_data *next; struct bp_data *prev; - Uint orig_instr; + BeamInstr orig_instr; + BeamInstr this_instr; /* key */ } BpDataDebug; -typedef struct bp_data_count { /* Call count */ +typedef struct bp_data_count { /* Call count */ struct bp_data *next; struct bp_data *prev; - Uint orig_instr; - Sint count; + BeamInstr orig_instr; + BeamInstr this_instr; /* key */ + erts_smp_atomic_t acount; } BpDataCount; +typedef struct { + Eterm pid; + Sint count; + Uint s_time; + Uint us_time; +} bp_data_time_item_t; + +typedef struct { + Uint n; + Uint used; + bp_data_time_item_t *item; +} bp_time_hash_t; + +typedef struct bp_data_time { /* Call time */ + struct bp_data *next; + struct bp_data *prev; + BeamInstr orig_instr; + BeamInstr this_instr; /* key */ + Uint pause; + Uint n; + bp_time_hash_t *hash; +} BpDataTime; + +typedef struct { + Uint ms; + Uint s; + Uint us; + BeamInstr *pc; +} process_breakpoint_time_t; /* used within psd */ + extern erts_smp_spinlock_t erts_bp_lock; +#define ERTS_BP_CALL_TIME_SCHEDULE_IN (0) +#define ERTS_BP_CALL_TIME_SCHEDULE_OUT (1) +#define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2) + +#define ERTS_BP_CALL_TIME_CALL (0) +#define ERTS_BP_CALL_TIME_RETURN (1) +#define ERTS_BP_CALL_TIME_TAIL_CALL (2) + #ifdef ERTS_SMP #define ErtsSmpBPLock(BDC) erts_smp_spin_lock(&erts_bp_lock) #define ErtsSmpBPUnlock(BDC) erts_smp_spin_unlock(&erts_bp_lock) @@ -85,31 +144,46 @@ extern erts_smp_spinlock_t erts_bp_lock; #define ErtsSmpBPUnlock(BDC) #endif -#define ErtsCountBreak(pc,instr_result) \ -do { \ - BpDataCount *bdc = (BpDataCount *) (pc)[-4]; \ - \ - ASSERT((pc)[-5] == (Uint) BeamOp(op_i_func_info_IaaI)); \ - ASSERT(bdc); \ - bdc = (BpDataCount *) bdc->next; \ - ASSERT(bdc); \ - (pc)[-4] = (Uint) bdc; \ - ErtsSmpBPLock(bdc); \ - if (bdc->count >= 0) bdc->count++; \ - ErtsSmpBPUnlock(bdc); \ - *(instr_result) = bdc->orig_instr; \ +ERTS_INLINE Uint bp_sched2ix(void); + +#ifdef ERTS_SMP +#define bp_sched2ix_proc(p) ((p)->scheduler_data->no - 1) +#else +#define bp_sched2ix_proc(p) (0) +#endif + +#define ErtsCountBreak(p, pc,instr_result) \ +do { \ + BpData **bds = (BpData **) (pc)[-4]; \ + BpDataCount *bdc = NULL; \ + Uint ix = bp_sched2ix_proc( (p) ); \ + long count = 0; \ + \ + ASSERT((pc)[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); \ + ASSERT(bds); \ + bdc = (BpDataCount *) bds[ix]; \ + bdc = (BpDataCount *) bdc->next; \ + ASSERT(bdc); \ + bds[ix] = (BpData *) bdc; \ + count = erts_smp_atomic_read(&bdc->acount); \ + if (count >= 0) erts_smp_atomic_inc(&bdc->acount); \ + *(instr_result) = bdc->orig_instr; \ } while (0) -#define ErtsBreakSkip(pc,instr_result) \ -do { \ - BpData *bd = (BpData *) (pc)[-4]; \ - \ - ASSERT((pc)[-5] == (Uint) BeamOp(op_i_func_info_IaaI)); \ - ASSERT(bd); \ - bd = bd->next; \ - ASSERT(bd); \ - (pc)[-4] = (Uint) bd; \ - *(instr_result) = bd->orig_instr; \ +#define ErtsBreakSkip(p, pc,instr_result) \ +do { \ + BpData **bds = (BpData **) (pc)[-4]; \ + BpData *bd = NULL; \ + Uint ix = bp_sched2ix_proc( (p) ); \ + \ + ASSERT((pc)[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); \ + ASSERT(bds); \ + bd = bds[ix]; \ + ASSERT(bd); \ + bd = bd->next; \ + ASSERT(bd); \ + bds[ix] = bd; \ + *(instr_result) = bd->orig_instr; \ } while (0) enum erts_break_op{ @@ -133,9 +207,9 @@ int erts_clear_trace_break(Eterm mfa[3], int specified); int erts_set_mtrace_break(Eterm mfa[3], int specified, Binary *match_spec, Eterm tracer_pid); int erts_clear_mtrace_break(Eterm mfa[3], int specified); -void erts_set_mtrace_bif(Uint *pc, Binary *match_spec, +void erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, Eterm tracer_pid); -void erts_clear_mtrace_bif(Uint *pc); +void erts_clear_mtrace_bif(BeamInstr *pc); int erts_set_debug_break(Eterm mfa[3], int specified); int erts_clear_debug_break(Eterm mfa[3], int specified); int erts_set_count_break(Eterm mfa[3], int specified, enum erts_break_op); @@ -144,22 +218,33 @@ int erts_clear_count_break(Eterm mfa[3], int specified); int erts_clear_break(Eterm mfa[3], int specified); int erts_clear_module_break(Module *modp); -int erts_clear_function_break(Module *modp, Uint *pc); +int erts_clear_function_break(Module *modp, BeamInstr *pc); -Uint erts_trace_break(Process *p, Uint *pc, Eterm *args, +BeamInstr erts_trace_break(Process *p, BeamInstr *pc, Eterm *args, Uint32 *ret_flags, Eterm *tracer_pid); -Uint32 erts_bif_mtrace(Process *p, Uint *pc, Eterm *args, +Uint32 erts_bif_mtrace(Process *p, BeamInstr *pc, Eterm *args, int local, Eterm *tracer_pid); -int erts_is_trace_break(Uint *pc, Binary **match_spec_ret, +int erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret); -int erts_is_mtrace_break(Uint *pc, Binary **match_spec_ret, +int erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, Eterm *tracer_pid_rte); -int erts_is_mtrace_bif(Uint *pc, Binary **match_spec_ret, +int erts_is_mtrace_bif(BeamInstr *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret); -int erts_is_native_break(Uint *pc); -int erts_is_count_break(Uint *pc, Sint *count_ret); +int erts_is_native_break(BeamInstr *pc); +int erts_is_count_break(BeamInstr *pc, Sint *count_ret); +int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *call_time); + +void erts_trace_time_break(Process *p, BeamInstr *pc, BpDataTime *bdt, Uint type); +void erts_schedule_time_break(Process *p, Uint out); +int erts_set_time_break(Eterm mfa[3], int specified, enum erts_break_op); +int erts_clear_time_break(Eterm mfa[3], int specified); + +int erts_is_time_trace_bif(Process *p, BeamInstr *pc, Eterm *call_time); +void erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op); +void erts_clear_time_trace_bif(BeamInstr *pc); +BpData *erts_get_time_break(Process *p, BeamInstr *pc); -Uint *erts_find_local_func(Eterm mfa[3]); +BeamInstr *erts_find_local_func(Eterm mfa[3]); #endif /* _BEAM_BP_H */ diff --git a/erts/emulator/beam/beam_catches.c b/erts/emulator/beam/beam_catches.c index d5cef1cad2..e795b4efbd 100644 --- a/erts/emulator/beam/beam_catches.c +++ b/erts/emulator/beam/beam_catches.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2000-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -26,7 +26,7 @@ /* XXX: should use dynamic reallocation */ #define TABSIZ (16*1024) static struct { - Eterm *cp; + BeamInstr *cp; unsigned cdr; } beam_catches[TABSIZ]; @@ -39,7 +39,7 @@ void beam_catches_init(void) high_mark = 0; } -unsigned beam_catches_cons(Eterm *cp, unsigned cdr) +unsigned beam_catches_cons(BeamInstr *cp, unsigned cdr) { int i; @@ -65,7 +65,7 @@ unsigned beam_catches_cons(Eterm *cp, unsigned cdr) return i; } -Eterm *beam_catches_car(unsigned i) +BeamInstr *beam_catches_car(unsigned i) { if( i >= TABSIZ ) { fprintf(stderr, @@ -75,7 +75,7 @@ Eterm *beam_catches_car(unsigned i) return beam_catches[i].cp; } -void beam_catches_delmod(unsigned head, Eterm *code, unsigned code_bytes) +void beam_catches_delmod(unsigned head, BeamInstr *code, unsigned code_bytes) { unsigned i, cdr; diff --git a/erts/emulator/beam/beam_catches.h b/erts/emulator/beam/beam_catches.h index ccf33d5e86..6223427f0d 100644 --- a/erts/emulator/beam/beam_catches.h +++ b/erts/emulator/beam/beam_catches.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2000-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -23,9 +23,9 @@ #define BEAM_CATCHES_NIL (-1) void beam_catches_init(void); -unsigned beam_catches_cons(Eterm* cp, unsigned cdr); -Eterm *beam_catches_car(unsigned i); -void beam_catches_delmod(unsigned head, Eterm* code, unsigned code_bytes); +unsigned beam_catches_cons(BeamInstr* cp, unsigned cdr); +BeamInstr *beam_catches_car(unsigned i); +void beam_catches_delmod(unsigned head, BeamInstr* code, unsigned code_bytes); #define catch_pc(x) beam_catches_car(catch_val((x))) diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 4242a4161e..b0bf14b94f 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1998-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1998-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -43,12 +43,13 @@ #else # define HEXF "%08bpX" #endif +#define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm)))) void dbg_bt(Process* p, Eterm* sp); -void dbg_where(Eterm* addr, Eterm x0, Eterm* reg); +void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg); static void print_big(int to, void *to_arg, Eterm* addr); -static int print_op(int to, void *to_arg, int op, int size, Eterm* addr); +static int print_op(int to, void *to_arg, int op, int size, BeamInstr* addr); Eterm erts_debug_same_2(Process* p, Eterm term1, Eterm term2) { @@ -124,6 +125,38 @@ erts_debug_breakpoint_2(Process* p, Eterm MFA, Eterm bool) BIF_ERROR(p, BADARG); } +#if 0 /* Kept for conveninence when hard debugging. */ +void debug_dump_code(BeamInstr *I, int num) +{ + BeamInstr *code_ptr = I; + BeamInstr *end = code_ptr + num; + erts_dsprintf_buf_t *dsbufp; + BeamInstr instr; + int i; + + dsbufp = erts_create_tmp_dsbuf(0); + while (code_ptr < end) { + erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr); + instr = (BeamInstr) code_ptr[0]; + for (i = 0; i < NUM_SPECIFIC_OPS; i++) { + if (instr == (BeamInstr) BeamOp(i) && opc[i].name[0] != '\0') { + code_ptr += print_op(ERTS_PRINT_DSBUF, (void *) dsbufp, + i, opc[i].sz-1, code_ptr+1) + 1; + break; + } + } + if (i >= NUM_SPECIFIC_OPS) { + erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, + "unknown " HEXF "\n", instr); + code_ptr++; + } + } + dsbufp->str[dsbufp->str_len] = 0; + erts_fprintf(stderr,"%s", dsbufp->str); + erts_destroy_tmp_dsbuf(dsbufp); +} +#endif + Eterm erts_debug_disassemble_1(Process* p, Eterm addr) { @@ -132,16 +165,16 @@ erts_debug_disassemble_1(Process* p, Eterm addr) Eterm* tp; Eterm bin; Eterm mfa; - Eterm* funcinfo = NULL; /* Initialized to eliminate warning. */ - Uint* code_base; - Uint* code_ptr = NULL; /* Initialized to eliminate warning. */ - Uint instr; - Uint uaddr; + BeamInstr* funcinfo = NULL; /* Initialized to eliminate warning. */ + BeamInstr* code_base; + BeamInstr* code_ptr = NULL; /* Initialized to eliminate warning. */ + BeamInstr instr; + BeamInstr uaddr; Uint hsz; int i; - if (term_to_Uint(addr, &uaddr)) { - code_ptr = (Uint *) uaddr; + if (term_to_UWord(addr, &uaddr)) { + code_ptr = (BeamInstr *) uaddr; if ((funcinfo = find_function_from_pc(code_ptr)) == NULL) { BIF_RET(am_false); } @@ -180,14 +213,14 @@ erts_debug_disassemble_1(Process* p, Eterm addr) * But this code_ptr will point to the start of the Export, * not the function's func_info instruction. BOOM !? */ - code_ptr = ((Eterm *) ep->address) - 5; + code_ptr = ((BeamInstr *) ep->address) - 5; funcinfo = code_ptr+2; } else if (modp == NULL || (code_base = modp->code) == NULL) { BIF_RET(am_undef); } else { n = code_base[MI_NUM_FUNCTIONS]; for (i = 0; i < n; i++) { - code_ptr = (Uint *) code_base[MI_FUNCTIONS+i]; + code_ptr = (BeamInstr *) code_base[MI_FUNCTIONS+i]; if (code_ptr[3] == name && code_ptr[4] == arity) { funcinfo = code_ptr+2; break; @@ -203,9 +236,9 @@ erts_debug_disassemble_1(Process* p, Eterm addr) dsbufp = erts_create_tmp_dsbuf(0); erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr); - instr = (Uint) code_ptr[0]; + instr = (BeamInstr) code_ptr[0]; for (i = 0; i < NUM_SPECIFIC_OPS; i++) { - if (instr == (Uint) BeamOp(i) && opc[i].name[0] != '\0') { + if (instr == (BeamInstr) BeamOp(i) && opc[i].name[0] != '\0') { code_ptr += print_op(ERTS_PRINT_DSBUF, (void *) dsbufp, i, opc[i].sz-1, code_ptr+1) + 1; break; @@ -219,12 +252,12 @@ erts_debug_disassemble_1(Process* p, Eterm addr) bin = new_binary(p, (byte *) dsbufp->str, (int) dsbufp->str_len); erts_destroy_tmp_dsbuf(dsbufp); hsz = 4+4; - (void) erts_bld_uint(NULL, &hsz, (Uint) code_ptr); + (void) erts_bld_uword(NULL, &hsz, (BeamInstr) code_ptr); hp = HAlloc(p, hsz); - addr = erts_bld_uint(&hp, NULL, (Uint) code_ptr); + addr = erts_bld_uword(&hp, NULL, (BeamInstr) code_ptr); ASSERT(is_atom(funcinfo[0])); ASSERT(is_atom(funcinfo[1])); - mfa = TUPLE3(hp, funcinfo[0], funcinfo[1], make_small(funcinfo[2])); + mfa = TUPLE3(hp, (Eterm) funcinfo[0], (Eterm) funcinfo[1], make_small((Eterm) funcinfo[2])); hp += 4; return TUPLE3(hp, addr, bin, mfa); } @@ -236,20 +269,20 @@ dbg_bt(Process* p, Eterm* sp) while (sp < stack) { if (is_CP(*sp)) { - Eterm* addr = find_function_from_pc(cp_val(*sp)); + BeamInstr* addr = find_function_from_pc(cp_val(*sp)); if (addr) erts_fprintf(stderr, HEXF ": %T:%T/%bpu\n", - addr, addr[0], addr[1], addr[2]); + addr, (Eterm) addr[0], (Eterm) addr[1], (Uint) addr[2]); } sp++; } } void -dbg_where(Eterm* addr, Eterm x0, Eterm* reg) +dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg) { - Eterm* f = find_function_from_pc(addr); + BeamInstr* f = find_function_from_pc(addr); if (f == NULL) { erts_fprintf(stderr, "???\n"); @@ -259,7 +292,7 @@ dbg_where(Eterm* addr, Eterm x0, Eterm* reg) addr = f; arity = addr[2]; - erts_fprintf(stderr, HEXF ": %T:%T(", addr, addr[0], addr[1]); + erts_fprintf(stderr, HEXF ": %T:%T(", addr, (Eterm) addr[0], (Eterm) addr[1]); for (i = 0; i < arity; i++) erts_fprintf(stderr, i ? ", %T" : "%T", i ? reg[i] : x0); erts_fprintf(stderr, ")\n"); @@ -267,18 +300,18 @@ dbg_where(Eterm* addr, Eterm x0, Eterm* reg) } static int -print_op(int to, void *to_arg, int op, int size, Eterm* addr) +print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) { int i; - Uint tag; + BeamInstr tag; char* sign; char* start_prog; /* Start of program for packer. */ char* prog; /* Current position in packer program. */ - Uint stack[8]; /* Stack for packer. */ - Uint* sp = stack; /* Points to next free position. */ - Uint packed = 0; /* Accumulator for packed operations. */ - Uint args[8]; /* Arguments for this instruction. */ - Uint* ap; /* Pointer to arguments. */ + BeamInstr stack[8]; /* Stack for packer. */ + BeamInstr* sp = stack; /* Points to next free position. */ + BeamInstr packed = 0; /* Accumulator for packed operations. */ + BeamInstr args[8]; /* Arguments for this instruction. */ + BeamInstr* ap; /* Pointer to arguments. */ start_prog = opc[op].pack; @@ -288,7 +321,7 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr) * Avoid copying because instructions containing bignum operands * are bigger than actually declared. */ - ap = (Uint *) addr; + ap = (BeamInstr *) addr; } else { /* * Copy all arguments to a local buffer for the unpacking. @@ -324,8 +357,8 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr) packed >>= BEAM_TIGHT_SHIFT; break; case '6': /* Shift 16 steps */ - *ap++ = packed & 0xffff; - packed >>= 16; + *ap++ = packed & BEAM_LOOSE_MASK; + packed >>= BEAM_LOOSE_SHIFT; break; case 'p': *sp++ = *--ap; @@ -390,11 +423,11 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr) case 'i': /* Tagged integer */ case 'c': /* Tagged constant */ case 'q': /* Tagged literal */ - erts_print(to, to_arg, "%T", *ap); + erts_print(to, to_arg, "%T", (Eterm) *ap); ap++; break; case 'A': - erts_print(to, to_arg, "%d", arityval(ap[0])); + erts_print(to, to_arg, "%d", arityval( (Eterm) ap[0])); ap++; break; case 'd': /* Destination (x(0), x(N), y(N)) */ @@ -421,30 +454,36 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr) ap++; break; case 'f': /* Destination label */ - erts_print(to, to_arg, "f(%X)", *ap); - ap++; + { + BeamInstr* f = find_function_from_pc((BeamInstr *)*ap); + if (f+3 != (BeamInstr *) *ap) { + erts_print(to, to_arg, "f(" HEXF ")", *ap); + } else { + erts_print(to, to_arg, "%T:%T/%bpu", (Eterm) f[0], (Eterm) f[1], (Eterm) f[2]); + } + ap++; + } break; case 'p': /* Pointer (to label) */ { - Eterm* f = find_function_from_pc((Eterm *)*ap); - - if (f+3 != (Eterm *) *ap) { - erts_print(to, to_arg, "p(%X)", *ap); + BeamInstr* f = find_function_from_pc((BeamInstr *)*ap); + if (f+3 != (BeamInstr *) *ap) { + erts_print(to, to_arg, "p(" HEXF ")", *ap); } else { - erts_print(to, to_arg, "%T:%T/%bpu", f[0], f[1], f[2]); + erts_print(to, to_arg, "%T:%T/%bpu", (Eterm) f[0], (Eterm) f[1], (Eterm) f[2]); } ap++; } break; case 'j': /* Pointer (to label) */ - erts_print(to, to_arg, "j(%X)", *ap); + erts_print(to, to_arg, "j(" HEXF ")", *ap); ap++; break; case 'e': /* Export entry */ { Export* ex = (Export *) *ap; erts_print(to, to_arg, - "%T:%T/%bpu", ex->code[0], ex->code[1], ex->code[2]); + "%T:%T/%bpu", (Eterm) ex->code[0], (Eterm) ex->code[1], (Uint) ex->code[2]); ap++; } break; @@ -467,7 +506,7 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr) ap++; break; case 'P': /* Byte offset into tuple (see beam_load.c) */ - erts_print(to, to_arg, "%d", (*ap / sizeof(Eterm*)) - 1); + erts_print(to, to_arg, "%d", (*ap / sizeof(Eterm)) - 1); ap++; break; case 'l': /* fr(N) */ @@ -494,7 +533,7 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr) int n = ap[-1]; while (n > 0) { - erts_print(to, to_arg, "%T f(%X) ", ap[0], ap[1]); + erts_print(to, to_arg, "%T f(" HEXF ") ", (Eterm) ap[0], ap[1]); ap += 2; size += 2; n--; @@ -505,7 +544,7 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr) { int n; for (n = ap[-2]; n > 0; n--) { - erts_print(to, to_arg, "f(%X) ", ap[0]); + erts_print(to, to_arg, "f(" HEXF ") ", ap[0]); ap++; size++; } @@ -513,11 +552,12 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr) break; case op_i_select_big_sf: while (ap[0]) { - int arity = thing_arityval(ap[0]); - print_big(to, to_arg, ap); - size += arity+1; - ap += arity+1; - erts_print(to, to_arg, " f(%X) ", ap[0]); + Eterm *bigp = (Eterm *) ap; + int arity = thing_arityval(*bigp); + print_big(to, to_arg, bigp); + size += TermWords(arity+1); + ap += TermWords(arity+1); + erts_print(to, to_arg, " f(" HEXF ") ", ap[0]); ap++; size++; } @@ -541,8 +581,8 @@ print_big(int to, void *to_arg, Eterm* addr) erts_print(to, to_arg, "-#integer(%d) = {", i); else erts_print(to, to_arg, "#integer(%d) = {", i); - erts_print(to, to_arg, "%d", BIG_DIGIT(addr, 0)); + erts_print(to, to_arg, "0x%x", BIG_DIGIT(addr, 0)); for (k = 1; k < i; k++) - erts_print(to, to_arg, ",%d", BIG_DIGIT(addr, k)); + erts_print(to, to_arg, ",0x%x", BIG_DIGIT(addr, k)); erts_print(to, to_arg, "}"); } diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 2f7f48193d..8a0e12dd4f 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -117,6 +117,7 @@ do { \ #endif #define GET_BIF_ADDRESS(p) ((BifFunction) (((Export *) p)->code[4])) +#define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm)))) /* @@ -138,8 +139,8 @@ do { \ #define VALID_INSTR(IP) (0 <= (int)(IP) && ((int)(IP) < (NUMBER_OF_OPCODES*2+10))) #else #define VALID_INSTR(IP) \ - ((Sint)LabelAddr(emulator_loop) <= (Sint)(IP) && \ - (Sint)(IP) < (Sint)LabelAddr(end_emulator_loop)) + ((SWord)LabelAddr(emulator_loop) <= (SWord)(IP) && \ + (SWord)(IP) < (SWord)LabelAddr(end_emulator_loop)) #endif /* NO_JUMP_TABLE */ #define SET_CP(p, ip) \ @@ -181,11 +182,11 @@ do { \ #define StoreBifResult(Dst, Result) \ do { \ - Eterm* stb_next; \ + BeamInstr* stb_next; \ Eterm stb_reg; \ stb_reg = Arg(Dst); \ I += (Dst) + 2; \ - stb_next = (Eterm *) *I; \ + stb_next = (BeamInstr *) *I; \ CHECK_TERM(Result); \ switch (beam_reg_tag(stb_reg)) { \ case R_REG_DEF: \ @@ -205,7 +206,7 @@ do { \ c_p->cp = 0; \ } while(0) -#define RESTORE_CP(X) SET_CP(c_p, cp_val(*(X))) +#define RESTORE_CP(X) SET_CP(c_p, (BeamInstr *) cp_val(*(X))) #define ISCATCHEND(instr) ((Eterm *) *(instr) == OpCode(catch_end_y)) @@ -213,13 +214,13 @@ do { \ * Special Beam instructions. */ -Eterm beam_apply[2]; -Eterm beam_exit[1]; -Eterm beam_continue_exit[1]; +BeamInstr beam_apply[2]; +BeamInstr beam_exit[1]; +BeamInstr beam_continue_exit[1]; -Eterm* em_call_error_handler; -Eterm* em_apply_bif; -Eterm* em_call_traced_function; +BeamInstr* em_call_error_handler; +BeamInstr* em_apply_bif; +BeamInstr* em_call_traced_function; /* NOTE These should be the only variables containing trace instructions. @@ -227,9 +228,10 @@ Eterm* em_call_traced_function; ** for the refering variable (one of these), and rouge references ** will most likely cause chaos. */ -Eterm beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ -Eterm beam_return_trace[1]; /* OpCode(i_return_trace) */ -Eterm beam_exception_trace[1]; /* UGLY also OpCode(i_return_trace) */ +BeamInstr beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ +BeamInstr beam_return_trace[1]; /* OpCode(i_return_trace) */ +BeamInstr beam_exception_trace[1]; /* UGLY also OpCode(i_return_trace) */ +BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ /* * All Beam instructions in numerical order. @@ -522,8 +524,8 @@ extern int count_instructions; #define DispatchMacro() \ do { \ - Eterm* dis_next; \ - dis_next = (Eterm *) *I; \ + BeamInstr* dis_next; \ + dis_next = (BeamInstr *) *I; \ CHECK_ARGS(I); \ if (FCALLS > 0 || FCALLS > neg_o_reds) { \ FCALLS--; \ @@ -535,8 +537,8 @@ extern int count_instructions; #define DispatchMacroFun() \ do { \ - Eterm* dis_next; \ - dis_next = (Eterm *) *I; \ + BeamInstr* dis_next; \ + dis_next = (BeamInstr *) *I; \ CHECK_ARGS(I); \ if (FCALLS > 0 || FCALLS > neg_o_reds) { \ FCALLS--; \ @@ -590,7 +592,7 @@ extern int count_instructions; ASSERT(VALID_INSTR(*I)); \ Goto(*I) -#define PreFetch(N, Dst) do { Dst = (Eterm *) *(I + N + 1); } while (0) +#define PreFetch(N, Dst) do { Dst = (BeamInstr *) *(I + N + 1); } while (0) #define NextPF(N, Dst) \ I += N + 1; \ ASSERT(VALID_INSTR(Dst)); \ @@ -644,7 +646,7 @@ extern int count_instructions; #define DeallocateReturn(Deallocate) \ do { \ int words_to_pop = (Deallocate); \ - SET_I(cp_val(*E)); \ + SET_I((BeamInstr *) cp_val(*E)); \ E = ADD_BYTE_OFFSET(E, words_to_pop); \ CHECK_TERM(r(0)); \ Goto(*I); \ @@ -657,19 +659,19 @@ extern int count_instructions; #define MoveCall(Src, Dest, CallDest, Size) \ (Dest) = (Src); \ SET_CP(c_p, I+Size+1); \ - SET_I((Eterm *) CallDest); \ + SET_I((BeamInstr *) CallDest); \ Dispatch(); #define MoveCallLast(Src, Dest, CallDest, Deallocate) \ (Dest) = (Src); \ RESTORE_CP(E); \ E = ADD_BYTE_OFFSET(E, (Deallocate)); \ - SET_I((Eterm *) CallDest); \ + SET_I((BeamInstr *) CallDest); \ Dispatch(); #define MoveCallOnly(Src, Dest, CallDest) \ (Dest) = (Src); \ - SET_I((Eterm *) CallDest); \ + SET_I((BeamInstr *) CallDest); \ Dispatch(); #define GetList(Src, H, T) do { \ @@ -677,47 +679,48 @@ extern int count_instructions; H = CAR(tmp_ptr); \ T = CDR(tmp_ptr); } while (0) -#define GetTupleElement(Src, Element, Dest) \ - do { \ - tmp_arg1 = (Eterm) (((unsigned char *) tuple_val(Src)) + (Element)); \ - (Dest) = (*(Eterm *)tmp_arg1); \ +#define GetTupleElement(Src, Element, Dest) \ + do { \ + tmp_arg1 = (Eterm) COMPRESS_POINTER(((unsigned char *) tuple_val(Src)) + \ + (Element)); \ + (Dest) = (*(Eterm *) EXPAND_POINTER(tmp_arg1)); \ } while (0) -#define ExtractNextElement(Dest) \ - tmp_arg1 += sizeof(Eterm); \ - (Dest) = (* (Eterm *) (((unsigned char *) tmp_arg1))) +#define ExtractNextElement(Dest) \ + tmp_arg1 += sizeof(Eterm); \ + (Dest) = (* (Eterm *) (((unsigned char *) EXPAND_POINTER(tmp_arg1)))) -#define ExtractNextElement2(Dest) \ - do { \ - Eterm* ene_dstp = &(Dest); \ - ene_dstp[0] = ((Eterm *) tmp_arg1)[1]; \ - ene_dstp[1] = ((Eterm *) tmp_arg1)[2]; \ - tmp_arg1 += sizeof(Eterm) + sizeof(Eterm); \ +#define ExtractNextElement2(Dest) \ + do { \ + Eterm* ene_dstp = &(Dest); \ + ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1]; \ + ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2]; \ + tmp_arg1 += sizeof(Eterm) + sizeof(Eterm); \ } while (0) #define ExtractNextElement3(Dest) \ do { \ Eterm* ene_dstp = &(Dest); \ - ene_dstp[0] = ((Eterm *) tmp_arg1)[1]; \ - ene_dstp[1] = ((Eterm *) tmp_arg1)[2]; \ - ene_dstp[2] = ((Eterm *) tmp_arg1)[3]; \ + ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1]; \ + ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2]; \ + ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3]; \ tmp_arg1 += 3*sizeof(Eterm); \ } while (0) #define ExtractNextElement4(Dest) \ do { \ Eterm* ene_dstp = &(Dest); \ - ene_dstp[0] = ((Eterm *) tmp_arg1)[1]; \ - ene_dstp[1] = ((Eterm *) tmp_arg1)[2]; \ - ene_dstp[2] = ((Eterm *) tmp_arg1)[3]; \ - ene_dstp[3] = ((Eterm *) tmp_arg1)[4]; \ + ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1]; \ + ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2]; \ + ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3]; \ + ene_dstp[3] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[4]; \ tmp_arg1 += 4*sizeof(Eterm); \ } while (0) #define ExtractElement(Element, Dest) \ do { \ tmp_arg1 += (Element); \ - (Dest) = (* (Eterm *) tmp_arg1); \ + (Dest) = (* (Eterm *) EXPAND_POINTER(tmp_arg1)); \ } while (0) #define PutTuple(Arity, Src, Dest) \ @@ -759,8 +762,13 @@ extern int count_instructions; #define IsTuple(X, Action) if (is_not_tuple(X)) Action -#define IsArity(Pointer, Arity, Fail) \ - if (*(Eterm *)(tmp_arg1 = (Eterm)tuple_val(Pointer)) != (Arity)) { Fail; } +#define IsArity(Pointer, Arity, Fail) \ + if (*(Eterm *) \ + EXPAND_POINTER(tmp_arg1 = (Eterm) \ + COMPRESS_POINTER(tuple_val(Pointer))) != (Arity)) \ + { \ + Fail; \ + } #define IsFunction(X, Action) \ do { \ @@ -776,11 +784,14 @@ extern int count_instructions; } \ } while (0) -#define IsTupleOfArity(Src, Arity, Fail) \ - do { \ - if (is_not_tuple(Src) || *(Eterm *)(tmp_arg1 = (Eterm) tuple_val(Src)) != Arity) { \ - Fail; \ - } \ +#define IsTupleOfArity(Src, Arity, Fail) \ + do { \ + if (is_not_tuple(Src) || \ + *(Eterm *) \ + EXPAND_POINTER(tmp_arg1 = \ + (Eterm) COMPRESS_POINTER(tuple_val(Src))) != Arity) { \ + Fail; \ + } \ } while (0) #define IsBoolean(X, Fail) if ((X) != am_true && (X) != am_false) { Fail; } @@ -791,7 +802,7 @@ extern int count_instructions; #define IsBitstring(Src, Fail) \ if (is_not_binary(Src)) { Fail; } -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP #define BsSafeMul(A, B, Fail, Target) \ do { Uint64 _res = (A) * (B); \ if (_res / B != A) { Fail; } \ @@ -974,33 +985,33 @@ extern int count_instructions; #define IsRef(Src, Fail) if (is_not_ref(Src)) { Fail; } static BifFunction translate_gc_bif(void* gcf); -static Eterm* handle_error(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf); -static Eterm* next_catch(Process* c_p, Eterm *reg); +static BeamInstr* handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf); +static BeamInstr* next_catch(Process* c_p, Eterm *reg); static void terminate_proc(Process* c_p, Eterm Value); static Eterm add_stacktrace(Process* c_p, Eterm Value, Eterm exc); -static void save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, +static void save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, Eterm args); static struct StackTrace * get_trace_from_exc(Eterm exc); static Eterm make_arglist(Process* c_p, Eterm* reg, int a); -static Eterm call_error_handler(Process* p, Eterm* ip, Eterm* reg); -static Eterm call_breakpoint_handler(Process* p, Eterm* fi, Eterm* reg); -static Uint* fixed_apply(Process* p, Eterm* reg, Uint arity); -static Eterm* apply(Process* p, Eterm module, Eterm function, +static Eterm call_error_handler(Process* p, BeamInstr* ip, Eterm* reg); +static Eterm call_breakpoint_handler(Process* p, BeamInstr* fi, Eterm* reg); +static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity); +static BeamInstr* apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg); static int hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg); -static Eterm* call_fun(Process* p, int arity, Eterm* reg, Eterm args); -static Eterm* apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg); +static BeamInstr* call_fun(Process* p, int arity, Eterm* reg, Eterm args); +static BeamInstr* apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg); static Eterm new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free); -#if defined(_OSE_) || defined(VXWORKS) +#if defined(VXWORKS) static int init_done; #endif void init_emulator(void) { -#if defined(_OSE_) || defined(VXWORKS) +#if defined(VXWORKS) init_done = 0; #endif process_main(); @@ -1039,7 +1050,7 @@ init_emulator(void) */ void process_main(void) { -#if !defined(_OSE_) && !defined(VXWORKS) +#if !defined(VXWORKS) static int init_done = 0; #endif Process* c_p = NULL; @@ -1078,7 +1089,7 @@ void process_main(void) /* * Pointer to next threaded instruction. */ - register Eterm *I REG_I = NULL; + register BeamInstr *I REG_I = NULL; /* Number of reductions left. This function * returns to the scheduler when FCALLS reaches zero. @@ -1090,9 +1101,14 @@ void process_main(void) */ register Eterm tmp_arg1 REG_tmp_arg1 = NIL; register Eterm tmp_arg2 REG_tmp_arg2 = NIL; - Eterm tmp_big[2]; /* Temporary buffer for small bignums. */ +#if HEAP_ON_C_STACK + Eterm tmp_big[2]; /* Temporary buffer for small bignums if HEAP_ON_C_STACK. */ +#else + Eterm *tmp_big; /* Temporary buffer for small bignums if !HEAP_ON_C_STACK. */ +#endif #ifndef ERTS_SMP +#if !HALFWORD_HEAP static Eterm save_reg[ERTS_X_REGS_ALLOCATED]; /* X registers -- not used directly, but * through 'reg', because using it directly @@ -1100,7 +1116,7 @@ void process_main(void) * while using it through reg needs only * one. */ - +#endif /* * Floating point registers. */ @@ -1141,13 +1157,17 @@ void process_main(void) * Note: c_p->arity must be set to reflect the number of useful terms in * c_p->arg_reg before calling the scheduler. */ - if (!init_done) { init_done = 1; goto init_emulator; } #ifndef ERTS_SMP +#if !HALFWORD_HEAP reg = save_reg; /* XXX: probably wastes a register on x86 */ +#else + /* Registers need to be heap allocated (correct memory range) for tracing to work */ + reg = erts_alloc(ERTS_ALC_T_BEAM_REGISTER, ERTS_X_REGS_ALLOCATED * sizeof(Eterm)); +#endif #endif c_p = NULL; reds_used = 0; @@ -1168,11 +1188,14 @@ void process_main(void) reg = c_p->scheduler_data->save_reg; freg = c_p->scheduler_data->freg; #endif +#if !HEAP_ON_C_STACK + tmp_big = ERTS_PROC_GET_SCHDATA(c_p)->beam_emu_tmp_heap; +#endif ERL_BITS_RELOAD_STATEP(c_p); { int reds; Eterm* argp; - Eterm* next; + BeamInstr *next; int i; argp = c_p->arg_reg; @@ -1199,7 +1222,7 @@ void process_main(void) FCALLS = REDS_IN(c_p) = reds; } - next = (Eterm *) *I; + next = (BeamInstr *) *I; r(0) = c_p->arg_reg[0]; #ifdef HARDDEBUG if (c_p->arity > 0) { @@ -1291,7 +1314,7 @@ void process_main(void) } /* FALL THROUGH */ OpCase(i_call_only_f): { - SET_I((Eterm *) Arg(0)); + SET_I((BeamInstr *) Arg(0)); Dispatch(); } @@ -1302,7 +1325,7 @@ void process_main(void) OpCase(i_call_last_fP): { RESTORE_CP(E); E = ADD_BYTE_OFFSET(E, Arg(1)); - SET_I((Eterm *) Arg(0)); + SET_I((BeamInstr *) Arg(0)); Dispatch(); } @@ -1313,7 +1336,7 @@ void process_main(void) /* FALL THROUGH */ OpCase(i_call_f): { SET_CP(c_p, I+2); - SET_I((Eterm *) Arg(0)); + SET_I((BeamInstr *) Arg(0)); Dispatch(); } @@ -1349,7 +1372,7 @@ void process_main(void) Dispatchx(); OpCase(init_y): { - Eterm* next; + BeamInstr *next; PreFetch(1, next); make_blank(yb(Arg(0))); @@ -1357,7 +1380,7 @@ void process_main(void) } OpCase(i_trim_I): { - Eterm* next; + BeamInstr *next; Uint words; Uint cp; @@ -1383,7 +1406,7 @@ void process_main(void) } OpCase(test_heap_1_put_list_Iy): { - Eterm* next; + BeamInstr *next; PreFetch(2, next); TestHeap(Arg(0), 1); @@ -1392,20 +1415,6 @@ void process_main(void) NextPF(2, next); } - OpCase(put_string_IId): - { - unsigned char* s; - int len; - Eterm result; - - len = Arg(0); /* Length. */ - result = NIL; - for (s = (unsigned char *) Arg(1); len > 0; s--, len--) { - PutList(make_small(*s), result, result, StoreSimpleDest); - } - StoreBifResult(2, result); - } - /* * Send is almost a standard call-BIF with two arguments, except for: * 1) It cannot be traced. @@ -1414,7 +1423,7 @@ void process_main(void) */ OpCase(send): { - Eterm* next; + BeamInstr *next; Eterm result; PRE_BIF_SWAPOUT(c_p); @@ -1429,7 +1438,7 @@ void process_main(void) NextPF(0, next); } else if (c_p->freason == TRAP) { SET_CP(c_p, I+1); - SET_I((Eterm *) c_p->def_arg_reg[3]); + SET_I(*((BeamInstr **) (BeamInstr) ((c_p)->def_arg_reg + 3))); SWAPIN; r(0) = c_p->def_arg_reg[0]; x(1) = c_p->def_arg_reg[1]; @@ -1531,6 +1540,10 @@ void process_main(void) /* * Skeleton for receive statement: * + * recv_mark L1 Optional + * call make_ref/monitor Optional + * ... + * recv_set L1 Optional * L1: <-------------------+ * <-----------+ | * | | @@ -1549,13 +1562,41 @@ void process_main(void) * */ + OpCase(recv_mark_f): { + /* + * Save the current position in message buffer and the + * the label for the loop_rec/2 instruction for the + * the receive statement. + */ + c_p->msg.mark = (BeamInstr *) Arg(0); + c_p->msg.saved_last = c_p->msg.last; + Next(1); + } + + OpCase(i_recv_set): { + /* + * If the mark is valid (points to the loop_rec/2 + * instruction that follows), we know that the saved + * position points to the first message that could + * possibly be matched out. + * + * If the mark is invalid, we do nothing, meaning that + * we will look through all messages in the message queue. + */ + if (c_p->msg.mark == (BeamInstr *) (I+1)) { + c_p->msg.save = c_p->msg.saved_last; + } + I++; + /* Fall through to the loop_rec/2 instruction */ + } + /* * Pick up the next message and place it in x(0). * If no message, jump to a wait or wait_timeout instruction. */ OpCase(i_loop_rec_fr): { - Eterm* next; + BeamInstr *next; ErlMessage* msgp; loop_rec__: @@ -1579,7 +1620,7 @@ void process_main(void) erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); else { #endif - SET_I((Eterm *) Arg(0)); + SET_I((BeamInstr *) Arg(0)); Goto(*I); /* Jump to a wait or wait_timeout instruction */ #ifdef ERTS_SMP } @@ -1615,7 +1656,7 @@ void process_main(void) * Remove a (matched) message from the message queue. */ OpCase(remove_message): { - Eterm* next; + BeamInstr *next; ErlMessage* msgp; PROCESS_MAIN_CHK_LOCKS(c_p); @@ -1660,7 +1701,7 @@ void process_main(void) * message didn't match), then jump to the loop_rec instruction. */ OpCase(loop_rec_end_f): { - SET_I((Eterm *) Arg(0)); + SET_I((BeamInstr *) Arg(0)); SAVE_MESSAGE(c_p); goto loop_rec__; } @@ -1690,12 +1731,12 @@ void process_main(void) } GetArg1(1, timeout_value); if (timeout_value != make_small(0)) { -#if !defined(ARCH_64) +#if !defined(ARCH_64) || HALFWORD_HEAP Uint time_val; #endif if (is_small(timeout_value) && signed_val(timeout_value) > 0 && -#if defined(ARCH_64) +#if defined(ARCH_64) && !HALFWORD_HEAP ((unsigned_val(timeout_value) >> 32) == 0) #else 1 @@ -1706,14 +1747,16 @@ void process_main(void) * c_p->def_arg_reg[0]. Note that it is safe to use this * location because there are no living x registers in * a receive statement. + * Note that for the halfword emulator, the two first elements + * of the array are used. */ - c_p->def_arg_reg[0] = (Eterm) (I+3); + *((BeamInstr **) (UWord) c_p->def_arg_reg) = I+3; set_timer(c_p, unsigned_val(timeout_value)); } else if (timeout_value == am_infinity) { c_p->flags |= F_TIMO; -#if !defined(ARCH_64) +#if !defined(ARCH_64) || HALFWORD_HEAP } else if (term_to_Uint(timeout_value, &time_val)) { - c_p->def_arg_reg[0] = (Eterm) (I+3); + *((BeamInstr **) (UWord) c_p->def_arg_reg) = I+3; set_timer(c_p, time_val); #endif } else { /* Wrong time */ @@ -1742,7 +1785,7 @@ void process_main(void) wait2: { ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - c_p->i = (Eterm *) Arg(0); /* L1 */ + c_p->i = (BeamInstr *) Arg(0); /* L1 */ SWAPOUT; c_p->arity = 0; c_p->status = P_WAITING; @@ -1770,7 +1813,7 @@ void process_main(void) * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag. */ if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) { - c_p->def_arg_reg[0] = (Eterm) (I+3); + *((BeamInstr **) (UWord) c_p->def_arg_reg) = I+3; set_timer(c_p, Arg(1)); } goto wait2; @@ -1785,7 +1828,7 @@ void process_main(void) } OpCase(timeout): { - Eterm* next; + BeamInstr *next; PreFetch(0, next); if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) { @@ -1805,8 +1848,8 @@ void process_main(void) do_binary_search: { struct Pairs { - Eterm val; - Eterm* addr; + BeamInstr val; + BeamInstr* addr; }; struct Pairs* low; struct Pairs* high; @@ -1846,7 +1889,7 @@ void process_main(void) Goto(*I); } } - SET_I((Eterm *) Arg(1)); + SET_I((BeamInstr *) Arg(1)); Goto(*I); } @@ -1858,11 +1901,11 @@ void process_main(void) if (is_small(index)) { index = signed_val(index); if (index < Arg(2)) { - SET_I((Eterm *) (&Arg(3))[index]); + SET_I((BeamInstr *) (&Arg(3))[index]); Goto(*I); } } - SET_I((Eterm *) Arg(1)); + SET_I((BeamInstr *) Arg(1)); Goto(*I); } @@ -1874,11 +1917,11 @@ void process_main(void) if (is_small(index)) { index = (Uint) (signed_val(index) - Arg(3)); if (index < Arg(2)) { - SET_I((Eterm *) (&Arg(4))[index]); + SET_I((BeamInstr *) (&Arg(4))[index]); Goto(*I); } } - SET_I((Eterm *) Arg(1)); + SET_I((BeamInstr *) Arg(1)); Goto(*I); } @@ -1915,7 +1958,7 @@ void process_main(void) if (is_value(result)) { StoreBifResult(3, result); } - SET_I((Eterm *) Arg(0)); + SET_I((BeamInstr *) Arg(0)); Goto(*I); } @@ -1955,7 +1998,7 @@ void process_main(void) GcBifFunction bf; Eterm arg; Eterm result; - Uint live = Arg(3); + Uint live = (Uint) Arg(3); GetArg1(2, arg); reg[0] = r(0); @@ -1976,7 +2019,7 @@ void process_main(void) StoreBifResult(4, result); } if (Arg(0) != 0) { - SET_I((Eterm *) Arg(0)); + SET_I((BeamInstr *) Arg(0)); Goto(*I); } reg[0] = arg; @@ -1984,6 +2027,81 @@ void process_main(void) goto post_error_handling; } + OpCase(i_gc_bif2_jIId): /* Note, one less parameter than the i_gc_bif1 + and i_gc_bif3 */ + { + typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); + GcBifFunction bf; + Eterm result; + Uint live = (Uint) Arg(2); + + reg[0] = r(0); + reg[live++] = tmp_arg1; + reg[live] = tmp_arg2; + bf = (GcBifFunction) Arg(1); + c_p->fcalls = FCALLS; + SWAPOUT; + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + result = (*bf)(c_p, reg, live); + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + SWAPIN; + r(0) = reg[0]; + ERTS_HOLE_CHECK(c_p); + FCALLS = c_p->fcalls; + if (is_value(result)) { + StoreBifResult(3, result); + } + if (Arg(0) != 0) { + SET_I((BeamInstr *) Arg(0)); + Goto(*I); + } + reg[0] = tmp_arg1; + reg[1] = tmp_arg2; + I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); + goto post_error_handling; + } + + OpCase(i_gc_bif3_jIsId): + { + typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); + GcBifFunction bf; + Eterm arg; + Eterm result; + Uint live = (Uint) Arg(3); + + GetArg1(2, arg); + reg[0] = r(0); + reg[live++] = arg; + reg[live++] = tmp_arg1; + reg[live] = tmp_arg2; + bf = (GcBifFunction) Arg(1); + c_p->fcalls = FCALLS; + SWAPOUT; + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + result = (*bf)(c_p, reg, live); + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + SWAPIN; + r(0) = reg[0]; + ERTS_HOLE_CHECK(c_p); + FCALLS = c_p->fcalls; + if (is_value(result)) { + StoreBifResult(4, result); + } + if (Arg(0) != 0) { + SET_I((BeamInstr *) Arg(0)); + Goto(*I); + } + reg[0] = arg; + reg[1] = tmp_arg1; + reg[2] = tmp_arg2; + I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); + goto post_error_handling; + } + /* * Guards bifs and, or, xor in guards. */ @@ -2004,7 +2122,7 @@ void process_main(void) if (is_value(result)) { StoreBifResult(2, result); } - SET_I((Eterm *) Arg(0)); + SET_I((BeamInstr *) Arg(0)); Goto(*I); } @@ -2040,7 +2158,7 @@ void process_main(void) */ OpCase(call_bif0_e): { - Eterm (*bf)(Process*, Uint*) = GET_BIF_ADDRESS(Arg(0)); + Eterm (*bf)(Process*, BeamInstr*) = GET_BIF_ADDRESS(Arg(0)); PRE_BIF_SWAPOUT(c_p); c_p->fcalls = FCALLS - 1; @@ -2073,9 +2191,9 @@ void process_main(void) OpCase(call_bif1_e): { - Eterm (*bf)(Process*, Eterm, Uint*) = GET_BIF_ADDRESS(Arg(0)); + Eterm (*bf)(Process*, Eterm, BeamInstr*) = GET_BIF_ADDRESS(Arg(0)); Eterm result; - Eterm* next; + BeamInstr *next; c_p->fcalls = FCALLS - 1; if (FCALLS <= 0) { @@ -2108,9 +2226,9 @@ void process_main(void) OpCase(call_bif2_e): { - Eterm (*bf)(Process*, Eterm, Eterm, Uint*) = GET_BIF_ADDRESS(Arg(0)); + Eterm (*bf)(Process*, Eterm, Eterm, BeamInstr*) = GET_BIF_ADDRESS(Arg(0)); Eterm result; - Eterm* next; + BeamInstr *next; PRE_BIF_SWAPOUT(c_p); c_p->fcalls = FCALLS - 1; @@ -2145,9 +2263,9 @@ void process_main(void) OpCase(call_bif3_e): { - Eterm (*bf)(Process*, Eterm, Eterm, Eterm, Uint*) = GET_BIF_ADDRESS(Arg(0)); + Eterm (*bf)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = GET_BIF_ADDRESS(Arg(0)); Eterm result; - Eterm* next; + BeamInstr *next; PRE_BIF_SWAPOUT(c_p); c_p->fcalls = FCALLS - 1; @@ -2168,7 +2286,7 @@ void process_main(void) } else if (c_p->freason == TRAP) { call_bif_trap3: SET_CP(c_p, I+2); - SET_I((Eterm *)c_p->def_arg_reg[3]); + SET_I(*((BeamInstr **) (UWord) ((c_p)->def_arg_reg + 3))); SWAPIN; r(0) = c_p->def_arg_reg[0]; x(1) = c_p->def_arg_reg[1]; @@ -2276,7 +2394,7 @@ void process_main(void) lb_Cl_error: { if (Arg(0) != 0) { OpCase(jump_f): { - SET_I((Eterm *) Arg(0)); + SET_I((BeamInstr *) Arg(0)); Goto(*I); } } @@ -2468,7 +2586,7 @@ void process_main(void) goto lb_Cl_error; OpCase(i_apply): { - Eterm* next; + BeamInstr *next; SWAPOUT; next = apply(c_p, r(0), x(1), x(2), reg); SWAPIN; @@ -2483,13 +2601,13 @@ void process_main(void) } OpCase(i_apply_last_P): { - Eterm* next; + BeamInstr *next; SWAPOUT; next = apply(c_p, r(0), x(1), x(2), reg); SWAPIN; if (next != NULL) { r(0) = reg[0]; - SET_CP(c_p, (Eterm *) E[0]); + SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); E = ADD_BYTE_OFFSET(E, Arg(0)); SET_I(next); Dispatch(); @@ -2499,7 +2617,7 @@ void process_main(void) } OpCase(i_apply_only): { - Eterm* next; + BeamInstr *next; SWAPOUT; next = apply(c_p, r(0), x(1), x(2), reg); SWAPIN; @@ -2513,7 +2631,7 @@ void process_main(void) } OpCase(apply_I): { - Eterm* next; + BeamInstr *next; reg[0] = r(0); SWAPOUT; @@ -2530,7 +2648,7 @@ void process_main(void) } OpCase(apply_last_IP): { - Eterm* next; + BeamInstr *next; reg[0] = r(0); SWAPOUT; @@ -2538,7 +2656,7 @@ void process_main(void) SWAPIN; if (next != NULL) { r(0) = reg[0]; - SET_CP(c_p, (Eterm *) E[0]); + SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); E = ADD_BYTE_OFFSET(E, Arg(1)); SET_I(next); Dispatch(); @@ -2548,7 +2666,7 @@ void process_main(void) } OpCase(i_apply_fun): { - Eterm* next; + BeamInstr *next; SWAPOUT; next = apply_fun(c_p, r(0), x(1), reg); @@ -2563,14 +2681,14 @@ void process_main(void) } OpCase(i_apply_fun_last_P): { - Eterm* next; + BeamInstr *next; SWAPOUT; next = apply_fun(c_p, r(0), x(1), reg); SWAPIN; if (next != NULL) { r(0) = reg[0]; - SET_CP(c_p, (Eterm *) E[0]); + SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); E = ADD_BYTE_OFFSET(E, Arg(0)); SET_I(next); Dispatchfun(); @@ -2579,7 +2697,7 @@ void process_main(void) } OpCase(i_apply_fun_only): { - Eterm* next; + BeamInstr *next; SWAPOUT; next = apply_fun(c_p, r(0), x(1), reg); @@ -2593,10 +2711,11 @@ void process_main(void) } OpCase(i_call_fun_I): { - Eterm* next; + BeamInstr *next; SWAPOUT; reg[0] = r(0); + next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE); SWAPIN; if (next != NULL) { @@ -2609,7 +2728,7 @@ void process_main(void) } OpCase(i_call_fun_last_IP): { - Eterm* next; + BeamInstr *next; SWAPOUT; reg[0] = r(0); @@ -2617,7 +2736,7 @@ void process_main(void) SWAPIN; if (next != NULL) { r(0) = reg[0]; - SET_CP(c_p, (Eterm *) E[0]); + SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); E = ADD_BYTE_OFFSET(E, Arg(1)); SET_I(next); Dispatchfun(); @@ -2722,7 +2841,7 @@ void process_main(void) tmp_arg1 = *tuple_val(tmp_arg1); goto do_binary_search; } - SET_I((Eterm *) Arg(1)); + SET_I((BeamInstr *) Arg(1)); Goto(*I); } @@ -2748,16 +2867,18 @@ void process_main(void) given = big_val(tmp_arg1); given_arity = given[0]; given_size = thing_arityval(given_arity); - bigp = &Arg(2); + bigp = (Eterm *) &Arg(2); while ((arity = bigp[0]) > given_arity) { - bigp += thing_arityval(arity) + 2; + bigp += (TermWords(thing_arityval(arity) + 1) + 1) * (sizeof(BeamInstr)/sizeof(Eterm)); } while (bigp[0] == given_arity) { if (memcmp(bigp+1, given+1, sizeof(Eterm)*given_size) == 0) { - SET_I((Eterm *) bigp[given_size+1]); + BeamInstr *tmp = + ((BeamInstr *) (UWord) bigp) + TermWords(given_size + 1); + SET_I((BeamInstr *) *tmp); Goto(*I); } - bigp += thing_arityval(arity) + 2; + bigp += (TermWords(thing_arityval(arity) + 1) + 1) * (sizeof(BeamInstr)/sizeof(Eterm)); } } @@ -2765,18 +2886,18 @@ void process_main(void) * Failed. */ - SET_I((Eterm *) Arg(1)); + SET_I((BeamInstr *) Arg(1)); Goto(*I); } -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP OpCase(i_select_float_sfI): { Uint f; int n; struct ValLabel { Uint f; - Eterm* addr; + BeamInstr* addr; }; struct ValLabel* ptr; @@ -2804,7 +2925,7 @@ void process_main(void) struct ValLabel { Uint fpart1; Uint fpart2; - Eterm* addr; + BeamInstr* addr; }; struct ValLabel* ptr; @@ -2822,7 +2943,7 @@ void process_main(void) } ptr++; } - SET_I((Eterm *) Arg(1)); + SET_I((BeamInstr *) Arg(1)); Goto(*I); } #endif @@ -2830,7 +2951,7 @@ void process_main(void) OpCase(set_tuple_element_sdP): { Eterm element; Eterm tuple; - Eterm* next; + BeamInstr *next; Eterm* p; PreFetch(3, next); @@ -3025,7 +3146,7 @@ void process_main(void) switch (tmp_arg2) { case 3: { - Eterm (*bf)(Process*, Eterm, Eterm, Eterm, Uint*) = vbf; + Eterm (*bf)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = vbf; ASSERT(!ERTS_PROC_IS_EXITING(c_p)); tmp_arg1 = (*bf)(c_p, r(0), x(1), x(2), I); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1)); @@ -3034,7 +3155,7 @@ void process_main(void) break; case 2: { - Eterm (*bf)(Process*, Eterm, Eterm, Uint*) = vbf; + Eterm (*bf)(Process*, Eterm, Eterm, BeamInstr*) = vbf; ASSERT(!ERTS_PROC_IS_EXITING(c_p)); tmp_arg1 = (*bf)(c_p, r(0), x(1), I); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1)); @@ -3043,7 +3164,7 @@ void process_main(void) break; case 1: { - Eterm (*bf)(Process*, Eterm, Uint*) = vbf; + Eterm (*bf)(Process*, Eterm, BeamInstr*) = vbf; ASSERT(!ERTS_PROC_IS_EXITING(c_p)); tmp_arg1 = (*bf)(c_p, r(0), I); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1)); @@ -3052,7 +3173,7 @@ void process_main(void) break; case 0: { - Eterm (*bf)(Process*, Uint*) = vbf; + Eterm (*bf)(Process*, BeamInstr*) = vbf; ASSERT(!ERTS_PROC_IS_EXITING(c_p)); tmp_arg1 = (*bf)(c_p, I); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1)); @@ -3076,7 +3197,7 @@ apply_bif_or_nif_epilogue: SET_I(c_p->cp); Goto(*I); } else if (c_p->freason == TRAP) { - SET_I((Eterm *)c_p->def_arg_reg[3]); + SET_I(*((BeamInstr **) (UWord) ((c_p)->def_arg_reg + 3))); r(0) = c_p->def_arg_reg[0]; x(1) = c_p->def_arg_reg[1]; x(2) = c_p->def_arg_reg[2]; @@ -3097,12 +3218,6 @@ apply_bif_or_nif_epilogue: StoreBifResult(1, result); } - OpCase(i_put_tuple_only_Ad): { - tmp_arg1 = make_tuple(HTOP); - *HTOP++ = Arg(0); - StoreBifResult(1, tmp_arg1); - } - OpCase(case_end_s): GetArg1(0, tmp_arg1); c_p->fvalue = tmp_arg1; @@ -3270,12 +3385,12 @@ apply_bif_or_nif_epilogue: HTOP += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = num_bytes; - pb->next = MSO(c_p).mso; - MSO(c_p).mso = pb; + pb->next = MSO(c_p).first; + MSO(c_p).first = (struct erl_off_heap_header*) pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; - MSO(c_p).overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm)); new_binary = make_binary(pb); goto do_bits_sub_bin; } @@ -3371,13 +3486,13 @@ apply_bif_or_nif_epilogue: HTOP += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = tmp_arg1; - pb->next = MSO(c_p).mso; - MSO(c_p).mso = pb; + pb->next = MSO(c_p).first; + MSO(c_p).first = (struct erl_off_heap_header*) pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; - MSO(c_p).overhead += tmp_arg1 / sizeof(Eterm); + OH_OVERHEAD(&(MSO(c_p)), tmp_arg1 / sizeof(Eterm)); StoreBifResult(2, make_binary(pb)); } @@ -3413,42 +3528,6 @@ apply_bif_or_nif_epilogue: } } - OpCase(i_bs_bits_to_bytes_rjd): { - tmp_arg1 = r(0); - goto do_bits_to_bytes; - } - - OpCase(i_bs_bits_to_bytes_yjd): { - tmp_arg1 = yb(Arg(0)); - I++; - goto do_bits_to_bytes; - - OpCase(i_bs_bits_to_bytes_xjd): { - tmp_arg1 = xb(Arg(0)); - I++; - } - - do_bits_to_bytes: - { - if (is_valid_bit_size(tmp_arg1)) { - tmp_arg1 = make_small(unsigned_val(tmp_arg1) >> 3); - } else { - Uint bytes; - if (!term_to_Uint(tmp_arg1, &bytes)) { - goto badarg; - } - tmp_arg1 = bytes; - if ((tmp_arg1 & 0x07) != 0) { - goto badarg; - } - SWAPOUT; - tmp_arg1 = erts_make_integer(tmp_arg1 >> 3, c_p); - HTOP = HEAP_TOP(c_p); - } - StoreBifResult(1, tmp_arg1); - } - } - OpCase(i_bs_add_jId): { Uint Unit = Arg(1); if (is_both_small(tmp_arg1, tmp_arg2)) { @@ -3486,7 +3565,7 @@ apply_bif_or_nif_epilogue: /* * Now we know that one of the arguments is - * not at small. We must convert both arguments + * not a small. We must convert both arguments * to Uints and check for errors at the same time. * * Error checking is tricky. @@ -3534,7 +3613,7 @@ apply_bif_or_nif_epilogue: OpCase(bs_put_string_II): { - Eterm* next; + BeamInstr *next; PreFetch(2, next); erts_new_bs_put_string(ERL_BITS_ARGS_2((byte *) Arg(1), Arg(0))); NextPF(2, next); @@ -3705,7 +3784,7 @@ apply_bif_or_nif_epilogue: { Eterm header; - Eterm* next; + BeamInstr *next; Uint slots; OpCase(i_bs_start_match2_rfIId): { @@ -3771,7 +3850,7 @@ apply_bif_or_nif_epilogue: } OpCase(bs_test_zero_tail2_fr): { - Eterm* next; + BeamInstr *next; ErlBinMatchBuffer *_mb; PreFetch(1, next); @@ -3783,7 +3862,7 @@ apply_bif_or_nif_epilogue: } OpCase(bs_test_zero_tail2_fx): { - Eterm* next; + BeamInstr *next; ErlBinMatchBuffer *_mb; PreFetch(2, next); @@ -3795,7 +3874,7 @@ apply_bif_or_nif_epilogue: } OpCase(bs_test_tail_imm2_frI): { - Eterm* next; + BeamInstr *next; ErlBinMatchBuffer *_mb; PreFetch(2, next); _mb = ms_matchbuffer(r(0)); @@ -3805,7 +3884,7 @@ apply_bif_or_nif_epilogue: NextPF(2, next); } OpCase(bs_test_tail_imm2_fxI): { - Eterm* next; + BeamInstr *next; ErlBinMatchBuffer *_mb; PreFetch(3, next); _mb = ms_matchbuffer(xb(Arg(1))); @@ -3816,7 +3895,7 @@ apply_bif_or_nif_epilogue: } OpCase(bs_test_unit_frI): { - Eterm* next; + BeamInstr *next; ErlBinMatchBuffer *_mb; PreFetch(2, next); _mb = ms_matchbuffer(r(0)); @@ -3826,7 +3905,7 @@ apply_bif_or_nif_epilogue: NextPF(2, next); } OpCase(bs_test_unit_fxI): { - Eterm* next; + BeamInstr *next; ErlBinMatchBuffer *_mb; PreFetch(3, next); _mb = ms_matchbuffer(xb(Arg(1))); @@ -3837,7 +3916,7 @@ apply_bif_or_nif_epilogue: } OpCase(bs_test_unit8_fr): { - Eterm* next; + BeamInstr *next; ErlBinMatchBuffer *_mb; PreFetch(1, next); _mb = ms_matchbuffer(r(0)); @@ -3847,7 +3926,7 @@ apply_bif_or_nif_epilogue: NextPF(1, next); } OpCase(bs_test_unit8_fx): { - Eterm* next; + BeamInstr *next; ErlBinMatchBuffer *_mb; PreFetch(2, next); _mb = ms_matchbuffer(xb(Arg(1))); @@ -3931,11 +4010,11 @@ apply_bif_or_nif_epilogue: _integer = get_int32(_mb->base + _mb->offset/8); } _mb->offset += 32; -#ifndef ARCH_64 +#if !defined(ARCH_64) || HALFWORD_HEAP if (IS_USMALL(0, _integer)) { #endif _result = make_small(_integer); -#ifndef ARCH_64 +#if !defined(ARCH_64) || HALFWORD_HEAP } else { TestHeap(BIG_UINT_HEAP_SIZE, Arg(1)); _result = uint_to_big((Uint) _integer, HTOP); @@ -4172,7 +4251,7 @@ apply_bif_or_nif_epilogue: do_bs_match_string: { - Eterm* next; + BeamInstr *next; byte* bytes; Uint bits; ErlBinMatchBuffer* mb; @@ -4199,7 +4278,7 @@ apply_bif_or_nif_epilogue: } OpCase(i_bs_save2_rI): { - Eterm* next; + BeamInstr *next; ErlBinMatchState *_ms; PreFetch(1, next); _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0)); @@ -4207,7 +4286,7 @@ apply_bif_or_nif_epilogue: NextPF(1, next); } OpCase(i_bs_save2_xI): { - Eterm* next; + BeamInstr *next; ErlBinMatchState *_ms; PreFetch(2, next); _ms = (ErlBinMatchState*) boxed_val((Eterm) xb(Arg(0))); @@ -4216,7 +4295,7 @@ apply_bif_or_nif_epilogue: } OpCase(i_bs_restore2_rI): { - Eterm* next; + BeamInstr *next; ErlBinMatchState *_ms; PreFetch(1, next); _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0)); @@ -4224,7 +4303,7 @@ apply_bif_or_nif_epilogue: NextPF(1, next); } OpCase(i_bs_restore2_xI): { - Eterm* next; + BeamInstr *next; ErlBinMatchState *_ms; PreFetch(2, next); _ms = (ErlBinMatchState*) boxed_val((Eterm) xb(Arg(0))); @@ -4241,7 +4320,7 @@ apply_bif_or_nif_epilogue: * deallocate not followed by a return, and that should work. */ OpCase(deallocate_I): { - Eterm* next; + BeamInstr *next; PreFetch(1, next); D(Arg(0)); @@ -4264,7 +4343,7 @@ apply_bif_or_nif_epilogue: */ OpCase(call_traced_function): { if (IS_TRACED_FL(c_p, F_TRACE_CALLS)) { - unsigned offset = offsetof(Export, code) + 3*sizeof(Eterm); + unsigned offset = offsetof(Export, code) + 3*sizeof(BeamInstr); Export* ep = (Export *) (((char *)I)-offset); Uint32 flags; @@ -4291,26 +4370,25 @@ apply_bif_or_nif_epilogue: } E -= 3; ASSERT(c_p->htop <= E && E <= c_p->hend); - ASSERT(is_CP((Eterm)(ep->code))); + ASSERT(is_CP((BeamInstr)(ep->code))); ASSERT(is_internal_pid(c_p->tracer_proc) || is_internal_port(c_p->tracer_proc)); - E[2] = make_cp(c_p->cp); + E[2] = make_cp(c_p->cp); /* Code in lower range on halfword */ E[1] = am_true; /* Process tracer */ E[0] = make_cp(ep->code); - c_p->cp = (Eterm*) - make_cp(flags & MATCH_SET_EXCEPTION_TRACE - ? beam_exception_trace : beam_return_trace); + c_p->cp = (flags & MATCH_SET_EXCEPTION_TRACE) + ? beam_exception_trace : beam_return_trace; erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); c_p->trace_flags |= F_EXCEPTION_TRACE; erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); } } - SET_I((Uint *) Arg(0)); + SET_I((BeamInstr *)Arg(0)); Dispatch(); } OpCase(return_trace): { - Uint* code = (Uint *) E[0]; + BeamInstr* code = (BeamInstr *) (UWord) E[0]; SWAPOUT; /* Needed for shared heap */ ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); @@ -4318,44 +4396,117 @@ apply_bif_or_nif_epilogue: ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); SWAPIN; c_p->cp = NULL; - SET_I((Eterm *) E[2]); + SET_I((BeamInstr *) cp_val(E[2])); E += 3; Goto(*I); } OpCase(i_count_breakpoint): { - Uint real_I; + BeamInstr real_I; - ErtsCountBreak((Uint *) I, &real_I); + ErtsCountBreak(c_p, (BeamInstr *) I, &real_I); + ASSERT(VALID_INSTR(real_I)); + Goto(real_I); + } + + /* need to send mfa instead of bdt pointer + * the pointer might be deallocated. + */ + + OpCase(i_time_breakpoint): { + BeamInstr real_I; + BpData **bds = (BpData **) (I)[-4]; + BpDataTime *bdt = NULL; + Uint ix = 0; +#ifdef ERTS_SMP + ix = c_p->scheduler_data->no - 1; +#else + ix = 0; +#endif + bdt = (BpDataTime *)bds[ix]; + + ASSERT((I)[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + ASSERT(bdt); + bdt = (BpDataTime *) bdt->next; + ASSERT(bdt); + bds[ix] = (BpData *) bdt; + real_I = bdt->orig_instr; ASSERT(VALID_INSTR(real_I)); + + if (IS_TRACED_FL(c_p, F_TRACE_CALLS) && !(bdt->pause)) { + if ( (*(c_p->cp) == (BeamInstr) OpCode(i_return_time_trace)) || + (*(c_p->cp) == (BeamInstr) OpCode(return_trace)) || + (*(c_p->cp) == (BeamInstr) OpCode(i_return_to_trace))) { + /* This _IS_ a tail recursive call */ + SWAPOUT; + erts_trace_time_break(c_p, I, bdt, ERTS_BP_CALL_TIME_TAIL_CALL); + SWAPIN; + } else { + SWAPOUT; + erts_trace_time_break(c_p, I, bdt, ERTS_BP_CALL_TIME_CALL); + + /* r register needs to be copied to the array + * for the garbage collector + */ + ASSERT(c_p->htop <= E && E <= c_p->hend); + if (E - 2 < HTOP) { + reg[0] = r(0); + PROCESS_MAIN_CHK_LOCKS(c_p); + FCALLS -= erts_garbage_collect(c_p, 2, reg, I[-1]); + PROCESS_MAIN_CHK_LOCKS(c_p); + r(0) = reg[0]; + } + SWAPIN; + + ASSERT(c_p->htop <= E && E <= c_p->hend); + + E -= 2; + E[0] = make_cp(I); + E[1] = make_cp(c_p->cp); /* original return address */ + c_p->cp = beam_return_time_trace; + } + } + Goto(real_I); } + OpCase(i_return_time_trace): { + BeamInstr *pc = (BeamInstr *) (UWord) E[0]; + SWAPOUT; + erts_trace_time_break(c_p, pc, NULL, ERTS_BP_CALL_TIME_RETURN); + SWAPIN; + c_p->cp = NULL; + SET_I((BeamInstr *) cp_val(E[1])); + E += 2; + Goto(*I); + } + OpCase(i_trace_breakpoint): if (! IS_TRACED_FL(c_p, F_TRACE_CALLS)) { - Uint real_I; + BeamInstr real_I; - ErtsBreakSkip((Uint *) I, &real_I); + ErtsBreakSkip(c_p, (BeamInstr *) I, &real_I); Goto(real_I); } /* Fall through to next case */ OpCase(i_mtrace_breakpoint): { - Uint real_I; + BeamInstr real_I; Uint32 flags; Eterm tracer_pid; - Uint *cpp; + Uint* cpp; int return_to_trace = 0, need = 0; flags = 0; SWAPOUT; reg[0] = r(0); - if (*cp_val((Eterm)c_p->cp) - == (Uint) OpCode(return_trace)) { - cpp = (Uint*)&E[2]; - } else if (*cp_val((Eterm)c_p->cp) - == (Uint) OpCode(i_return_to_trace)) { + if (*(c_p->cp) == (BeamInstr) OpCode(return_trace)) { + cpp = &E[2]; + } else if (*(c_p->cp) == (BeamInstr) OpCode(i_return_to_trace)) { return_to_trace = !0; - cpp = (Uint*)&E[0]; + cpp = &E[0]; + } else if (*(c_p->cp) == (BeamInstr) OpCode(i_return_time_trace)) { + return_to_trace = !0; + cpp = &E[0]; } else { cpp = NULL; } @@ -4364,19 +4515,21 @@ apply_bif_or_nif_epilogue: * return_trace and/or i_return_to_trace stackframes * on the stack, they are not intermixed with y registers */ - Eterm *cp_save = c_p->cp; + BeamInstr *cp_save = c_p->cp; for (;;) { ASSERT(is_CP(*cpp)); - if (*cp_val(*cpp) == (Uint) OpCode(return_trace)) { + if (*cp_val(*cpp) == (BeamInstr) OpCode(return_trace)) { cpp += 3; - } else if (*cp_val(*cpp) == (Uint) OpCode(i_return_to_trace)) { + } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_to_trace)) { return_to_trace = !0; cpp += 1; + } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_time_trace)) { + cpp += 2; } else break; } - c_p->cp = (Eterm *) *cpp; - ASSERT(is_CP((Eterm)c_p->cp)); + c_p->cp = (BeamInstr *) cp_val(*cpp); + ASSERT(is_CP(*cpp)); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); real_I = erts_trace_break(c_p, I, reg, &flags, &tracer_pid); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); @@ -4412,12 +4565,12 @@ apply_bif_or_nif_epilogue: E -= 1; ASSERT(c_p->htop <= E && E <= c_p->hend); E[0] = make_cp(c_p->cp); - c_p->cp = (Eterm *) make_cp(beam_return_to_trace); + c_p->cp = (BeamInstr *) beam_return_to_trace; } if (flags & MATCH_SET_RX_TRACE) { E -= 3; ASSERT(c_p->htop <= E && E <= c_p->hend); - ASSERT(is_CP((Eterm) (I - 3))); + ASSERT(is_CP((Eterm) (UWord) (I - 3))); ASSERT(am_true == tracer_pid || is_internal_pid(tracer_pid) || is_internal_port(tracer_pid)); E[2] = make_cp(c_p->cp); @@ -4425,9 +4578,9 @@ apply_bif_or_nif_epilogue: E[0] = make_cp(I - 3); /* We ARE at the beginning of an instruction, the funcinfo is above i. */ - c_p->cp = (Eterm*) - make_cp(flags & MATCH_SET_EXCEPTION_TRACE - ? beam_exception_trace : beam_return_trace); + c_p->cp = + (flags & MATCH_SET_EXCEPTION_TRACE) + ? beam_exception_trace : beam_return_trace; erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); c_p->trace_flags |= F_EXCEPTION_TRACE; erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); @@ -4440,10 +4593,10 @@ apply_bif_or_nif_epilogue: Uint *cpp = (Uint*) E; for(;;) { ASSERT(is_CP(*cpp)); - if (*cp_val(*cpp) == (Uint) OpCode(return_trace)) { + if (*cp_val(*cpp) == (BeamInstr) OpCode(return_trace)) { do ++cpp; while(is_not_CP(*cpp)); cpp += 2; - } else if (*cp_val(*cpp) == (Uint) OpCode(i_return_to_trace)) { + } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_to_trace)) { do ++cpp; while(is_not_CP(*cpp)); } else break; } @@ -4454,7 +4607,7 @@ apply_bif_or_nif_epilogue: SWAPIN; } c_p->cp = NULL; - SET_I((Eterm *) E[0]); + SET_I((BeamInstr *) cp_val(E[0])); E += 1; Goto(*I); } @@ -4465,7 +4618,7 @@ apply_bif_or_nif_epilogue: OpCase(i_global_cons): { - Eterm *next; + BeamInstr *next; #ifdef HYBRID Eterm *hp; @@ -4487,7 +4640,7 @@ apply_bif_or_nif_epilogue: OpCase(i_global_tuple): { - Eterm *next; + BeamInstr *next; int len; #ifdef HYBRID Eterm list; @@ -4522,7 +4675,7 @@ apply_bif_or_nif_epilogue: OpCase(i_global_copy): { - Eterm *next; + BeamInstr *next; PreFetch(0,next); #ifdef HYBRID if (!IS_CONST(r(0))) @@ -4551,7 +4704,7 @@ apply_bif_or_nif_epilogue: OpCase(fmove_ql): { Eterm fr = Arg(1); - Eterm* next; + BeamInstr *next; PreFetch(2, next); GET_DOUBLE(Arg(0), *(FloatDef*)ADD_BYTE_OFFSET(freg, fr)); @@ -4561,7 +4714,7 @@ apply_bif_or_nif_epilogue: OpCase(fmove_dl): { Eterm targ1; Eterm fr = Arg(1); - Eterm* next; + BeamInstr *next; PreFetch(2, next); GetR(0, targ1); @@ -4582,7 +4735,7 @@ apply_bif_or_nif_epilogue: OpCase(fconv_dl): { Eterm targ1; Eterm fr = Arg(1); - Eterm* next; + BeamInstr *next; GetR(0, targ1); PreFetch(2, next); @@ -4611,7 +4764,7 @@ apply_bif_or_nif_epilogue: erl_exit(1, "fclearerror/i_fcheckerror without fpe signals (beam_emu)"); #else OpCase(fclearerror): { - Eterm* next; + BeamInstr *next; PreFetch(0, next); ERTS_FP_CHECK_INIT(c_p); @@ -4619,7 +4772,7 @@ apply_bif_or_nif_epilogue: } OpCase(i_fcheckerror): { - Eterm* next; + BeamInstr *next; PreFetch(0, next); ERTS_FP_ERROR(c_p, freg[0].fd, goto fbadarith); @@ -4633,7 +4786,7 @@ apply_bif_or_nif_epilogue: OpCase(i_fadd_lll): { - Eterm* next; + BeamInstr *next; PreFetch(3, next); ERTS_FP_CHECK_INIT(c_p); @@ -4642,7 +4795,7 @@ apply_bif_or_nif_epilogue: NextPF(3, next); } OpCase(i_fsub_lll): { - Eterm* next; + BeamInstr *next; PreFetch(3, next); ERTS_FP_CHECK_INIT(c_p); @@ -4651,7 +4804,7 @@ apply_bif_or_nif_epilogue: NextPF(3, next); } OpCase(i_fmul_lll): { - Eterm* next; + BeamInstr *next; PreFetch(3, next); ERTS_FP_CHECK_INIT(c_p); @@ -4660,7 +4813,7 @@ apply_bif_or_nif_epilogue: NextPF(3, next); } OpCase(i_fdiv_lll): { - Eterm* next; + BeamInstr *next; PreFetch(3, next); ERTS_FP_CHECK_INIT(c_p); @@ -4669,7 +4822,7 @@ apply_bif_or_nif_epilogue: NextPF(3, next); } OpCase(i_fnegate_ll): { - Eterm* next; + BeamInstr *next; PreFetch(2, next); ERTS_FP_CHECK_INIT(c_p); @@ -4736,7 +4889,7 @@ apply_bif_or_nif_epilogue: neg_o_reds = -c_p->def_arg_reg[4]; FCALLS = c_p->fcalls; SWAPIN; - switch( c_p->def_arg_reg[3] ) { + switch( c_p->def_arg_reg[3] ) { /* Halfword wont work with hipe yet! */ case HIPE_MODE_SWITCH_RES_RETURN: ASSERT(is_value(reg[0])); MoveReturn(reg[0], r(0)); @@ -4748,7 +4901,7 @@ apply_bif_or_nif_epilogue: /* This can be used to call any function value, but currently it's only used to call closures referring to unloaded modules. */ { - Eterm *next; + BeamInstr *next; next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE); SWAPIN; @@ -4881,13 +5034,15 @@ apply_bif_or_nif_epilogue: em_call_error_handler = OpCode(call_error_handler); em_call_traced_function = OpCode(call_traced_function); em_apply_bif = OpCode(apply_bif); - beam_apply[0] = (Eterm) OpCode(i_apply); - beam_apply[1] = (Eterm) OpCode(normal_exit); - beam_exit[0] = (Eterm) OpCode(error_action_code); - beam_continue_exit[0] = (Eterm) OpCode(continue_exit); - beam_return_to_trace[0] = (Eterm) OpCode(i_return_to_trace); - beam_return_trace[0] = (Eterm) OpCode(return_trace); - beam_exception_trace[0] = (Eterm) OpCode(return_trace); /* UGLY */ + + beam_apply[0] = (BeamInstr) OpCode(i_apply); + beam_apply[1] = (BeamInstr) OpCode(normal_exit); + beam_exit[0] = (BeamInstr) OpCode(error_action_code); + beam_continue_exit[0] = (BeamInstr) OpCode(continue_exit); + beam_return_to_trace[0] = (BeamInstr) OpCode(i_return_to_trace); + beam_return_trace[0] = (BeamInstr) OpCode(return_trace); + beam_exception_trace[0] = (BeamInstr) OpCode(return_trace); /* UGLY */ + beam_return_time_trace[0] = (BeamInstr) OpCode(i_return_time_trace); /* * Enter all BIFs into the export table. @@ -4897,8 +5052,10 @@ apply_bif_or_nif_epilogue: bif_table[i].name, bif_table[i].arity); bif_export[i] = ep; - ep->code[3] = (Eterm) OpCode(apply_bif); - ep->code[4] = (Eterm) bif_table[i].f; + ep->code[3] = (BeamInstr) OpCode(apply_bif); + ep->code[4] = (BeamInstr) bif_table[i].f; + /* XXX: set func info for bifs */ + ((BeamInstr*)ep->code + 3)[-5] = (BeamInstr) BeamOp(op_i_func_info_IaaI); } return; @@ -4943,6 +5100,10 @@ translate_gc_bif(void* gcf) return round_1; } else if (gcf == erts_gc_trunc_1) { return round_1; + } else if (gcf == erts_gc_binary_part_2) { + return binary_part_2; + } else if (gcf == erts_gc_binary_part_3) { + return binary_part_3; } else { erl_exit(1, "bad gc bif"); } @@ -5001,8 +5162,8 @@ Eterm error_atom[NUMBER_EXIT_CODES] = { * at the point of the original exception. */ -static Eterm* -handle_error(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf) +static BeamInstr* +handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) { Eterm* hp; Eterm Value = c_p->fvalue; @@ -5056,7 +5217,7 @@ handle_error(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf) /* Find a handler or die */ if ((c_p->catches > 0 || IS_TRACED_FL(c_p, F_EXCEPTION_TRACE)) && !(c_p->freason & EXF_PANIC)) { - Eterm *new_pc; + BeamInstr *new_pc; /* The Beam handler code (catch_end or try_end) checks reg[0] for THE_NON_VALUE to see if the previous code finished abnormally. If so, reg[1], reg[2] and reg[3] should hold the @@ -5082,30 +5243,37 @@ handle_error(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf) /* * Find the nearest catch handler */ -static Eterm* +static BeamInstr* next_catch(Process* c_p, Eterm *reg) { int active_catches = c_p->catches > 0; int have_return_to_trace = 0; Eterm *ptr, *prev, *return_to_trace_ptr = NULL; - Uint i_return_trace = beam_return_trace[0]; - Uint i_return_to_trace = beam_return_to_trace[0]; + + BeamInstr i_return_trace = beam_return_trace[0]; + BeamInstr i_return_to_trace = beam_return_to_trace[0]; + BeamInstr i_return_time_trace = beam_return_time_trace[0]; + ptr = prev = c_p->stop; ASSERT(is_CP(*ptr)); ASSERT(ptr <= STACK_START(c_p)); if (ptr == STACK_START(c_p)) return NULL; if ((is_not_CP(*ptr) || (*cp_val(*ptr) != i_return_trace && - *cp_val(*ptr) != i_return_to_trace)) + *cp_val(*ptr) != i_return_to_trace && + *cp_val(*ptr) != i_return_time_trace )) && c_p->cp) { /* Can not follow cp here - code may be unloaded */ - Uint *cpp = cp_val((Eterm) c_p->cp); + BeamInstr *cpp = c_p->cp; if (cpp == beam_exception_trace) { - erts_trace_exception(c_p, (Eterm*) ptr[0], + erts_trace_exception(c_p, cp_val(ptr[0]), reg[1], reg[2], ptr+1); /* Skip return_trace parameters */ ptr += 2; } else if (cpp == beam_return_trace) { /* Skip return_trace parameters */ ptr += 2; + } else if (cpp == beam_return_time_trace) { + /* Skip return_trace parameters */ + ptr += 1; } else if (cpp == beam_return_to_trace) { have_return_to_trace = !0; /* Record next cp */ } @@ -5123,7 +5291,7 @@ next_catch(Process* c_p, Eterm *reg) { if (is_catch(*ptr) && active_catches) goto found_catch; } if (cp_val(*prev) == beam_exception_trace) { - erts_trace_exception(c_p, (Eterm*) ptr[0], + erts_trace_exception(c_p, cp_val(ptr[0]), reg[1], reg[2], ptr+1); } /* Skip return_trace parameters */ @@ -5135,6 +5303,13 @@ next_catch(Process* c_p, Eterm *reg) { } have_return_to_trace = !0; /* Record next cp */ return_to_trace_ptr = NULL; + } else if (*cp_val(*prev) == i_return_time_trace) { + /* Skip stack frame variables */ + while (++ptr, ptr < STACK_START(c_p) && is_not_CP(*ptr)) { + if (is_catch(*ptr) && active_catches) goto found_catch; + } + /* Skip return_trace parameters */ + ptr += 1; } else { if (have_return_to_trace) { /* Record this cp as possible return_to trace cp */ @@ -5252,7 +5427,7 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) { */ static void -save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf, +save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, Eterm args) { struct StackTrace* s; int sz; @@ -5263,7 +5438,7 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf, } /* Create a container for the exception data */ - sz = (offsetof(struct StackTrace, trace) + sizeof(Eterm)*depth + sz = (offsetof(struct StackTrace, trace) + sizeof(BeamInstr *)*depth + sizeof(Eterm) - 1) / sizeof(Eterm); s = (struct StackTrace *) HAlloc(c_p, 1 + sz); /* The following fields are inside the bignum */ @@ -5298,7 +5473,6 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf, ASSERT(c_p->current); s->current = c_p->current; a = s->current[2]; - ASSERT(s->current[2] <= 3); } /* Save first stack entry */ ASSERT(pc); @@ -5350,9 +5524,10 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf, /* Save the actual stack trace */ if (depth > 0) { - Eterm *ptr, *prev = s->depth ? s->trace[s->depth-1] : NULL; - Uint i_return_trace = beam_return_trace[0]; - Uint i_return_to_trace = beam_return_to_trace[0]; + Eterm *ptr; + BeamInstr *prev = s->depth ? s->trace[s->depth-1] : NULL; + BeamInstr i_return_trace = beam_return_trace[0]; + BeamInstr i_return_to_trace = beam_return_to_trace[0]; /* * Traverse the stack backwards and add all unique continuation * pointers to the buffer, up to the maximum stack trace size. @@ -5365,7 +5540,7 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf, *cp_val(*ptr) != i_return_to_trace)) && c_p->cp) { /* Can not follow cp here - code may be unloaded */ - Uint *cpp = cp_val((Eterm) c_p->cp); + BeamInstr *cpp = c_p->cp; if (cpp == beam_exception_trace || cpp == beam_return_trace) { /* Skip return_trace parameters */ ptr += 2; @@ -5385,7 +5560,7 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf, /* Skip stack frame variables */ do ++ptr; while (is_not_CP(*ptr)); } else { - Eterm *cp = (Eterm *)(*ptr); + BeamInstr *cp = cp_val(*ptr); if (cp != prev) { /* Record non-duplicates only */ prev = cp; @@ -5457,9 +5632,12 @@ build_stacktrace(Process* c_p, Eterm exc) { struct StackTrace* s; Eterm args; int depth; - Eterm* current; + BeamInstr* current; +#if HALFWORD_HEAP + BeamInstr current_buff[3]; +#endif Eterm Where = NIL; - Eterm* next_p = &Where; + Eterm *next_p = &Where; if (! (s = get_trace_from_exc(exc))) { return NIL; @@ -5487,7 +5665,14 @@ build_stacktrace(Process* c_p, Eterm exc) { * (e.g. spawn_link(erlang, abs, [1])). */ if (current == NULL) { +#if HALFWORD_HEAP + current = current_buff; + current[0] = (BeamInstr) c_p->initial[0]; + current[1] = (BeamInstr) c_p->initial[1]; + current[2] = (BeamInstr) c_p->initial[2]; +#else current = c_p->initial; +#endif args = am_true; /* Just in case */ } else { args = get_args_from_exc(exc); @@ -5523,7 +5708,7 @@ build_stacktrace(Process* c_p, Eterm exc) { * Finally, we go through the saved continuation pointers. */ for (i = 0; i < depth; i++) { - Eterm *fi = find_function_from_pc((Eterm *) s->trace[i]); + BeamInstr *fi = find_function_from_pc((BeamInstr *) s->trace[i]); if (fi == NULL) continue; mfa = TUPLE3(hp, fi[0], fi[1], make_small(fi[2])); hp += 4; @@ -5540,7 +5725,7 @@ build_stacktrace(Process* c_p, Eterm exc) { static Eterm -call_error_handler(Process* p, Eterm* fi, Eterm* reg) +call_error_handler(Process* p, BeamInstr* fi, Eterm* reg) { Eterm* hp; Export* ep; @@ -5588,7 +5773,7 @@ call_error_handler(Process* p, Eterm* fi, Eterm* reg) } static Eterm -call_breakpoint_handler(Process* p, Eterm* fi, Eterm* reg) +call_breakpoint_handler(Process* p, BeamInstr* fi, Eterm* reg) { Eterm* hp; Export* ep; @@ -5681,7 +5866,7 @@ apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity, return ep; } -static Uint* +static BeamInstr* apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) { int arity; @@ -5763,7 +5948,7 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) return ep->address; } -static Uint* +static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity) { Export* ep; @@ -5869,7 +6054,7 @@ hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg) c_p->stop = STACK_START(c_p); c_p->catches = 0; c_p->i = beam_apply; - c_p->cp = (Eterm *) beam_apply+1; + c_p->cp = (BeamInstr *) beam_apply+1; /* * If there are no waiting messages, garbage collect and @@ -5899,7 +6084,7 @@ hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg) return 1; } -static Uint* +static BeamInstr* call_fun(Process* p, /* Current process. */ int arity, /* Number of arguments for Fun. */ Eterm* reg, /* Contents of registers. */ @@ -5919,7 +6104,7 @@ call_fun(Process* p, /* Current process. */ if (is_fun_header(hdr)) { ErlFunThing* funp = (ErlFunThing *) fun_val(fun); ErlFunEntry* fe; - Eterm* code_ptr; + BeamInstr* code_ptr; Eterm* var_ptr; int actual_arity; unsigned num_free; @@ -6014,8 +6199,12 @@ call_fun(Process* p, /* Current process. */ } } } else if (is_export_header(hdr)) { - Export* ep = (Export *) (export_val(fun))[1]; - int actual_arity = (int) ep->code[2]; + Export *ep; + int actual_arity; + + ep = *((Export **) (export_val(fun) + 1)); + actual_arity = (int) ep->code[2]; + if (arity == actual_arity) { return ep->address; } else { @@ -6082,7 +6271,7 @@ call_fun(Process* p, /* Current process. */ } } -static Eterm* +static BeamInstr* apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg) { int arity; @@ -6134,8 +6323,8 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) erts_refc_inc(&fe->refc, 2); funp->thing_word = HEADER_FUN; #ifndef HYBRID /* FIND ME! */ - funp->next = MSO(p).funs; - MSO(p).funs = funp; + funp->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*) funp; #endif funp->fe = fe; funp->num_free = num_free; @@ -6176,7 +6365,7 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity) if ((ep = export_get(&e)) == NULL) { return 0; } - return ep->address == ep->code+3 && (ep->code[3] == (Uint) em_apply_bif); + return ep->address == ep->code+3 && (ep->code[3] == (BeamInstr) em_apply_bif); } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 99fab28dce..74777e4c26 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -51,9 +51,9 @@ ErlDrvBinary* erts_gzinflate_buffer(char*, int); #define EXPORTED 2 #ifdef NO_JUMP_TABLE -# define BeamOpCode(Op) ((Uint)(Op)) +# define BeamOpCode(Op) ((BeamInstr)(Op)) #else -# define BeamOpCode(Op) ((Eterm)beam_ops[Op]) +# define BeamOpCode(Op) ((BeamInstr)beam_ops[Op]) #endif #if defined(WORDS_BIGENDIAN) @@ -94,7 +94,7 @@ typedef struct { typedef struct { unsigned type; /* Type of operand. */ - Uint val; /* Value of operand. */ + BeamInstr val; /* Value of operand. */ Uint bigarity; /* Arity for bignumbers (only). */ } GenOpArg; @@ -142,7 +142,7 @@ typedef struct { typedef struct { Eterm function; /* Tagged atom for function. */ int arity; /* Arity. */ - Eterm* address; /* Address to function in code. */ + BeamInstr* address; /* Address to function in code. */ } ExportEntry; #define MakeIffId(a, b, c, d) \ @@ -274,13 +274,12 @@ typedef struct { int num_functions; /* Number of functions in module. */ int num_labels; /* Number of labels. */ int code_buffer_size; /* Size of code buffer in words. */ - Eterm* code; /* Loaded code. */ + BeamInstr* code; /* Loaded code. */ int ci; /* Current index into loaded code. */ Label* labels; - Uint put_strings; /* Linked list of put_string instructions. */ - Uint new_bs_put_strings; /* Linked list of i_new_bs_put_string instructions. */ + BeamInstr new_bs_put_strings; /* Linked list of i_new_bs_put_string instructions. */ StringPatch* string_patches; /* Linked list of position into string table to patch. */ - Uint catches; /* Linked list of catch_yf instructions. */ + BeamInstr catches; /* Linked list of catch_yf instructions. */ unsigned loaded_size; /* Final size of code when loaded. */ byte mod_md5[16]; /* MD5 for module code. */ int may_load_nif; /* true if NIFs may later be loaded for this module */ @@ -341,7 +340,7 @@ typedef struct { #define GetTagAndValue(Stp, Tag, Val) \ do { \ - Uint __w; \ + BeamInstr __w; \ GetByte(Stp, __w); \ Tag = __w & 0x07; \ if ((__w & 0x08) == 0) { \ @@ -388,7 +387,7 @@ typedef struct { goto load_error; \ } else { \ int __n = (N); \ - Uint __result = 0; \ + BeamInstr __result = 0; \ Stp->file_left -= (unsigned) __n; \ while (__n-- > 0) { \ __result = __result << 8 | *Stp->file_p++; \ @@ -465,7 +464,7 @@ static int bin_load(Process *c_p, ErtsProcLocks c_p_locks, static void init_state(LoaderState* stp); static int insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, - Eterm* code, Uint size, Uint catches); + BeamInstr* code, Uint size, BeamInstr catches); static int scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory); static int load_atom_table(LoaderState* stp); @@ -487,9 +486,6 @@ static GenOp* const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpArg* Rest); static GenOp* gen_func_info(LoaderState* stp, GenOpArg mod, GenOpArg Func, GenOpArg arity, GenOpArg label); -static GenOp* -gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, - GenOpArg Src, GenOpArg Dst); static int freeze_code(LoaderState* stp); @@ -499,8 +495,8 @@ static void load_printf(int line, LoaderState* context, char *fmt, ...); static int transform_engine(LoaderState* st); static void id_to_string(Uint id, char* s); static void new_genop(LoaderState* stp); -static int get_int_val(LoaderState* stp, Uint len_code, Uint* result); -static int get_erlang_integer(LoaderState* stp, Uint len_code, Uint* result); +static int get_int_val(LoaderState* stp, Uint len_code, BeamInstr* result); +static int get_erlang_integer(LoaderState* stp, Uint len_code, BeamInstr* result); static int new_label(LoaderState* stp); static void new_literal_patch(LoaderState* stp, int pos); static void new_string_patch(LoaderState* stp, int pos); @@ -513,7 +509,7 @@ static Eterm compilation_info_for_module(Process* p, Eterm mod); static Eterm native_addresses(Process* p, Eterm mod); int patch_funentries(Eterm Patchlist); int patch(Eterm Addresses, Uint fe); -static int safe_mul(Uint a, Uint b, Uint* resp); +static int safe_mul(UWord a, UWord b, UWord* resp); static int must_swap_floats; @@ -591,7 +587,18 @@ erts_load_module(Process *c_p, } return result; } - +/* #define LOAD_MEMORY_HARD_DEBUG 1*/ + +#if defined(LOAD_MEMORY_HARD_DEBUG) && defined(DEBUG) +/* Requires allocators ERTS_ALLOC_UTIL_HARD_DEBUG also set in erl_alloc_util.h */ +extern void check_allocators(void); +extern void check_allocated_block(Uint type, void *blk); +#define CHKALLOC() check_allocators() +#define CHKBLK(TYPE,BLK) if ((BLK) != NULL) check_allocated_block((TYPE),(BLK)) +#else +#define CHKALLOC() /* nothing */ +#define CHKBLK(TYPE,BLK) /* nothing */ +#endif static int bin_load(Process *c_p, ErtsProcLocks c_p_locks, @@ -608,6 +615,12 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * Scan the IFF file. */ +#if defined(LOAD_MEMORY_HARD_DEBUG) && defined(DEBUG) + erts_fprintf(stderr,"Loading a module\n"); +#endif + + CHKALLOC(); + CHKBLK(ERTS_ALC_T_CODE,state.code); state.file_name = "IFF header for Beam file"; state.file_p = bytes; state.file_left = unloaded_size; @@ -619,6 +632,7 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * Read the header for the code chunk. */ + CHKBLK(ERTS_ALC_T_CODE,state.code); define_file(&state, "code chunk header", CODE_CHUNK); if (!read_code_header(&state)) { goto load_error; @@ -628,6 +642,7 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * Read the atom table. */ + CHKBLK(ERTS_ALC_T_CODE,state.code); define_file(&state, "atom table", ATOM_CHUNK); if (!load_atom_table(&state)) { goto load_error; @@ -637,6 +652,7 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * Read the import table. */ + CHKBLK(ERTS_ALC_T_CODE,state.code); define_file(&state, "import table", IMP_CHUNK); if (!load_import_table(&state)) { goto load_error; @@ -646,6 +662,7 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * Read the lambda (fun) table. */ + CHKBLK(ERTS_ALC_T_CODE,state.code); if (state.chunks[LAMBDA_CHUNK].size > 0) { define_file(&state, "lambda (fun) table", LAMBDA_CHUNK); if (!read_lambda_table(&state)) { @@ -657,6 +674,7 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * Read the literal table. */ + CHKBLK(ERTS_ALC_T_CODE,state.code); if (state.chunks[LITERAL_CHUNK].size > 0) { define_file(&state, "literals table (constant pool)", LITERAL_CHUNK); if (!read_literal_table(&state)) { @@ -668,18 +686,25 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * Load the code chunk. */ + CHKBLK(ERTS_ALC_T_CODE,state.code); state.file_name = "code chunk"; state.file_p = state.code_start; state.file_left = state.code_size; - if (!load_code(&state) || !freeze_code(&state)) { + if (!load_code(&state)) { + goto load_error; + } + CHKBLK(ERTS_ALC_T_CODE,state.code); + if (!freeze_code(&state)) { goto load_error; } + /* * Read and validate the export table. (This must be done after * loading the code, because it contains labels.) */ + CHKBLK(ERTS_ALC_T_CODE,state.code); define_file(&state, "export table", EXP_CHUNK); if (!read_export_table(&state)) { goto load_error; @@ -690,16 +715,25 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * exported and imported functions. This can't fail. */ + CHKBLK(ERTS_ALC_T_CODE,state.code); rval = insert_new_code(c_p, c_p_locks, state.group_leader, state.module, state.code, state.loaded_size, state.catches); if (rval < 0) { goto load_error; } + CHKBLK(ERTS_ALC_T_CODE,state.code); final_touch(&state); /* * Loading succeded. */ + CHKBLK(ERTS_ALC_T_CODE,state.code); +#if defined(LOAD_MEMORY_HARD_DEBUG) && defined(DEBUG) + erts_fprintf(stderr,"Loaded %T\n",*modp); +#if 0 + debug_dump_code(state.code,state.ci); +#endif +#endif rval = 0; state.code = NULL; /* Prevent code from being freed. */ *modp = state.module; @@ -791,7 +825,7 @@ init_state(LoaderState* stp) static int insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, - Eterm group_leader, Eterm module, Eterm* code, Uint size, Uint catches) + Eterm group_leader, Eterm module, BeamInstr* code, Uint size, BeamInstr catches) { Module* modp; int rval; @@ -833,7 +867,7 @@ insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, modules[i] = modules[i-1]; } modules[i].start = code; - modules[i].end = (Eterm *) (((byte *)code) + size); + modules[i].end = (BeamInstr *) (((byte *)code) + size); num_loaded_modules++; mid_module = &modules[num_loaded_modules/2]; return 0; @@ -1083,7 +1117,7 @@ load_import_table(LoaderState* stp) * the BIF function. */ if ((e = erts_find_export_entry(mod, func, arity)) != NULL) { - if (e->code[3] == (Uint) em_apply_bif) { + if (e->code[3] == (BeamInstr) em_apply_bif) { stp->import[i].bf = (BifFunction) e->code[4]; if (func == am_load_nif && mod == am_erlang && arity == 2) { stp->may_load_nif = 1; @@ -1151,7 +1185,7 @@ read_export_table(LoaderState* stp) * redefine). */ if ((e = erts_find_export_entry(stp->module, func, arity)) != NULL) { - if (e->code[3] == (Uint) em_apply_bif) { + if (e->code[3] == (BeamInstr) em_apply_bif) { int j; for (j = 0; j < sizeof(allow_redef)/sizeof(allow_redef[0]); j++) { @@ -1220,7 +1254,7 @@ static int read_literal_table(LoaderState* stp) { int i; - Uint uncompressed_sz; + BeamInstr uncompressed_sz; byte* uncompressed = 0; GetInt(stp, 4, uncompressed_sz); @@ -1338,8 +1372,8 @@ read_code_header(LoaderState* stp) * Initialize code area. */ stp->code_buffer_size = erts_next_heap_size(2048 + stp->num_functions, 0); - stp->code = (Eterm*) erts_alloc(ERTS_ALC_T_CODE, - sizeof(Eterm) * stp->code_buffer_size); + stp->code = (BeamInstr *) erts_alloc(ERTS_ALC_T_CODE, + sizeof(BeamInstr) * stp->code_buffer_size); stp->code[MI_NUM_FUNCTIONS] = stp->num_functions; stp->ci = MI_FUNCTIONS + stp->num_functions + 1; @@ -1350,7 +1384,6 @@ read_code_header(LoaderState* stp) stp->code[MI_COMPILE_SIZE_ON_HEAP] = 0; stp->code[MI_NUM_BREAKPOINTS] = 0; - stp->put_strings = 0; stp->new_bs_put_strings = 0; stp->catches = 0; return 1; @@ -1365,16 +1398,18 @@ read_code_header(LoaderState* stp) LoadError2(Stp, "bad tag %d; expected %d", Actual, Expected); \ } else {} -#define Need(w) \ - ASSERT(ci <= code_buffer_size); \ - if (code_buffer_size < ci+(w)) { \ - code_buffer_size = erts_next_heap_size(ci+(w), 0); \ - stp->code = code \ - = (Eterm *) erts_realloc(ERTS_ALC_T_CODE, \ - (void *) code, \ - code_buffer_size * sizeof(Eterm)); \ - } +#define CodeNeed(w) do { \ + ASSERT(ci <= code_buffer_size); \ + if (code_buffer_size < ci+(w)) { \ + code_buffer_size = erts_next_heap_size(ci+(w), 0); \ + stp->code = code \ + = (BeamInstr *) erts_realloc(ERTS_ALC_T_CODE, \ + (void *) code, \ + code_buffer_size * sizeof(BeamInstr)); \ + } \ +} while (0) +#define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm)))) static int @@ -1387,7 +1422,7 @@ load_code(LoaderState* stp) char* sign; int arg; /* Number of current argument. */ int num_specific; /* Number of specific ops for current. */ - Eterm* code; + BeamInstr* code; int code_buffer_size; int specific; Uint last_label = 0; /* Number of last label. */ @@ -1446,7 +1481,7 @@ load_code(LoaderState* stp) if (((First) & 0x08) == 0) { \ Val = (First) >> 4; \ } else if (((First) & 0x10) == 0) { \ - Uint __w; \ + BeamInstr __w; \ GetByte(Stp, __w); \ Val = (((First) >> 5) << 8) | __w; \ } else { \ @@ -1455,7 +1490,7 @@ load_code(LoaderState* stp) } while (0) for (arg = 0; arg < arity; arg++) { - Uint first; + BeamInstr first; GetByte(stp, first); last_op->a[arg].type = first & 0x07; @@ -1464,7 +1499,7 @@ load_code(LoaderState* stp) if ((first & 0x08) == 0) { last_op->a[arg].val = first >> 4; } else if ((first & 0x10) == 0) { - Uint w; + BeamInstr w; GetByte(stp, w); ASSERT(first < 0x800); last_op->a[arg].val = ((first >> 5) << 8) | w; @@ -1523,7 +1558,7 @@ load_code(LoaderState* stp) break; case TAG_z: { - Uint ext_tag; + BeamInstr ext_tag; unsigned tag; GetValue(stp, first, ext_tag); @@ -1531,14 +1566,15 @@ load_code(LoaderState* stp) case 0: /* Floating point number */ { Eterm* hp; -# ifndef ARCH_64 +/* XXX:PaN - Halfword should use ARCH_64 variant instead */ +#if !defined(ARCH_64) || HALFWORD_HEAP Uint high, low; # endif last_op->a[arg].val = new_literal(stp, &hp, FLOAT_SIZE_OBJECT); hp[0] = HEADER_FLONUM; last_op->a[arg].type = TAG_q; -# ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP GetInt(stp, 8, hp[1]); # else GetInt(stp, 4, high); @@ -1575,10 +1611,10 @@ load_code(LoaderState* stp) break; case 3: /* Allocation list. */ { - Uint n; - Uint type; - Uint val; - Uint words = 0; + BeamInstr n; + BeamInstr type; + BeamInstr val; + BeamInstr words = 0; stp->new_float_instructions = 1; GetTagAndValue(stp, tag, n); @@ -1607,7 +1643,7 @@ load_code(LoaderState* stp) } case 4: /* Literal. */ { - Uint val; + BeamInstr val; GetTagAndValue(stp, tag, val); VerifyTag(stp, tag, TAG_u); @@ -1734,7 +1770,7 @@ load_code(LoaderState* stp) } stp->specific_op = specific; - Need(opc[stp->specific_op].sz+2); /* Extra margin for packing */ + CodeNeed(opc[stp->specific_op].sz+2); /* Extra margin for packing */ code[ci++] = BeamOpCode(stp->specific_op); } @@ -1772,7 +1808,7 @@ load_code(LoaderState* stp) case 'c': /* Tagged constant */ switch (tag) { case TAG_i: - code[ci++] = make_small(tmp_op->a[arg].val); + code[ci++] = (BeamInstr) make_small((Uint) tmp_op->a[arg].val); break; case TAG_a: code[ci++] = tmp_op->a[arg].val; @@ -1802,7 +1838,7 @@ load_code(LoaderState* stp) code[ci++] = make_yreg(tmp_op->a[arg].val); break; case TAG_i: - code[ci++] = make_small(tmp_op->a[arg].val); + code[ci++] = (BeamInstr) make_small((Uint)tmp_op->a[arg].val); break; case TAG_a: code[ci++] = tmp_op->a[arg].val; @@ -1896,12 +1932,12 @@ load_code(LoaderState* stp) if (stp->import[i].bf == NULL) { LoadError1(stp, "not a BIF: import table index %d", i); } - code[ci++] = (Eterm) stp->import[i].bf; + code[ci++] = (BeamInstr) stp->import[i].bf; break; case 'P': /* Byte offset into tuple */ VerifyTag(stp, tag, TAG_u); tmp = tmp_op->a[arg].val; - code[ci++] = (Eterm) ((tmp_op->a[arg].val+1) * sizeof(Eterm *)); + code[ci++] = (BeamInstr) ((tmp_op->a[arg].val+1) * sizeof(Eterm)); break; case 'l': /* Floating point register. */ VerifyTag(stp, tag_to_letter[tag], *sign); @@ -1925,17 +1961,17 @@ load_code(LoaderState* stp) for ( ; arg < tmp_op->arity; arg++) { switch (tmp_op->a[arg].type) { case TAG_i: - Need(1); + CodeNeed(1); code[ci++] = make_small(tmp_op->a[arg].val); break; case TAG_u: case TAG_a: case TAG_v: - Need(1); + CodeNeed(1); code[ci++] = tmp_op->a[arg].val; break; case TAG_f: - Need(1); + CodeNeed(1); code[ci] = stp->labels[tmp_op->a[arg].val].patches; stp->labels[tmp_op->a[arg].val].patches = ci; ci++; @@ -1947,24 +1983,41 @@ load_code(LoaderState* stp) lit = stp->literals[tmp_op->a[arg].val].term; if (is_big(lit)) { Eterm* bigp; + Eterm *tmp; Uint size; + Uint term_size; bigp = big_val(lit); - size = bignum_header_arity(*bigp); - Need(size+1); - code[ci++] = *bigp++; - while (size-- > 0) { - code[ci++] = *bigp++; + term_size = bignum_header_arity(*bigp); + size = TermWords(term_size + 1); + CodeNeed(size); + tmp = (Eterm *) (code + ci); + *tmp++ = *bigp++; + while (term_size-- > 0) { + *tmp++ = *bigp++; } + ci +=size; } else if (is_float(lit)) { -#ifdef ARCH_64 - Need(1); +#if defined(ARCH_64) && !HALFWORD_HEAP + CodeNeed(1); code[ci++] = float_val(stp->literals[tmp_op->a[arg].val].term)[1]; +#elif HALFWORD_HEAP + Eterm* fptr; + Uint size; + Eterm *tmp; + + fptr = float_val(stp->literals[tmp_op->a[arg].val].term)+1; + size = TermWords(2); + CodeNeed(size); + tmp = (Eterm *) (code + ci); + *tmp++ = *fptr++; + *tmp = *fptr; + ci += size; #else Eterm* fptr; fptr = float_val(stp->literals[tmp_op->a[arg].val].term)+1; - Need(2); + CodeNeed(2); code[ci++] = *fptr++; code[ci++] = *fptr; #endif @@ -1984,10 +2037,10 @@ load_code(LoaderState* stp) */ if (opc[stp->specific_op].pack[0]) { char* prog; /* Program for packing engine. */ - Uint stack[8]; /* Stack. */ - Uint* sp = stack; /* Points to next free position. */ - Uint packed = 0; /* Accumulator for packed operations. */ - + BeamInstr stack[8]; /* Stack. */ + BeamInstr* sp = stack; /* Points to next free position. */ + BeamInstr packed = 0; /* Accumulator for packed operations. */ + for (prog = opc[stp->specific_op].pack; *prog; prog++) { switch (*prog) { case 'g': /* Get instruction; push on stack. */ @@ -2000,7 +2053,7 @@ load_code(LoaderState* stp) packed = (packed << BEAM_TIGHT_SHIFT) | code[--ci]; break; case '6': /* Shift 16 steps */ - packed = (packed << 16) | code[--ci]; + packed = (packed << BEAM_LOOSE_SHIFT) | code[--ci]; break; case 'p': /* Put instruction (from stack). */ code[ci++] = *--sp; @@ -2037,9 +2090,9 @@ load_code(LoaderState* stp) /* Must make room for call_nif op */ int pad = MIN_FUNC_SZ - (finfo_ix - last_func_start); ASSERT(pad > 0 && pad < MIN_FUNC_SZ); - Need(pad); - sys_memmove(&code[finfo_ix+pad], &code[finfo_ix], FINFO_SZ*sizeof(Eterm)); - sys_memset(&code[finfo_ix], 0, pad*sizeof(Eterm)); + CodeNeed(pad); + sys_memmove(&code[finfo_ix+pad], &code[finfo_ix], FINFO_SZ*sizeof(BeamInstr)); + sys_memset(&code[finfo_ix], 0, pad*sizeof(BeamInstr)); ci += pad; stp->labels[last_label].value += pad; } @@ -2050,6 +2103,7 @@ load_code(LoaderState* stp) */ stp->function = code[ci-2]; stp->arity = code[ci-1]; + ASSERT(stp->labels[last_label].value == ci - FINFO_SZ); offset = MI_FUNCTIONS + function_number; code[offset] = stp->labels[last_label].patches; @@ -2072,34 +2126,6 @@ load_code(LoaderState* stp) /* Remember offset for the on_load function. */ stp->on_load = ci; break; - case op_put_string_IId: - { - /* - * At entry: - * - * code[ci-4] &&lb_put_string_IId - * code[ci-3] length of string - * code[ci-2] offset into string table - * code[ci-1] destination register - * - * Since we don't know the address of the string table yet, - * just check the offset and length for validity, and use - * the instruction field as a link field to link all put_string - * instructions into a single linked list. At exit: - * - * code[ci-4] pointer to next put_string instruction (or 0 - * if this is the last) - */ - Uint offset = code[ci-2]; - Uint len = code[ci-3]; - unsigned strtab_size = stp->chunks[STR_CHUNK].size; - if (offset > strtab_size || offset + len > strtab_size) { - LoadError2(stp, "invalid string reference %d, size %d", offset, len); - } - code[ci-4] = stp->put_strings; - stp->put_strings = ci - 4; - } - break; case op_bs_put_string_II: { /* @@ -2161,7 +2187,6 @@ load_code(LoaderState* stp) } } -#undef Need load_error: return 0; @@ -2373,7 +2398,7 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, GenOpArg Flags, GenOpArg Dst) { GenOp* op; - Uint bits; + UWord bits; NEW_GENOP(stp, op); @@ -2838,14 +2863,14 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time) op->a[1].type = TAG_u; if (Time.type == TAG_i && (timeout = Time.val) >= 0 && -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP (timeout >> 32) == 0 #else 1 #endif ) { op->a[1].val = timeout; -#if !defined(ARCH_64) +#if !defined(ARCH_64) || HALFWORD_HEAP } else if (Time.type == TAG_q) { Eterm big; @@ -2856,11 +2881,13 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time) if (big_arity(big) > 1 || big_sign(big)) { goto error; } else { - (void) term_to_Uint(big, &op->a[1].val); + Uint u; + (void) term_to_Uint(big, &u); + op->a[1].val = (BeamInstr) u; } #endif } else { -#if !defined(ARCH_64) +#if !defined(ARCH_64) || HALFWORD_HEAP error: #endif op->op = genop_i_wait_error_0; @@ -2883,14 +2910,14 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time) op->a[1].type = TAG_u; if (Time.type == TAG_i && (timeout = Time.val) >= 0 && -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP (timeout >> 32) == 0 #else 1 #endif ) { op->a[1].val = timeout; -#ifndef ARCH_64 +#if !defined(ARCH_64) || HALFWORD_HEAP } else if (Time.type == TAG_q) { Eterm big; @@ -2901,11 +2928,13 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time) if (big_arity(big) > 1 || big_sign(big)) { goto error; } else { - (void) term_to_Uint(big, &op->a[1].val); + Uint u; + (void) term_to_Uint(big, &u); + op->a[1].val = (BeamInstr) u; } #endif } else { -#ifndef ARCH_64 +#if !defined(ARCH_64) || HALFWORD_HEAP error: #endif op->op = genop_i_wait_error_locked_0; @@ -3321,15 +3350,21 @@ gen_make_fun2(LoaderState* stp, GenOpArg idx) op->op = genop_i_make_fun_2; op->arity = 2; op->a[0].type = TAG_u; - op->a[0].val = (Uint) fe; + op->a[0].val = (BeamInstr) fe; op->a[1].type = TAG_u; op->a[1].val = stp->lambdas[idx.val].num_free; op->next = NULL; return op; } - +/* + * Rewrite gc_bifs with one parameter (the common case). Utilized + * in ops.tab to rewrite instructions calling bif's in guards + * to use a garbage collecting implementation. The instructions + * are sometimes once again rewritten to handle literals (putting the + * parameter in the mostly unused r[0] before the instruction is executed). + */ static GenOp* -gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, +gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, GenOpArg Src, GenOpArg Dst) { GenOp* op; @@ -3341,22 +3376,24 @@ gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, op->a[0] = Fail; op->a[1].type = TAG_u; bf = stp->import[Bif.val].bf; + /* The translations here need to have a reverse counterpart in + beam_emu.c:translate_gc_bif for error handling to work properly. */ if (bf == length_1) { - op->a[1].val = (Uint) (void *) erts_gc_length_1; + op->a[1].val = (BeamInstr) (void *) erts_gc_length_1; } else if (bf == size_1) { - op->a[1].val = (Uint) (void *) erts_gc_size_1; + op->a[1].val = (BeamInstr) (void *) erts_gc_size_1; } else if (bf == bit_size_1) { - op->a[1].val = (Uint) (void *) erts_gc_bit_size_1; + op->a[1].val = (BeamInstr) (void *) erts_gc_bit_size_1; } else if (bf == byte_size_1) { - op->a[1].val = (Uint) (void *) erts_gc_byte_size_1; + op->a[1].val = (BeamInstr) (void *) erts_gc_byte_size_1; } else if (bf == abs_1) { - op->a[1].val = (Uint) (void *) erts_gc_abs_1; + op->a[1].val = (BeamInstr) (void *) erts_gc_abs_1; } else if (bf == float_1) { - op->a[1].val = (Uint) (void *) erts_gc_float_1; + op->a[1].val = (BeamInstr) (void *) erts_gc_float_1; } else if (bf == round_1) { - op->a[1].val = (Uint) (void *) erts_gc_round_1; + op->a[1].val = (BeamInstr) (void *) erts_gc_round_1; } else if (bf == trunc_1) { - op->a[1].val = (Uint) (void *) erts_gc_trunc_1; + op->a[1].val = (BeamInstr) (void *) erts_gc_trunc_1; } else { abort(); } @@ -3367,6 +3404,77 @@ gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, return op; } +/* + * This is used by the ops.tab rule that rewrites gc_bifs with two parameters + * The instruction returned is then again rewritten to an i_load instruction + * folowed by i_gc_bif2_jIId, to handle literals properly. + * As opposed to the i_gc_bif1_jIsId, the instruction i_gc_bif2_jIId is + * always rewritten, regardless of if there actually are any literals. + */ +static GenOp* +gen_guard_bif2(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, + GenOpArg S1, GenOpArg S2, GenOpArg Dst) +{ + GenOp* op; + BifFunction bf; + + NEW_GENOP(stp, op); + op->op = genop_ii_gc_bif2_6; + op->arity = 6; + op->a[0] = Fail; + op->a[1].type = TAG_u; + bf = stp->import[Bif.val].bf; + /* The translations here need to have a reverse counterpart in + beam_emu.c:translate_gc_bif for error handling to work properly. */ + if (bf == binary_part_2) { + op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_2; + } else { + abort(); + } + op->a[2] = S1; + op->a[3] = S2; + op->a[4] = Live; + op->a[5] = Dst; + op->next = NULL; + return op; +} + +/* + * This is used by the ops.tab rule that rewrites gc_bifs with three parameters + * The instruction returned is then again rewritten to a move instruction that + * uses r[0] for temp storage, followed by an i_load instruction, + * folowed by i_gc_bif3_jIsId, to handle literals properly. Rewriting + * always occur, as with the gc_bif2 counterpart. + */ +static GenOp* +gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, + GenOpArg S1, GenOpArg S2, GenOpArg S3, GenOpArg Dst) +{ + GenOp* op; + BifFunction bf; + + NEW_GENOP(stp, op); + op->op = genop_ii_gc_bif3_7; + op->arity = 7; + op->a[0] = Fail; + op->a[1].type = TAG_u; + bf = stp->import[Bif.val].bf; + /* The translations here need to have a reverse counterpart in + beam_emu.c:translate_gc_bif for error handling to work properly. */ + if (bf == binary_part_3) { + op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_3; + } else { + abort(); + } + op->a[2] = S1; + op->a[3] = S2; + op->a[4] = S3; + op->a[5] = Live; + op->a[6] = Dst; + op->next = NULL; + return op; +} + /* * Freeze the code in memory, move the string table into place, @@ -3376,7 +3484,8 @@ gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, static int freeze_code(LoaderState* stp) { - Eterm* code = stp->code; + BeamInstr* code = stp->code; + Uint *literal_end = NULL; Uint index; int i; byte* str_table; @@ -3401,46 +3510,49 @@ freeze_code(LoaderState* stp) * Calculate the final size of the code. */ - size = (stp->ci + stp->total_literal_size) * sizeof(Eterm) + + size = (stp->ci * sizeof(BeamInstr)) + (stp->total_literal_size * sizeof(Eterm)) + strtab_size + attr_size + compile_size; /* * Move the code to its final location. */ - code = (Eterm *) erts_realloc(ERTS_ALC_T_CODE, (void *) code, size); - + code = (BeamInstr *) erts_realloc(ERTS_ALC_T_CODE, (void *) code, size); + CHKBLK(ERTS_ALC_T_CODE,code); /* * Place a pointer to the op_int_code_end instruction in the * function table in the beginning of the file. */ - code[MI_FUNCTIONS+stp->num_functions] = (Eterm) (code + stp->ci - 1); + code[MI_FUNCTIONS+stp->num_functions] = (BeamInstr) (code + stp->ci - 1); + CHKBLK(ERTS_ALC_T_CODE,code); /* * Store the pointer to the on_load function. */ if (stp->on_load) { - code[MI_ON_LOAD_FUNCTION_PTR] = (Eterm) (code + stp->on_load); + code[MI_ON_LOAD_FUNCTION_PTR] = (BeamInstr) (code + stp->on_load); } else { code[MI_ON_LOAD_FUNCTION_PTR] = 0; } + CHKBLK(ERTS_ALC_T_CODE,code); + literal_end = (Uint *) (code+stp->ci); /* * Place the literal heap directly after the code and fix up all - * put_literal instructions that refer to it. + * instructions that refer to it. */ { - Eterm* ptr; - Eterm* low; - Eterm* high; + Uint* ptr; + Uint* low; + Uint* high; LiteralPatch* lp; - low = code+stp->ci; + low = (Uint *) (code+stp->ci); high = low + stp->total_literal_size; - code[MI_LITERALS_START] = (Eterm) low; - code[MI_LITERALS_END] = (Eterm) high; + code[MI_LITERALS_START] = (BeamInstr) low; + code[MI_LITERALS_END] = (BeamInstr) high; ptr = low; for (i = 0; i < stp->num_literals; i++) { Uint offset; @@ -3472,7 +3584,7 @@ freeze_code(LoaderState* stp) } lp = stp->literal_patches; while (lp != 0) { - Uint* op_ptr; + BeamInstr* op_ptr; Uint literal; Literal* lit; @@ -3485,53 +3597,48 @@ freeze_code(LoaderState* stp) op_ptr[0] = literal; lp = lp->next; } - stp->ci += stp->total_literal_size; + literal_end += stp->total_literal_size; } /* * Place the string table and, optionally, attributes, after the literal heap. */ + CHKBLK(ERTS_ALC_T_CODE,code); - sys_memcpy(code+stp->ci, stp->chunks[STR_CHUNK].start, strtab_size); - str_table = (byte *) (code+stp->ci); + sys_memcpy(literal_end, stp->chunks[STR_CHUNK].start, strtab_size); + CHKBLK(ERTS_ALC_T_CODE,code); + str_table = (byte *) literal_end; if (attr_size) { byte* attr = str_table + strtab_size; sys_memcpy(attr, stp->chunks[ATTR_CHUNK].start, stp->chunks[ATTR_CHUNK].size); - code[MI_ATTR_PTR] = (Eterm) attr; - code[MI_ATTR_SIZE] = (Eterm) stp->chunks[ATTR_CHUNK].size; + code[MI_ATTR_PTR] = (BeamInstr) attr; + code[MI_ATTR_SIZE] = (BeamInstr) stp->chunks[ATTR_CHUNK].size; decoded_size = erts_decode_ext_size(attr, attr_size, 0); if (decoded_size < 0) { LoadError0(stp, "bad external term representation of module attributes"); } code[MI_ATTR_SIZE_ON_HEAP] = decoded_size; } + CHKBLK(ERTS_ALC_T_CODE,code); if (compile_size) { byte* compile_info = str_table + strtab_size + attr_size; + CHKBLK(ERTS_ALC_T_CODE,code); sys_memcpy(compile_info, stp->chunks[COMPILE_CHUNK].start, stp->chunks[COMPILE_CHUNK].size); - code[MI_COMPILE_PTR] = (Eterm) compile_info; - code[MI_COMPILE_SIZE] = (Eterm) stp->chunks[COMPILE_CHUNK].size; + CHKBLK(ERTS_ALC_T_CODE,code); + code[MI_COMPILE_PTR] = (BeamInstr) compile_info; + CHKBLK(ERTS_ALC_T_CODE,code); + code[MI_COMPILE_SIZE] = (BeamInstr) stp->chunks[COMPILE_CHUNK].size; + CHKBLK(ERTS_ALC_T_CODE,code); decoded_size = erts_decode_ext_size(compile_info, compile_size, 0); + CHKBLK(ERTS_ALC_T_CODE,code); if (decoded_size < 0) { LoadError0(stp, "bad external term representation of compilation information"); } + CHKBLK(ERTS_ALC_T_CODE,code); code[MI_COMPILE_SIZE_ON_HEAP] = decoded_size; } - - - /* - * Go through all put_strings instructions, restore the pointer to - * the instruction and convert string offsets to pointers (to the - * LAST character). - */ - - index = stp->put_strings; - while (index != 0) { - Uint next = code[index]; - code[index] = BeamOpCode(op_put_string_IId); - code[index+2] = (Uint) (str_table + code[index+2] + code[index+1] - 1); - index = next; - } + CHKBLK(ERTS_ALC_T_CODE,code); /* * Go through all i_new_bs_put_strings instructions, restore the pointer to @@ -3543,23 +3650,25 @@ freeze_code(LoaderState* stp) while (index != 0) { Uint next = code[index]; code[index] = BeamOpCode(op_bs_put_string_II); - code[index+2] = (Uint) (str_table + code[index+2]); + code[index+2] = (BeamInstr) (str_table + code[index+2]); index = next; } + CHKBLK(ERTS_ALC_T_CODE,code); { StringPatch* sp = stp->string_patches; while (sp != 0) { - Uint* op_ptr; + BeamInstr* op_ptr; byte* strp; op_ptr = code + sp->pos; strp = str_table + op_ptr[0]; - op_ptr[0] = (Eterm) strp; + op_ptr[0] = (BeamInstr) strp; sp = sp->next; } } + CHKBLK(ERTS_ALC_T_CODE,code); /* * Resolve all labels. @@ -3579,10 +3688,11 @@ freeze_code(LoaderState* stp) ASSERT(this_patch < stp->ci); next_patch = code[this_patch]; ASSERT(next_patch < stp->ci); - code[this_patch] = (Uint) (code + value); + code[this_patch] = (BeamInstr) (code + value); this_patch = next_patch; } } + CHKBLK(ERTS_ALC_T_CODE,code); /* * Fix all catch_yf instructions. @@ -3590,13 +3700,14 @@ freeze_code(LoaderState* stp) index = stp->catches; catches = BEAM_CATCHES_NIL; while (index != 0) { - Uint next = code[index]; + BeamInstr next = code[index]; code[index] = BeamOpCode(op_catch_yf); - catches = beam_catches_cons((Uint*)code[index+2], catches); + catches = beam_catches_cons((BeamInstr *)code[index+2], catches); code[index+2] = make_catch(catches); index = next; } stp->catches = catches; + CHKBLK(ERTS_ALC_T_CODE,code); /* * Save the updated code pointer and code size. @@ -3605,6 +3716,7 @@ freeze_code(LoaderState* stp) stp->code = code; stp->loaded_size = size; + CHKBLK(ERTS_ALC_T_CODE,code); return 1; load_error: @@ -3638,7 +3750,7 @@ final_touch(LoaderState* stp) * callable yet. */ ep->address = ep->code+3; - ep->code[4] = (Eterm) stp->export[i].address; + ep->code[4] = (BeamInstr) stp->export[i].address; } } @@ -3650,14 +3762,14 @@ final_touch(LoaderState* stp) Eterm mod; Eterm func; Uint arity; - Uint import; + BeamInstr import; Uint current; Uint next; mod = stp->import[i].module; func = stp->import[i].function; arity = stp->import[i].arity; - import = (Uint) erts_export_put(mod, func, arity); + import = (BeamInstr) erts_export_put(mod, func, arity); current = stp->import[i].patches; while (current != 0) { ASSERT(current < stp->ci); @@ -3675,7 +3787,7 @@ final_touch(LoaderState* stp) for (i = 0; i < stp->num_lambdas; i++) { unsigned entry_label = stp->lambdas[i].label; ErlFunEntry* fe = stp->lambdas[i].fe; - Eterm* code_ptr = (Eterm *) (stp->code + stp->labels[entry_label].value); + BeamInstr* code_ptr = (BeamInstr *) (stp->code + stp->labels[entry_label].value); if (fe->address[0] != 0) { /* @@ -3807,7 +3919,7 @@ transform_engine(LoaderState* st) if (i >= st->num_imports || st->import[i].bf == NULL) goto restart; if (bif_number != -1 && - bif_export[bif_number]->code[4] != (Uint) st->import[i].bf) { + bif_export[bif_number]->code[4] != (BeamInstr) st->import[i].bf) { goto restart; } } @@ -4071,7 +4183,7 @@ load_printf(int line, LoaderState* context, char *fmt,...) static int -get_int_val(LoaderState* stp, Uint len_code, Uint* result) +get_int_val(LoaderState* stp, Uint len_code, BeamInstr* result) { Uint count; Uint val; @@ -4103,7 +4215,7 @@ get_int_val(LoaderState* stp, Uint len_code, Uint* result) static int -get_erlang_integer(LoaderState* stp, Uint len_code, Uint* result) +get_erlang_integer(LoaderState* stp, Uint len_code, BeamInstr* result) { Uint count; Sint val; @@ -4124,11 +4236,12 @@ get_erlang_integer(LoaderState* stp, Uint len_code, Uint* result) count = len_code + 2; } else { Uint tag; + UWord len_word; ASSERT(len_code == 7); - GetTagAndValue(stp, tag, len_code); + GetTagAndValue(stp, tag, len_word); VerifyTag(stp, TAG_u, tag); - count = len_code + 9; + count = len_word + 9; } /* @@ -4376,7 +4489,7 @@ functions_in_module(Process* p, /* Process whose heap to use. */ Eterm mod) /* Tagged atom for module. */ { Module* modp; - Eterm* code; + BeamInstr* code; int i; Uint num_functions; Eterm* hp; @@ -4394,9 +4507,9 @@ functions_in_module(Process* p, /* Process whose heap to use. */ num_functions = code[MI_NUM_FUNCTIONS]; hp = HAlloc(p, 5*num_functions); for (i = num_functions-1; i >= 0 ; i--) { - Eterm* func_info = (Eterm *) code[MI_FUNCTIONS+i]; - Eterm name = func_info[3]; - int arity = func_info[4]; + BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i]; + Eterm name = (Eterm) func_info[3]; + int arity = (int) func_info[4]; Eterm tuple; ASSERT(is_atom(name)); @@ -4419,7 +4532,7 @@ static Eterm native_addresses(Process* p, Eterm mod) { Module* modp; - Eterm* code; + BeamInstr* code; int i; Eterm* hp; Uint num_functions; @@ -4442,9 +4555,9 @@ native_addresses(Process* p, Eterm mod) hp = HAlloc(p, need); hp_end = hp + need; for (i = num_functions-1; i >= 0 ; i--) { - Eterm* func_info = (Eterm *) code[MI_FUNCTIONS+i]; - Eterm name = func_info[3]; - int arity = func_info[4]; + BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i]; + Eterm name = (Eterm) func_info[3]; + int arity = (int) func_info[4]; Eterm tuple; ASSERT(is_atom(name)); @@ -4486,7 +4599,7 @@ exported_from_module(Process* p, /* Process whose heap to use. */ Eterm tuple; if (ep->address == ep->code+3 && - ep->code[3] == (Eterm) em_call_error_handler) { + ep->code[3] == (BeamInstr) em_call_error_handler) { /* There is a call to the function, but it does not exist. */ continue; } @@ -4519,7 +4632,7 @@ attributes_for_module(Process* p, /* Process whose heap to use. */ { Module* modp; - Eterm* code; + BeamInstr* code; Eterm* hp; byte* ext; Eterm result = NIL; @@ -4559,7 +4672,7 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */ Eterm mod) /* Tagged atom for module. */ { Module* modp; - Eterm* code; + BeamInstr* code; Eterm* hp; byte* ext; Eterm result = NIL; @@ -4591,8 +4704,8 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */ /* * Returns a pointer to {module, function, arity}, or NULL if not found. */ -Eterm* -find_function_from_pc(Eterm* pc) +BeamInstr * +find_function_from_pc(BeamInstr* pc) { Range* low = modules; Range* high = low + num_loaded_modules; @@ -4604,9 +4717,9 @@ find_function_from_pc(Eterm* pc) } else if (pc > mid->end) { low = mid + 1; } else { - Eterm** low1 = (Eterm **) (mid->start + MI_FUNCTIONS); - Eterm** high1 = low1 + mid->start[MI_NUM_FUNCTIONS]; - Eterm** mid1; + BeamInstr** low1 = (BeamInstr **) (mid->start + MI_FUNCTIONS); + BeamInstr** high1 = low1 + mid->start[MI_NUM_FUNCTIONS]; + BeamInstr** mid1; while (low1 < high1) { mid1 = low1 + (high1-low1) / 2; @@ -4719,10 +4832,10 @@ code_module_md5_1(Process* p, Eterm Bin) #define WORDS_PER_FUNCTION 6 -static Eterm* -make_stub(Eterm* fp, Eterm mod, Eterm func, Uint arity, Uint native, Eterm OpCode) +static BeamInstr* +make_stub(BeamInstr* fp, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode) { - fp[0] = (Eterm) BeamOp(op_i_func_info_IaaI); + fp[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI); fp[1] = native; fp[2] = mod; fp[3] = func; @@ -4741,14 +4854,14 @@ static byte* stub_copy_info(LoaderState* stp, int chunk, /* Chunk: ATTR_CHUNK or COMPILE_CHUNK */ byte* info, /* Where to store info. */ - Eterm* ptr_word, /* Where to store pointer into info. */ - Eterm* size_word) /* Where to store size of info. */ + BeamInstr* ptr_word, /* Where to store pointer into info. */ + BeamInstr* size_word) /* Where to store size of info. */ { Sint decoded_size; Uint size = stp->chunks[chunk].size; if (size != 0) { memcpy(info, stp->chunks[chunk].start, size); - *ptr_word = (Eterm) info; + *ptr_word = (BeamInstr) info; decoded_size = erts_decode_ext_size(info, size, 0); if (decoded_size < 0) { return 0; @@ -4791,7 +4904,7 @@ stub_read_export_table(LoaderState* stp) } static void -stub_final_touch(LoaderState* stp, Eterm* fp) +stub_final_touch(LoaderState* stp, BeamInstr* fp) { int i; int n = stp->num_exps; @@ -4978,12 +5091,12 @@ Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) { LoaderState state; - Eterm Funcs; - Eterm Patchlist; + BeamInstr Funcs; + BeamInstr Patchlist; Eterm* tp; - Eterm* code = NULL; - Eterm* ptrs; - Eterm* fp; + BeamInstr* code = NULL; + BeamInstr* ptrs; + BeamInstr* fp; byte* info; Uint ci; int n; @@ -5072,7 +5185,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Allocate memory for the stub module. */ - code_size = ((WORDS_PER_FUNCTION+1)*n + MI_FUNCTIONS + 2) * sizeof(Eterm); + code_size = ((WORDS_PER_FUNCTION+1)*n + MI_FUNCTIONS + 2) * sizeof(BeamInstr); code_size += state.chunks[ATTR_CHUNK].size; code_size += state.chunks[COMPILE_CHUNK].size; code = erts_alloc_fnf(ERTS_ALC_T_CODE, code_size); @@ -5141,7 +5254,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Set the pointer and make the stub. Put a return instruction * as the body until we know what kind of trap we should put there. */ - ptrs[i] = (Eterm) fp; + ptrs[i] = (BeamInstr) fp; #ifdef HIPE op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */ #else @@ -5154,8 +5267,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Insert the last pointer and the int_code_end instruction. */ - ptrs[i] = (Eterm) fp; - *fp++ = (Eterm) BeamOp(op_int_code_end); + ptrs[i] = (BeamInstr) fp; + *fp++ = (BeamInstr) BeamOp(op_int_code_end); /* * Copy attributes and compilation information. @@ -5222,9 +5335,9 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) #undef WORDS_PER_FUNCTION -static int safe_mul(Uint a, Uint b, Uint* resp) +static int safe_mul(UWord a, UWord b, UWord* resp) { - Uint res = a * b; + Uint res = a * b; /* XXX:Pan - used in bit syntax, the multiplication has to be stored in Uint */ *resp = res; if (b == 0) { diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index c17844a553..26e3054c4b 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -44,13 +44,13 @@ extern void** beam_ops; #endif -extern Eterm beam_debug_apply[]; -extern Eterm* em_call_error_handler; -extern Eterm* em_apply_bif; -extern Eterm* em_call_traced_function; +extern BeamInstr beam_debug_apply[]; +extern BeamInstr* em_call_error_handler; +extern BeamInstr* em_apply_bif; +extern BeamInstr* em_call_traced_function; typedef struct { - Eterm* start; /* Pointer to start of module. */ - Eterm* end; /* Points one word beyond last function in module. */ + BeamInstr* start; /* Pointer to start of module. */ + BeamInstr* end; /* Points one word beyond last function in module. */ } Range; /* diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 1b670585a7..6e9755ad48 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -616,13 +616,15 @@ local_name_monitor(Process *p, Eterm target_name) rp = erts_whereis_process(p, p_locks, target_name, ERTS_PROC_LOCK_LINK, ERTS_P2P_FLG_ALLOW_OTHER_X); if (!rp) { - Eterm lhp[3]; + DeclareTmpHeap(lhp,3,p); Eterm item; + UseTmpHeap(3,p); erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); p_locks &= ~ERTS_PROC_LOCK_LINK; item = TUPLE2(lhp, target_name, erts_this_dist_entry->sysname); erts_queue_monitor_message(p, &p_locks, mon_ref, am_process, item, am_noproc); + UnUseTmpHeap(3,p); } else if (rp != p) { erts_add_monitor(&(p->monitors), MON_ORIGIN, mon_ref, rp->id, @@ -1130,6 +1132,34 @@ BIF_RETTYPE error_2(Process* p, Eterm value, Eterm args) } /**********************************************************************/ +/* + * This is like exactly like error/1. The only difference is + * that Dialyzer thinks that it it will return an arbitrary term. + * It is useful in stub functions for NIFs. + */ + +BIF_RETTYPE nif_error_1(Process* p, Eterm term) +{ + p->fvalue = term; + BIF_ERROR(p, EXC_ERROR); +} + +/**********************************************************************/ +/* + * This is like exactly like error/2. The only difference is + * that Dialyzer thinks that it it will return an arbitrary term. + * It is useful in stub functions for NIFs. + */ + +BIF_RETTYPE nif_error_2(Process* p, Eterm value, Eterm args) +{ + Eterm* hp = HAlloc(p, 3); + + p->fvalue = TUPLE2(hp, value, args); + BIF_ERROR(p, EXC_ERROR_2); +} + +/**********************************************************************/ /* this is like throw/1 except that we set freason to EXC_EXIT */ BIF_RETTYPE exit_1(BIF_ALIST_1) @@ -3466,9 +3496,16 @@ BIF_RETTYPE make_fun_3(BIF_ALIST_3) if (arity < 0) { goto error; } +#if HALFWORD_HEAP + hp = HAlloc(BIF_P, 3); + hp[0] = HEADER_EXPORT; + /* Yes, May be misaligned, but X86_64 will fix it... */ + *((Export **) (hp+1)) = erts_export_get_or_make_stub(BIF_ARG_1, BIF_ARG_2, (Uint) arity); +#else hp = HAlloc(BIF_P, 2); hp[0] = HEADER_EXPORT; hp[1] = (Eterm) erts_export_get_or_make_stub(BIF_ARG_1, BIF_ARG_2, (Uint) arity); +#endif BIF_RET(make_export(hp)); } @@ -3574,11 +3611,11 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1) etp = (ExternalThing *) HAlloc(BIF_P, EXTERNAL_THING_HEAD_SIZE + 1); etp->header = make_external_pid_header(1); - etp->next = MSO(BIF_P).externals; + etp->next = MSO(BIF_P).first; etp->node = enp; etp->data.ui[0] = make_pid_data(c, b); - MSO(BIF_P).externals = etp; + MSO(BIF_P).first = (struct erl_off_heap_header*) etp; erts_deref_dist_entry(dep); BIF_RET(make_external_pid(etp)); } @@ -3884,7 +3921,7 @@ BIF_RETTYPE hash_2(BIF_ALIST_2) if ((range = signed_val(BIF_ARG_2)) <= 0) { /* [1..MAX_SMALL] */ BIF_ERROR(BIF_P, BADARG); } -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP if (range > ((1L << 27) - 1)) BIF_ERROR(BIF_P, BADARG); #endif @@ -3956,7 +3993,7 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2) /* * Return either a small or a big. Use the heap for bigs if there is room. */ -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP BIF_RET(make_small(final_hash)); #else if (IS_USMALL(0, final_hash)) { @@ -4118,8 +4155,8 @@ void erts_init_bif(void) #else bif_return_trap_export.code[2] = 1; #endif - bif_return_trap_export.code[3] = (Eterm) em_apply_bif; - bif_return_trap_export.code[4] = (Eterm) &bif_return_trap; + bif_return_trap_export.code[3] = (BeamInstr) em_apply_bif; + bif_return_trap_export.code[4] = (BeamInstr) &bif_return_trap; flush_monitor_message_trap = erts_export_put(am_erlang, am_flush_monitor_message, @@ -4134,54 +4171,6 @@ void erts_init_bif(void) await_proc_exit_trap = erts_export_put(am_erlang,am_await_proc_exit,3); } -BIF_RETTYPE blocking_read_file_1(BIF_ALIST_1) -{ - Eterm bin; - Eterm* hp; - byte *buff; - int i, buff_size; - FILE *file; - struct stat file_info; - char *filename = NULL; - size_t size; - - i = list_length(BIF_ARG_1); - if (i < 0) { - BIF_ERROR(BIF_P, BADARG); - } - filename = erts_alloc(ERTS_ALC_T_TMP, i + 1); - if (intlist_to_buf(BIF_ARG_1, filename, i) != i) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); - filename[i] = '\0'; - - hp = HAlloc(BIF_P, 3); - - file = fopen(filename, "r"); - if(file == NULL){ - erts_free(ERTS_ALC_T_TMP, (void *) filename); - BIF_RET(TUPLE2(hp, am_error, am_nofile)); - } - - stat(filename, &file_info); - erts_free(ERTS_ALC_T_TMP, (void *) filename); - - buff_size = file_info.st_size; - buff = (byte *) erts_alloc_fnf(ERTS_ALC_T_TMP, buff_size); - if (!buff) { - fclose(file); - BIF_RET(TUPLE2(hp, am_error, am_allocator)); - } - size = fread(buff, 1, buff_size, file); - fclose(file); - if (size < 0) - size = 0; - else if (size > buff_size) - size = (size_t) buff_size; - bin = new_binary(BIF_P, buff, (int) size); - erts_free(ERTS_ALC_T_TMP, (void *) buff); - - BIF_RET(TUPLE2(hp, am_ok, bin)); -} #ifdef HARDDEBUG /* You'll need this line in bif.tab to be able to use this debug bif diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index 05e9b78c28..50f5f4fbd6 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -125,7 +125,7 @@ do { \ #define ERTS_BIF_PREP_TRAP0(Ret, Trap, Proc) \ do { \ (Proc)->arity = 0; \ - (Proc)->def_arg_reg[3] = (Eterm) (Trap->address); \ + *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address); \ (Proc)->freason = TRAP; \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -134,7 +134,8 @@ do { \ do { \ (Proc)->arity = 1; \ (Proc)->def_arg_reg[0] = (Eterm) (A0); \ - (Proc)->def_arg_reg[3] = (Eterm) ((Trap)->address); \ + *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address); \ + (Proc)->def_arg_reg[3] = (UWord) ((Trap)->address); \ (Proc)->freason = TRAP; \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -144,7 +145,8 @@ do { \ (Proc)->arity = 2; \ (Proc)->def_arg_reg[0] = (Eterm) (A0); \ (Proc)->def_arg_reg[1] = (Eterm) (A1); \ - (Proc)->def_arg_reg[3] = (Eterm) ((Trap)->address); \ + *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address); \ + (Proc)->def_arg_reg[3] = (UWord) ((Trap)->address); \ (Proc)->freason = TRAP; \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -155,14 +157,15 @@ do { \ (Proc)->def_arg_reg[0] = (Eterm) (A0); \ (Proc)->def_arg_reg[1] = (Eterm) (A1); \ (Proc)->def_arg_reg[2] = (Eterm) (A2); \ - (Proc)->def_arg_reg[3] = (Eterm) ((Trap)->address); \ + *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address); \ + (Proc)->def_arg_reg[3] = (UWord) ((Trap)->address); \ (Proc)->freason = TRAP; \ (Ret) = THE_NON_VALUE; \ } while (0) #define BIF_TRAP0(p, Trap_) do { \ (p)->arity = 0; \ - (p)->def_arg_reg[3] = (Eterm) ((Trap_)->address); \ + *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) ((Trap_)->address); \ (p)->freason = TRAP; \ return THE_NON_VALUE; \ } while(0) @@ -170,7 +173,7 @@ do { \ #define BIF_TRAP1(Trap_, p, A0) do { \ (p)->arity = 1; \ (p)->def_arg_reg[0] = (A0); \ - (p)->def_arg_reg[3] = (Eterm) ((Trap_)->address); \ + *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) ((Trap_)->address); \ (p)->freason = TRAP; \ return THE_NON_VALUE; \ } while(0) @@ -179,7 +182,7 @@ do { \ (p)->arity = 2; \ (p)->def_arg_reg[0] = (A0); \ (p)->def_arg_reg[1] = (A1); \ - (p)->def_arg_reg[3] = (Eterm) ((Trap_)->address); \ + *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) ((Trap_)->address); \ (p)->freason = TRAP; \ return THE_NON_VALUE; \ } while(0) @@ -189,14 +192,14 @@ do { \ (p)->def_arg_reg[0] = (A0); \ (p)->def_arg_reg[1] = (A1); \ (p)->def_arg_reg[2] = (A2); \ - (p)->def_arg_reg[3] = (Eterm) ((Trap_)->address); \ + *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) ((Trap_)->address); \ (p)->freason = TRAP; \ return THE_NON_VALUE; \ } while(0) #define BIF_TRAP_CODE_PTR_0(p, Code_) do { \ (p)->arity = 0; \ - (p)->def_arg_reg[3] = (Eterm) (Code_); \ + *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) (Code_); \ (p)->freason = TRAP; \ return THE_NON_VALUE; \ } while(0) diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index b6fa06354a..0674aae77f 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -686,8 +686,6 @@ bif 'erl.system.code':make_stub_module/3 ebif_code_make_stub_module_3 bif code:is_module_native/1 bif 'erl.system.code':is_native/1 ebif_code_is_native_1 code_is_module_native_1 -bif erlang:blocking_read_file/1 - # # New Bifs in R9C. # @@ -760,6 +758,43 @@ bif erlang:finish_after_on_load/2 bif erlang:binary_to_term/2 # +# The binary match bifs (New in R14A - EEP9) +# + +# +# The searching/splitting/substituting thingies +# +ubif erlang:binary_part/2 +ubif erlang:binary_part/3 + +bif binary:compile_pattern/1 +bif binary:match/2 +bif binary:match/3 +bif binary:matches/2 +bif binary:matches/3 +bif binary:longest_common_prefix/1 +bif binary:longest_common_suffix/1 +bif binary:first/1 +bif binary:last/1 +bif binary:at/2 +bif binary:part/2 binary_binary_part_2 +bif binary:part/3 binary_binary_part_3 +bif binary:bin_to_list/1 +bif binary:bin_to_list/2 +bif binary:bin_to_list/3 +bif binary:list_to_bin/1 +bif binary:copy/1 +bif binary:copy/2 +bif binary:referenced_byte_size/1 +bif binary:encode_unsigned/1 +bif binary:encode_unsigned/2 +bif binary:decode_unsigned/1 +bif binary:decode_unsigned/2 + +bif erlang:nif_error/1 +bif erlang:nif_error/2 + +# # Obsolete # diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index 03c88da8c6..ff15d834ab 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -1459,7 +1459,31 @@ Eterm uint_to_big(Uint x, Eterm *y) BIG_DIGIT(y, 0) = x; return make_big(y); } +/* +** convert UWord to bigint +** (must only be used if x is to big to be stored as a small) +** Allocation is tricky, the heap need has to be calculated +** with the macro BIG_UWORD_HEAP_SIZE(x) +*/ +Eterm uword_to_big(UWord x, Eterm *y) +{ +#if HALFWORD_HEAP + Uint upper = x >> 32; + Uint lower = x & 0xFFFFFFFFUL; + if (upper == 0) { + *y = make_pos_bignum_header(1); + } else { + *y = make_pos_bignum_header(2); + BIG_DIGIT(y, 1) = upper; + } + BIG_DIGIT(y, 0) = lower; +#else + *y = make_pos_bignum_header(1); + BIG_DIGIT(y, 0) = x; +#endif + return make_big(y); +} /* ** convert signed int to bigint @@ -1480,19 +1504,19 @@ Eterm small_to_big(Sint x, Eterm *y) Eterm erts_uint64_to_big(Uint64 x, Eterm **hpp) { Eterm *hp = *hpp; -#ifdef ARCH_32 +#if defined(ARCH_32) || HALFWORD_HEAP if (x >= (((Uint64) 1) << 32)) { *hp = make_pos_bignum_header(2); BIG_DIGIT(hp, 0) = (Uint) (x & ((Uint) 0xffffffff)); BIG_DIGIT(hp, 1) = (Uint) ((x >> 32) & ((Uint) 0xffffffff)); - *hpp += 2; + *hpp += 3; } else #endif { *hp = make_pos_bignum_header(1); BIG_DIGIT(hp, 0) = (Uint) x; - *hpp += 1; + *hpp += 2; } return make_big(hp); } @@ -1507,7 +1531,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp) neg = 1; x = -x; } -#ifdef ARCH_32 +#if defined(ARCH_32) || HALFWORD_HEAP if (x >= (((Uint64) 1) << 32)) { if (neg) *hp = make_neg_bignum_header(2); @@ -1515,7 +1539,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp) *hp = make_pos_bignum_header(2); BIG_DIGIT(hp, 0) = (Uint) (x & ((Uint) 0xffffffff)); BIG_DIGIT(hp, 1) = (Uint) ((x >> 32) & ((Uint) 0xffffffff)); - *hpp += 2; + *hpp += 3; } else #endif @@ -1525,7 +1549,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp) else *hp = make_pos_bignum_header(1); BIG_DIGIT(hp, 0) = (Uint) x; - *hpp += 1; + *hpp += 2; } return make_big(hp); } @@ -1854,6 +1878,87 @@ term_to_Uint(Eterm term, Uint *up) } } +int +term_to_UWord(Eterm term, UWord *up) +{ +#if SIZEOF_VOID_P == ERTS_SIZEOF_ETERM + return term_to_Uint(term,up); +#else + if (is_small(term)) { + Sint i = signed_val(term); + if (i < 0) { + *up = BADARG; + return 0; + } + *up = (UWord) i; + return 1; + } else if (is_big(term)) { + ErtsDigit* xr = big_v(term); + dsize_t xl = big_size(term); + UWord uval = 0; + int n = 0; + + if (big_sign(term)) { + *up = BADARG; + return 0; + } else if (xl*D_EXP > sizeof(UWord)*8) { + *up = SYSTEM_LIMIT; + return 0; + } + while (xl-- > 0) { + uval |= ((UWord)(*xr++)) << n; + n += D_EXP; + } + *up = uval; + return 1; + } else { + *up = BADARG; + return 0; + } +#endif +} + +int +term_to_Uint64(Eterm term, Uint64 *up) +{ +#if SIZEOF_VOID_P == 8 + return term_to_UWord(term,up); +#else + if (is_small(term)) { + Sint i = signed_val(term); + if (i < 0) { + *up = BADARG; + return 0; + } + *up = (Uint64) i; + return 1; + } else if (is_big(term)) { + ErtsDigit* xr = big_v(term); + dsize_t xl = big_size(term); + Uint64 uval = 0; + int n = 0; + + if (big_sign(term)) { + *up = BADARG; + return 0; + } else if (xl*D_EXP > sizeof(Uint64)*8) { + *up = SYSTEM_LIMIT; + return 0; + } + while (xl-- > 0) { + uval |= ((Uint64)(*xr++)) << n; + n += D_EXP; + } + *up = uval; + return 1; + } else { + *up = BADARG; + return 0; + } +#endif +} + + int term_to_Sint(Eterm term, Sint *sp) { if (is_small(term)) { @@ -1888,6 +1993,47 @@ int term_to_Sint(Eterm term, Sint *sp) } } +#if HAVE_INT64 +int term_to_Sint64(Eterm term, Sint64 *sp) +{ +#if ERTS_SIZEOF_ETERM == 8 + return term_to_Sint(term, sp); +#else + if (is_small(term)) { + *sp = signed_val(term); + return 1; + } else if (is_big(term)) { + ErtsDigit* xr = big_v(term); + dsize_t xl = big_size(term); + int sign = big_sign(term); + Uint64 uval = 0; + int n = 0; + + if (xl*D_EXP > sizeof(Uint64)*8) { + return 0; + } + while (xl-- > 0) { + uval |= ((Uint64)(*xr++)) << n; + n += D_EXP; + } + if (sign) { + uval = -uval; + if ((Sint64)uval > 0) + return 0; + } else { + if ((Sint64)uval < 0) + return 0; + } + *sp = uval; + return 1; + } else { + return 0; + } +#endif +} +#endif /* HAVE_INT64 */ + + /* ** Add and subtract */ diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index b8e38d482c..25466cd3c2 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -34,7 +34,7 @@ typedef Uint ErtsDigit; -#if (SIZEOF_VOID_P == 4) && defined(SIZEOF_LONG_LONG) && (SIZEOF_LONG_LONG == 8) +#if ((SIZEOF_VOID_P == 4) || HALFWORD_HEAP) && defined(SIZEOF_LONG_LONG) && (SIZEOF_LONG_LONG == 8) /* Assume 32-bit machine with long long support */ typedef Uint64 ErtsDoubleDigit; typedef Uint16 ErtsHalfDigit; @@ -58,7 +58,7 @@ typedef Uint32 ErtsHalfDigit; typedef Uint dsize_t; /* Vector size type */ -#define D_EXP (SIZEOF_VOID_P*8) +#define D_EXP (ERTS_SIZEOF_ETERM*8) #define D_MASK ((ErtsDigit)(-1)) /* D_BASE-1 */ /* macros for bignum objects */ @@ -88,7 +88,13 @@ typedef Uint dsize_t; /* Vector size type */ #define BIG_UINT_HEAP_SIZE (1 + 1) /* always, since sizeof(Uint) <= sizeof(Eterm) */ -#ifdef ARCH_32 +#if HALFWORD_HEAP +#define BIG_UWORD_HEAP_SIZE(UW) (((UW) >> (sizeof(Uint) * 8)) ? 3 : 2) +#else +#define BIG_UWORD_HEAP_SIZE(UW) BIG_UINT_HEAP_SIZE +#endif + +#if defined(ARCH_32) || HALFWORD_HEAP #define ERTS_UINT64_BIG_HEAP_SIZE__(X) \ ((X) >= (((Uint64) 1) << 32) ? (1 + 2) : (1 + 1)) @@ -136,6 +142,7 @@ int big_ucomp (Eterm, Eterm); int big_to_double(Eterm x, double* resp); Eterm small_to_big(Sint, Eterm*); Eterm uint_to_big(Uint, Eterm*); +Eterm uword_to_big(UWord, Eterm*); Eterm erts_make_integer(Uint, Process *); dsize_t big_bytes(Eterm); @@ -143,7 +150,12 @@ Eterm bytes_to_big(byte*, dsize_t, int, Eterm*); byte* big_to_bytes(Eterm, byte*); int term_to_Uint(Eterm, Uint*); +int term_to_UWord(Eterm, UWord*); int term_to_Sint(Eterm, Sint*); +#if HAVE_INT64 +int term_to_Uint64(Eterm, Uint64*); +int term_to_Sint64(Eterm, Sint64*); +#endif Uint32 big_to_uint32(Eterm b); int term_equals_2pow32(Eterm); diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index 08c64610a2..8ee8fbcb29 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -42,7 +42,7 @@ void erts_init_binary(void) { /* Verify Binary alignment... */ - if ((((Uint) &((Binary *) 0)->orig_bytes[0]) % ((Uint) 8)) != 0) { + if ((((UWord) &((Binary *) 0)->orig_bytes[0]) % ((UWord) 8)) != 0) { /* I assume that any compiler should be able to optimize this away. If not, this test is not very expensive... */ erl_exit(ERTS_ABORT_EXIT, @@ -88,8 +88,8 @@ new_binary(Process *p, byte *buf, int len) pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); pb->thing_word = HEADER_PROC_BIN; pb->size = len; - pb->next = MSO(p).mso; - MSO(p).mso = pb; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*)pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; @@ -97,7 +97,7 @@ new_binary(Process *p, byte *buf, int len) /* * Miscellanous updates. Return the tagged binary. */ - MSO(p).overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); return make_binary(pb); } @@ -127,8 +127,8 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, int len) pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); pb->thing_word = HEADER_PROC_BIN; pb->size = len; - pb->next = MSO(p).mso; - MSO(p).mso = pb; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*)pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; @@ -136,7 +136,7 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, int len) /* * Miscellanous updates. Return the tagged binary. */ - MSO(p).overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); return make_binary(pb); } @@ -180,7 +180,7 @@ erts_realloc_binary(Eterm bin, size_t size) } byte* -erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, unsigned extra) +erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, ErtsAlcType_t allocator, unsigned extra) { byte* bytes; Eterm* real_bin; @@ -208,7 +208,7 @@ erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, unsigned extra) bytes = (byte *)(&(((ErlHeapBin *) real_bin)->data)) + offs; } if (bit_offs) { - byte* buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, byte_size + extra); + byte* buf = (byte *) erts_alloc(allocator, byte_size + extra); *base_ptr = buf; buf += extra; erts_copy_bits(bytes, bit_offs, 1, buf, 0, 1, byte_size*8); @@ -346,29 +346,40 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1) /* Turn a possibly deep list of ints (and binaries) into */ /* One large binary object */ -BIF_RETTYPE list_to_binary_1(BIF_ALIST_1) +/* + * This bif also exists in the binary module, under the name + * binary:list_to_bin/1, why it's divided into interface and + * implementation. Also the backend for iolist_to_binary_1. + */ + +BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg) { Eterm bin; int i; int offset; byte* bytes; - if (is_nil(BIF_ARG_1)) { - BIF_RET(new_binary(BIF_P,(byte*)"",0)); + if (is_nil(arg)) { + BIF_RET(new_binary(p,(byte*)"",0)); } - if (is_not_list(BIF_ARG_1)) { + if (is_not_list(arg)) { goto error; } - if ((i = io_list_len(BIF_ARG_1)) < 0) { + if ((i = io_list_len(arg)) < 0) { goto error; } - bin = new_binary(BIF_P, (byte *)NULL, i); + bin = new_binary(p, (byte *)NULL, i); bytes = binary_bytes(bin); - offset = io_list_to_buf(BIF_ARG_1, (char*) bytes, i); + offset = io_list_to_buf(arg, (char*) bytes, i); ASSERT(offset == 0); BIF_RET(bin); - error: - BIF_ERROR(BIF_P, BADARG); + error: + BIF_ERROR(p, BADARG); +} + +BIF_RETTYPE list_to_binary_1(BIF_ALIST_1) +{ + return erts_list_to_binary_bif(BIF_P, BIF_ARG_1); } /* Turn a possibly deep list of ints (and binaries) into */ @@ -376,31 +387,10 @@ BIF_RETTYPE list_to_binary_1(BIF_ALIST_1) BIF_RETTYPE iolist_to_binary_1(BIF_ALIST_1) { - Eterm bin; - int i; - int offset; - byte* bytes; - if (is_binary(BIF_ARG_1)) { BIF_RET(BIF_ARG_1); } - if (is_nil(BIF_ARG_1)) { - BIF_RET(new_binary(BIF_P,(byte*)"",0)); - } - if (is_not_list(BIF_ARG_1)) { - goto error; - } - if ((i = io_list_len(BIF_ARG_1)) < 0) { - goto error; - } - bin = new_binary(BIF_P, (byte *)NULL, i); - bytes = binary_bytes(bin); - offset = io_list_to_buf(BIF_ARG_1, (char*) bytes, i); - ASSERT(offset == 0); - BIF_RET(bin); - - error: - BIF_ERROR(BIF_P, BADARG); + return erts_list_to_binary_bif(BIF_P, BIF_ARG_1); } BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1) @@ -497,16 +487,6 @@ BIF_RETTYPE split_binary_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); } -void -erts_cleanup_mso(ProcBin* pb) -{ - while (pb != NULL) { - ProcBin* next = pb->next; - if (erts_refc_dectest(&pb->val->refc, 0) == 0) - erts_bin_free(pb->val); - pb = next; - } -} /* * Local functions. @@ -675,3 +655,4 @@ bitstr_list_len(Eterm obj) DESTROY_ESTACK(s); return (Sint) -1; } + diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index cc69977b79..f339e19761 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ /* This File contains functions which are called if a user hits ^C */ @@ -38,10 +38,6 @@ #include "erl_instrument.h" #include "erl_bif_timer.h" -#ifdef _OSE_ -#include "time.h" -#endif - /* Forward declarations -- should really appear somewhere else */ static void process_killer(void); void do_break(void); @@ -262,12 +258,10 @@ print_process_info(int to, void *to_arg, Process *p) } { - long s = 0; int frags = 0; ErlHeapFragment *m = p->mbuf; while (m != NULL) { frags++; - s += m->size; m = m->next; } erts_print(to, to_arg, "Number of heap fragments: %d\n", frags); @@ -327,7 +321,7 @@ print_process_info(int to, void *to_arg, Process *p) (unsigned)(OLD_HEND(p) - OLD_HEAP(p)) ); erts_print(to, to_arg, "Heap unused: %bpu\n", (p->hend - p->htop)); erts_print(to, to_arg, "OldHeap unused: %bpu\n", - (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HEAP(p)) ); + (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HTOP(p)) ); if (garbing) { print_garb_info(to, to_arg, p); @@ -381,7 +375,7 @@ loaded(int to, void *to_arg) int i; int old = 0; int cur = 0; - Eterm* code; + BeamInstr* code; /* * Calculate and print totals. @@ -617,29 +611,29 @@ static void bin_check(void) { Process *rp; - ProcBin *bp; - int i, printed; + struct erl_off_heap_header* hdr; + int i, printed = 0; for (i=0; i < erts_max_processes; i++) { if ((rp = process_tab[i]) == NULL) continue; - if (!(bp = rp->off_heap.mso)) - continue; - printed = 0; - while (bp) { - if (printed == 0) { - erts_printf("Process %T holding binary data \n", rp->id); - printed = 1; + for (hdr = rp->off_heap.first; hdr; hdr = hdr->next) { + if (hdr->thing_word == HEADER_PROC_BIN) { + ProcBin *bp = (ProcBin*) hdr; + if (!printed) { + erts_printf("Process %T holding binary data \n", rp->id); + printed = 1; + } + erts_printf("0x%08lx orig_size: %ld, norefs = %ld\n", + (unsigned long)bp->val, + (long)bp->val->orig_size, + erts_smp_atomic_read(&bp->val->refc)); } - erts_printf("0x%08lx orig_size: %ld, norefs = %ld\n", - (unsigned long)bp->val, - (long)bp->val->orig_size, - erts_smp_atomic_read(&bp->val->refc)); - - bp = bp->next; } - if (printed == 1) + if (printed) { erts_printf("--------------------------------------\n"); + printed = 0; + } } /* db_bin_check() has to be rewritten for the AVL trees... */ /*db_bin_check();*/ diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 0a5050b1fe..8bee47232e 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -37,6 +37,8 @@ MA_STACK_DECLARE(dst); MA_STACK_DECLARE(offset); #endif +static void move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap*); + void init_copy(void) { @@ -86,7 +88,7 @@ size_object(Eterm obj) obj = *ptr++; if (!IS_CONST(obj)) { ESTACK_PUSH(s, obj); - } + } obj = *ptr; break; case TAG_PRIMARY_BOXED: @@ -99,7 +101,7 @@ size_object(Eterm obj) arity = header_arity(hdr); sum += arity + 1; if (arity == 0) { /* Empty tuple -- unusual. */ - goto size_common; + goto pop_next; } while (arity-- > 1) { obj = *++ptr; @@ -115,7 +117,6 @@ size_object(Eterm obj) ErlFunThing* funp = (ErlFunThing *) bptr; unsigned eterms = 1 /* creator */ + funp->num_free; unsigned sz = thing_arityval(hdr); - sum += 1 /* header */ + sz + eterms; bptr += 1 /* header */ + sz; while (eterms-- > 1) { @@ -151,7 +152,7 @@ size_object(Eterm obj) } else { sum += heap_bin_size(binary_size(obj)+extra_bytes); } - goto size_common; + goto pop_next; } break; case BIN_MATCHSTATE_SUBTAG: @@ -159,18 +160,12 @@ size_object(Eterm obj) "size_object: matchstate term not allowed"); default: sum += thing_arityval(hdr) + 1; - /* Fall through */ - size_common: - if (ESTACK_ISEMPTY(s)) { - DESTROY_ESTACK(s); - return sum; - } - obj = ESTACK_POP(s); - break; + goto pop_next; } } break; case TAG_PRIMARY_IMMED1: + pop_next: if (ESTACK_ISEMPTY(s)) { DESTROY_ESTACK(s); return sum; @@ -319,10 +314,10 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) *argp = make_binary(hbot); pb = (ProcBin*) hbot; erts_refc_inc(&pb->val->refc, 2); - pb->next = off_heap->mso; + pb->next = off_heap->first; pb->flags = 0; - off_heap->mso = pb; - off_heap->overhead += pb->size / sizeof(Eterm); + off_heap->first = (struct erl_off_heap_header*) pb; + OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); } break; case SUB_BINARY_SUBTAG: @@ -368,10 +363,10 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) to->val = from->val; erts_refc_inc(&to->val->refc, 2); to->bytes = from->bytes + offset; - to->next = off_heap->mso; + to->next = off_heap->first; to->flags = 0; - off_heap->mso = to; - off_heap->overhead += to->size / sizeof(Eterm); + off_heap->first = (struct erl_off_heap_header*) to; + OH_OVERHEAD(off_heap, to->size / sizeof(Eterm)); } *argp = make_binary(hbot); if (extra_bytes != 0) { @@ -401,8 +396,8 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } #ifndef HYBRID /* FIND ME! */ funp = (ErlFunThing *) tp; - funp->next = off_heap->funs; - off_heap->funs = funp; + funp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*) funp; erts_refc_inc(&funp->fe->refc, 2); #endif *argp = make_fun(tp); @@ -421,8 +416,8 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) *htop++ = *objp++; } - etp->next = off_heap->externals; - off_heap->externals = etp; + etp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)etp; erts_refc_inc(&etp->node->refc, 2); *argp = make_external(tp); @@ -655,9 +650,9 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs) *hp++ = *objp++; } erts_refc_inc(&pb->val->refc, 2); - pb->next = erts_global_offheap.mso; - erts_global_offheap.mso = pb; - erts_global_offheap.overhead += pb->size / sizeof(Eterm); + pb->next = erts_global_offheap.first; + erts_global_offheap.first = pb; + OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); continue; } @@ -677,9 +672,9 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs) while (i--) { *hp++ = *objp++; } -#ifndef HYBRID // FIND ME! - funp->next = erts_global_offheap.funs; - erts_global_offheap.funs = funp; +#ifndef HYBRID /* FIND ME! */ + funp->next = erts_global_offheap.first; + erts_global_offheap.first = funp; erts_refc_inc(&funp->fe->refc, 2); #endif for (i = k; i < j; i++) { @@ -723,8 +718,8 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs) *hp++ = *objp++; } - etp->next = erts_global_offheap.externals; - erts_global_offheap.externals = etp; + etp->next = erts_global_offheap.first; + erts_global_offheap.first = etp; erts_refc_inc(&etp->node->refc, 2); continue; } @@ -780,9 +775,9 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs) to_bin->size = real_size; to_bin->val = from_bin->val; to_bin->bytes = from_bin->bytes + sub_offset; - to_bin->next = erts_global_offheap.mso; - erts_global_offheap.mso = to_bin; - erts_global_offheap.overhead += to_bin->size / sizeof(Eterm); + to_bin->next = erts_global_offheap.first; + erts_global_offheap.first = to_bin; + OH_OVERHEAD(&erts_global_offheap, to_bin->size / sizeof(Eterm)); res_binary=make_binary(to_bin); hp += PROC_BIN_SIZE; } @@ -915,57 +910,43 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) break; case REFC_BINARY_SUBTAG: { - ProcBin* pb = (ProcBin *) (hp-1); - int tari = thing_arityval(val); - - sz -= tari; - while (tari--) { - *hp++ = *tp++; - } + ProcBin* pb = (ProcBin *) (tp-1); erts_refc_inc(&pb->val->refc, 2); - pb->next = off_heap->mso; - off_heap->mso = pb; - off_heap->overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); } - break; + goto off_heap_common; + case FUN_SUBTAG: { -#ifndef HYBRID /* FIND ME! */ - ErlFunThing* funp = (ErlFunThing *) (hp-1); -#endif - int tari = thing_arityval(val); - - sz -= tari; - while (tari--) { - *hp++ = *tp++; - } -#ifndef HYBRID /* FIND ME! */ - funp->next = off_heap->funs; - off_heap->funs = funp; + ErlFunThing* funp = (ErlFunThing *) (tp-1); erts_refc_inc(&funp->fe->refc, 2); -#endif } - break; + goto off_heap_common; + case EXTERNAL_PID_SUBTAG: case EXTERNAL_PORT_SUBTAG: case EXTERNAL_REF_SUBTAG: { - ExternalThing* etp = (ExternalThing *) (hp-1); + ExternalThing* etp = (ExternalThing *) (tp-1); + erts_refc_inc(&etp->node->refc, 2); + } + off_heap_common: + { + struct erl_off_heap_header* ohh = (struct erl_off_heap_header*)(hp-1); int tari = thing_arityval(val); - + sz -= tari; while (tari--) { *hp++ = *tp++; } - etp->next = off_heap->externals; - off_heap->externals = etp; - erts_refc_inc(&etp->node->refc, 2); + ohh->next = off_heap->first; + off_heap->first = ohh; } break; default: { int tari = header_arity(val); - + sz -= tari; while (tari--) { *hp++ = *tp++; @@ -979,3 +960,92 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) *hpp = hp; return make_tuple(ptr + offs); } + +/* Move all terms in heap fragments into heap. The terms must be guaranteed to + * be contained within the fragments. The source terms are destructed with + * move markers. + * Typically used to copy a multi-fragmented message (from NIF). + */ +void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, + Eterm* refs, unsigned nrefs) +{ + ErlHeapFragment* bp; + Eterm* hp_start = *hpp; + Eterm* hp_end; + Eterm* hp; + unsigned i; + + for (bp=first; bp!=NULL; bp=bp->next) { + move_one_frag(hpp, bp->mem, bp->used_size, off_heap); + OH_OVERHEAD(off_heap, bp->off_heap.overhead); + } + hp_end = *hpp; + for (hp=hp_start; hp<hp_end; ++hp) { + Eterm* ptr; + Eterm val; + Eterm gval = *hp; + switch (primary_tag(gval)) { + case TAG_PRIMARY_BOXED: + ptr = boxed_val(gval); + val = *ptr; + if (IS_MOVED_BOXED(val)) { + ASSERT(is_boxed(val)); + *hp = val; + } + break; + case TAG_PRIMARY_LIST: + ptr = list_val(gval); + val = *ptr; + if (IS_MOVED_CONS(val)) { + *hp = ptr[1]; + } + break; + case TAG_PRIMARY_HEADER: + if (header_is_thing(gval)) { + hp += thing_arityval(gval); + } + break; + } + } + for (i=0; i<nrefs; ++i) { + refs[i] = follow_moved(refs[i]); + } +} + +static void +move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap) +{ + Eterm* ptr = src; + Eterm* end = ptr + src_sz; + Eterm dummy_ref; + Eterm* hp = *hpp; + + while (ptr != end) { + Eterm val; + ASSERT(ptr < end); + val = *ptr; + ASSERT(val != ERTS_HOLE_MARKER); + if (is_header(val)) { + struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp; + ASSERT(ptr + header_arity(val) < end); + MOVE_BOXED(ptr, val, hp, &dummy_ref); + switch (val & _HEADER_SUBTAG_MASK) { + case REFC_BINARY_SUBTAG: + case FUN_SUBTAG: + case EXTERNAL_PID_SUBTAG: + case EXTERNAL_PORT_SUBTAG: + case EXTERNAL_REF_SUBTAG: + hdr->next = off_heap->first; + off_heap->first = hdr; + break; + } + } + else { /* must be a cons cell */ + ASSERT(ptr+1 < end); + MOVE_CONS(ptr, val, hp, &dummy_ref); + ptr += 2; + } + } + *hpp = hp; +} + diff --git a/erts/emulator/beam/decl.h b/erts/emulator/beam/decl.h deleted file mode 100644 index da1be29d53..0000000000 --- a/erts/emulator/beam/decl.h +++ /dev/null @@ -1,55 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - -#ifndef __DECL_H__ -#define __DECL_H__ - -#if defined(__STDC__) || defined(_MSC_VER) -#define EXTERN_FUNCTION(t, f, x) extern t f x -#define FUNCTION(t, f, x) t f x -#define _DOTS_ ... -#define _VOID_ void -#elif defined(__cplusplus) -#define EXTERN_FUNCTION(f, x) extern "C" { f x } -#define FUNCTION(t, f, x) t f x -#define _DOTS_ ... -#define _VOID_ void -#else -#define EXTERN_FUNCTION(t, f, x) extern t f (/*x*/) -#define FUNCTION(t, f, x) t f (/*x*/) -#define _DOTS_ -#define _VOID_ -#endif - -/* -** Example of declarations -** -** EXTERN_FUNCTION(void, foo, (int, int, char)); -** FUNCTION(void, bar, (int, char)); -** -** struct funcs { -** FUNCTION(int*, (*f1), (int, int)); -** FUNCTION(void, (*f2), (int, char)); -** FUNCTION(void, (*f3), (_VOID_)); -** FUNCTION(int, (*f4), (char*, _DOTS_)); -** }; -** -*/ - -#endif diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index e3094404e2..16b6aeac3f 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -228,6 +228,7 @@ int is_node_name_atom(Eterm a) typedef struct { DistEntry *dep; + Eterm *lhp; } NetExitsContext; /* @@ -253,8 +254,9 @@ static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp) erts_destroy_monitor(rmon); } } else { - Eterm lhp[3]; + DeclareTmpHeapNoproc(lhp,3); Eterm watched; + UseTmpHeapNoproc(3); ASSERT(mon->type == MON_TARGET); rmon = erts_remove_monitor(&(rp->monitors),mon->ref); /* ASSERT(rmon != NULL); can happen during process exit */ @@ -271,6 +273,7 @@ static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp) watched, am_noconnection); erts_destroy_monitor(rmon); } + UnUseTmpHeapNoproc(3); } erts_smp_proc_unlock(rp, rp_locks); done: @@ -632,19 +635,27 @@ static void clear_dist_entry(DistEntry *dep) int erts_dsig_send_link(ErtsDSigData *dsdp, Eterm local, Eterm remote) { - Eterm ctl_heap[4]; + DeclareTmpHeapNoproc(ctl_heap,4); Eterm ctl = TUPLE3(&ctl_heap[0], make_small(DOP_LINK), local, remote); + int res; + UseTmpHeapNoproc(4); - return dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + UnUseTmpHeapNoproc(4); + return res; } int erts_dsig_send_unlink(ErtsDSigData *dsdp, Eterm local, Eterm remote) { - Eterm ctl_heap[4]; + DeclareTmpHeapNoproc(ctl_heap,4); Eterm ctl = TUPLE3(&ctl_heap[0], make_small(DOP_UNLINK), local, remote); + int res; - return dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + UseTmpHeapNoproc(4); + res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + UnUseTmpHeapNoproc(4); + return res; } @@ -656,7 +667,10 @@ erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, Eterm ref, Eterm reason) { Eterm ctl; - Eterm ctl_heap[6]; + DeclareTmpHeapNoproc(ctl_heap,6); + int res; + + UseTmpHeapNoproc(6); ctl = TUPLE5(&ctl_heap[0], make_small(DOP_MONITOR_P_EXIT), watched, watcher, ref, reason); @@ -667,7 +681,9 @@ erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, erts_smp_de_links_unlock(dsdp->dep); #endif - return dsig_send(dsdp, ctl, THE_NON_VALUE, 1); + res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1); + UnUseTmpHeapNoproc(6); + return res; } /* We want to monitor a process (named or unnamed) on another node, we send: @@ -678,13 +694,17 @@ erts_dsig_send_monitor(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, Eterm ref) { Eterm ctl; - Eterm ctl_heap[5]; + DeclareTmpHeapNoproc(ctl_heap,5); + int res; + UseTmpHeapNoproc(5); ctl = TUPLE4(&ctl_heap[0], make_small(DOP_MONITOR_P), watcher, watched, ref); - return dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + UnUseTmpHeapNoproc(5); + return res; } /* A local process monitoring a remote one wants to stop monitoring, either @@ -696,23 +716,29 @@ erts_dsig_send_demonitor(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, Eterm ref, int force) { Eterm ctl; - Eterm ctl_heap[5]; + DeclareTmpHeapNoproc(ctl_heap,5); + int res; + UseTmpHeapNoproc(5); ctl = TUPLE4(&ctl_heap[0], make_small(DOP_DEMONITOR_P), watcher, watched, ref); - return dsig_send(dsdp, ctl, THE_NON_VALUE, force); + res = dsig_send(dsdp, ctl, THE_NON_VALUE, force); + UnUseTmpHeapNoproc(5); + return res; } int erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message) { Eterm ctl; - Eterm ctl_heap[5]; + DeclareTmpHeapNoproc(ctl_heap,5); Eterm token = NIL; Process *sender = dsdp->proc; + int res; + UseTmpHeapNoproc(5); if (SEQ_TRACE_TOKEN(sender) != NIL) { seq_trace_update_send(sender); token = SEQ_TRACE_TOKEN(sender); @@ -724,17 +750,21 @@ erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message) make_small(DOP_SEND_TT), am_Cookie, remote, token); else ctl = TUPLE3(&ctl_heap[0], make_small(DOP_SEND), am_Cookie, remote); - return dsig_send(dsdp, ctl, message, 0); + res = dsig_send(dsdp, ctl, message, 0); + UnUseTmpHeapNoproc(5); + return res; } int erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message) { Eterm ctl; - Eterm ctl_heap[6]; + DeclareTmpHeapNoproc(ctl_heap,6); Eterm token = NIL; Process *sender = dsdp->proc; + int res; + UseTmpHeapNoproc(6); if (SEQ_TRACE_TOKEN(sender) != NIL) { seq_trace_update_send(sender); token = SEQ_TRACE_TOKEN(sender); @@ -747,7 +777,9 @@ erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message) else ctl = TUPLE4(&ctl_heap[0], make_small(DOP_REG_SEND), sender->id, am_Cookie, remote_name); - return dsig_send(dsdp, ctl, message, 0); + res = dsig_send(dsdp, ctl, message, 0); + UnUseTmpHeapNoproc(6); + return res; } /* local has died, deliver the exit signal to remote */ @@ -756,8 +788,10 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, Eterm reason, Eterm token) { Eterm ctl; - Eterm ctl_heap[6]; + DeclareTmpHeapNoproc(ctl_heap,6); + int res; + UseTmpHeapNoproc(6); if (token != NIL) { seq_trace_update_send(dsdp->proc); seq_trace_output_exit(token, reason, SEQ_TRACE_SEND, remote, local); @@ -767,38 +801,58 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, ctl = TUPLE4(&ctl_heap[0], make_small(DOP_EXIT), local, remote, reason); } /* forced, i.e ignore busy */ - return dsig_send(dsdp, ctl, THE_NON_VALUE, 1); + res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1); + UnUseTmpHeapNoproc(6); + return res; } int erts_dsig_send_exit(ErtsDSigData *dsdp, Eterm local, Eterm remote, Eterm reason) { - Eterm ctl_heap[5]; - Eterm ctl = TUPLE4(&ctl_heap[0], - make_small(DOP_EXIT), local, remote, reason); + DeclareTmpHeapNoproc(ctl_heap,5); + int res; + Eterm ctl; + + UseTmpHeapNoproc(5); + ctl = TUPLE4(&ctl_heap[0], + make_small(DOP_EXIT), local, remote, reason); /* forced, i.e ignore busy */ - return dsig_send(dsdp, ctl, THE_NON_VALUE, 1); + res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1); + UnUseTmpHeapNoproc(5); + return res; } int erts_dsig_send_exit2(ErtsDSigData *dsdp, Eterm local, Eterm remote, Eterm reason) { - Eterm ctl_heap[5]; - Eterm ctl = TUPLE4(&ctl_heap[0], - make_small(DOP_EXIT2), local, remote, reason); + DeclareTmpHeapNoproc(ctl_heap,5); + int res; + Eterm ctl; - return dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + UseTmpHeapNoproc(5); + ctl = TUPLE4(&ctl_heap[0], + make_small(DOP_EXIT2), local, remote, reason); + + res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + UnUseTmpHeapNoproc(5); + return res; } int erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote) { - Eterm ctl_heap[4]; - Eterm ctl = TUPLE3(&ctl_heap[0], - make_small(DOP_GROUP_LEADER), leader, remote); + DeclareTmpHeapNoproc(ctl_heap,4); + int res; + Eterm ctl; + + UseTmpHeapNoproc(4); + ctl = TUPLE3(&ctl_heap[0], + make_small(DOP_GROUP_LEADER), leader, remote); - return dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + UnUseTmpHeapNoproc(4); + return res; } #if defined(PURIFY) @@ -832,6 +886,7 @@ erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote) ** ** assert hlen == 0 !!! */ + int erts_net_message(Port *prt, DistEntry *dep, byte *hbuf, @@ -839,6 +894,7 @@ int erts_net_message(Port *prt, byte *buf, int len) { +#define DIST_CTL_DEFAULT_SIZE 64 ErtsDistExternal ede; byte *t; Sint ctl_len; @@ -850,7 +906,7 @@ int erts_net_message(Port *prt, Eterm *tuple; Eterm reason; Process* rp; - Eterm ctl_default[64]; + DeclareTmpHeapNoproc(ctl_default,DIST_CTL_DEFAULT_SIZE); Eterm* ctl = ctl_default; ErlOffHeap off_heap; Eterm* hp; @@ -864,24 +920,25 @@ int erts_net_message(Port *prt, int orig_len = len; #endif + UseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); /* Thanks to Luke Gorrie */ - off_heap.mso = NULL; -#ifndef HYBRID /* FIND ME! */ - off_heap.funs = NULL; -#endif + off_heap.first = NULL; off_heap.overhead = 0; - off_heap.externals = NULL; ERTS_SMP_CHK_NO_PROC_LOCKS; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - if (!erts_is_alive) + if (!erts_is_alive) { + UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); return 0; + } if (hlen > 0) goto data_error; - if (len == 0) /* HANDLE TICK !!! */ + if (len == 0) { /* HANDLE TICK !!! */ + UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); return 0; + } #ifdef ERTS_RAW_DIST_MSG_DBG erts_fprintf(stderr, "<< "); @@ -922,7 +979,8 @@ int erts_net_message(Port *prt, goto data_error; } orig_ctl_len = ctl_len; - if (ctl_len > sizeof(ctl_default)/sizeof(ctl_default[0])) { + + if (ctl_len > DIST_CTL_DEFAULT_SIZE) { ctl = erts_alloc(ERTS_ALC_T_DCTRL_BUF, ctl_len * sizeof(Eterm)); } hp = ctl; @@ -1202,7 +1260,7 @@ int erts_net_message(Port *prt, {DOP_MONITOR_P_EXIT, Remote pid or name, Local pid, ref, reason} */ - Eterm lhp[3]; + DeclareTmpHeapNoproc(lhp,3); Eterm sysname; ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_MSG_SEND|ERTS_PROC_LOCK_LINK; @@ -1237,6 +1295,7 @@ int erts_net_message(Port *prt, erts_smp_proc_unlock(rp, rp_locks); break; } + UseTmpHeapNoproc(3); watched = (is_not_nil(mon->name) ? TUPLE2(&lhp[0], mon->name, sysname) @@ -1246,6 +1305,7 @@ int erts_net_message(Port *prt, ref, am_process, watched, reason); erts_smp_proc_unlock(rp, rp_locks); erts_destroy_monitor(mon); + UnUseTmpHeapNoproc(3); break; } @@ -1370,38 +1430,24 @@ int erts_net_message(Port *prt, } } - if (off_heap.mso) { - erts_cleanup_mso(off_heap.mso); - } - if (off_heap.externals) { - erts_cleanup_externals(off_heap.externals); - } + erts_cleanup_offheap(&off_heap); #ifndef HYBRID /* FIND ME! */ - if (off_heap.funs) { - erts_cleanup_funs(off_heap.funs); - } if (ctl != ctl_default) { erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl); } #endif + UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); ERTS_SMP_CHK_NO_PROC_LOCKS; return 0; data_error: - if (off_heap.mso) { - erts_cleanup_mso(off_heap.mso); - } - if (off_heap.externals) { - erts_cleanup_externals(off_heap.externals); - } + erts_cleanup_offheap(&off_heap); #ifndef HYBRID /* FIND ME! */ - if (off_heap.funs) { - erts_cleanup_funs(off_heap.funs); - } if (ctl != ctl_default) { erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl); } #endif + UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); erts_do_exit_port(prt, dep->cid, am_killed); ERTS_SMP_CHK_NO_PROC_LOCKS; return -1; @@ -1554,9 +1600,9 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) */ data_size >>= (10-4); -#if defined(ARCH_64) +#if defined(ARCH_64) && !HALFWORD_HEAP data_size &= 0x003fffffffffffff; -#elif defined(ARCH_32) +#elif defined(ARCH_32) || HALFWORD_HEAP data_size &= 0x003fffff; #else # error "Ohh come on ... !?!" @@ -1640,9 +1686,9 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) } -#if defined(ARCH_64) +#if defined(ARCH_64) && !HALFWORD_HEAP #define ERTS_PORT_REDS_MASK__ 0x003fffffffffffffL -#elif defined(ARCH_32) +#elif defined(ARCH_32) || HALFWORD_HEAP #define ERTS_PORT_REDS_MASK__ 0x003fffff #else # error "Ohh come on ... !?!" @@ -2547,12 +2593,15 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) int visible = 0; int hidden = 0; int this = 0; - Uint buf[2]; /* For one cons-cell */ + DeclareTmpHeap(buf,2,BIF_P); /* For one cons-cell */ DistEntry *dep; Eterm arg_list = BIF_ARG_1; #ifdef DEBUG Eterm* endp; #endif + + UseTmpHeap(2,BIF_P); + if (is_atom(BIF_ARG_1)) arg_list = CONS(buf, BIF_ARG_1, NIL); @@ -2563,13 +2612,14 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) case am_known: visible = hidden = not_connected = this = 1; break; case am_this: this = 1; break; case am_connected: visible = hidden = 1; break; - default: BIF_ERROR(BIF_P, BADARG); break; + default: goto error; break; } arg_list = CDR(list_val(arg_list)); } - if (is_not_nil(arg_list)) - BIF_ERROR(BIF_P, BADARG); + if (is_not_nil(arg_list)) { + goto error; + } length = 0; @@ -2591,7 +2641,7 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) if (length == 0) { erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); - BIF_RET(result); + goto done; } hp = HAlloc(BIF_P, 2*length); @@ -2620,7 +2670,14 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) } ASSERT(endp == hp); erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + +done: + UnUseTmpHeap(2,BIF_P); BIF_RET(result); + +error: + UnUseTmpHeap(2,BIF_P); + BIF_ERROR(BIF_P,BADARG); } /**********************************************************************/ diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index ea1abcaeed..fa19c7fb45 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -287,4 +287,5 @@ extern void erts_kill_dist_connection(DistEntry *dep, Uint32); extern Uint erts_dist_cache_size(void); + #endif diff --git a/erts/emulator/beam/elib_malloc.c b/erts/emulator/beam/elib_malloc.c deleted file mode 100644 index b18c48d8d6..0000000000 --- a/erts/emulator/beam/elib_malloc.c +++ /dev/null @@ -1,2334 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - -/* -** Description: Faster malloc(). -*/ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "sys.h" - -#ifdef ENABLE_ELIB_MALLOC - -#undef THREAD_SAFE_ELIB_MALLOC -#ifdef USE_THREADS -#define THREAD_SAFE_ELIB_MALLOC 1 -#else -#define THREAD_SAFE_ELIB_MALLOC 0 -#endif - -#include "erl_driver.h" -#include "erl_threads.h" -#include "elib_stat.h" -#include <stdio.h> -#include <stdlib.h> - -/* To avoid clobbering of names becaure of reclaim on VxWorks, - we undefine all possible malloc, calloc etc. */ -#undef malloc -#undef calloc -#undef free -#undef realloc - -#define ELIB_INLINE /* inline all possible functions */ - -#ifndef ELIB_ALIGN -#define ELIB_ALIGN sizeof(double) -#endif - -#ifndef ELIB_HEAP_SIZE -#define ELIB_HEAP_SIZE (64*1024) /* Default 64K */ -#endif - -#ifndef ELIB_HEAP_INCREAMENT -#define ELIB_HEAP_INCREAMENT (32*1024) /* Default 32K */ -#endif - -#ifndef ELIB_FAILURE -#define ELIB_FAILURE abort() -#endif - -#undef ASSERT -#ifdef DEBUG -#define ASSERT(B) \ - ((void) ((B) ? 1 : (fprintf(stderr, "%s:%d: Assertion failed: %s\n", \ - __FILE__, __LINE__, #B), abort(), 0))) -#else -#define ASSERT(B) ((void) 1) -#endif - -#ifndef USE_RECURSIVE_MALLOC_MUTEX -#define USE_RECURSIVE_MALLOC_MUTEX 0 -#endif - -#if USE_RECURSIVE_MALLOC_MUTEX -static erts_mtx_t malloc_mutex = ERTS_REC_MTX_INITER; -#else /* #if USE_RECURSIVE_MALLOC_MUTEX */ -static erts_mtx_t malloc_mutex = ERTS_MTX_INITER; -#if THREAD_SAFE_ELIB_MALLOC -static erts_cnd_t malloc_cond = ERTS_CND_INITER; -#endif -#endif /* #if USE_RECURSIVE_MALLOC_MUTEX */ - -typedef unsigned long EWord; /* Assume 32-bit in this implementation */ -typedef unsigned short EHalfWord; /* Assume 16-bit in this implementation */ -typedef unsigned char EByte; /* Assume 8-bit byte */ - - -#define elib_printf fprintf -#define elib_putc fputc - - -#if defined(__STDC__) || defined(__WIN32__) -#define CONCAT(x,y) x##y -#else -#define CONCAT(x,y) x/**/y -#endif - - -#ifdef ELIB_DEBUG -#define ELIB_PREFIX(fun, args) CONCAT(elib__,fun) args -#else -#define ELIB_PREFIX(fun, args) CONCAT(elib_,fun) args -#endif - -#if defined(__STDC__) -void *ELIB_PREFIX(malloc, (size_t)); -void *ELIB_PREFIX(calloc, (size_t, size_t)); -void ELIB_PREFIX(cfree, (EWord *)); -void ELIB_PREFIX(free, (EWord *)); -void *ELIB_PREFIX(realloc, (EWord *, size_t)); -void* ELIB_PREFIX(memresize, (EWord *, int)); -void* ELIB_PREFIX(memalign, (int, int)); -void* ELIB_PREFIX(valloc, (int)); -void* ELIB_PREFIX(pvalloc, (int)); -int ELIB_PREFIX(memsize, (EWord *)); -/* Extern interfaces used by VxWorks */ -size_t elib_sizeof(void *); -void elib_init(EWord *, EWord); -void elib_force_init(EWord *, EWord); -#endif - -#if defined(__STDC__) -/* define prototypes for missing */ -void* memalign(size_t a, size_t s); -void* pvalloc(size_t nb); -void* memresize(void *p, int nb); -int memsize(void *p); -#endif - -/* bytes to pages */ -#define PAGES(x) (((x)+page_size-1) / page_size) -#define PAGE_ALIGN(p) ((char*)((((EWord)(p))+page_size-1)&~(page_size-1))) - -/* bytes to words */ -#define WORDS(x) (((x)+sizeof(EWord)-1) / sizeof(EWord)) - -/* Align an address */ -#define ALIGN(p) ((EWord*)((((EWord)(p)+ELIB_ALIGN-1)&~(ELIB_ALIGN-1)))) - -/* Calculate the size needed to keep alignment */ - -#define ALIGN_BSZ(nb) ((nb+sizeof(EWord)+ELIB_ALIGN-1) & ~(ELIB_ALIGN-1)) - -#define ALIGN_WSZ(nb) WORDS(ALIGN_BSZ(nb)) - -#define ALIGN_SIZE(nb) (ALIGN_WSZ(nb) - 1) - - -/* PARAMETERS */ - -#if defined(ELIB_HEAP_SBRK) - -#undef PAGE_SIZE - -/* Get the system page size (NEED MORE DEFINES HERE) */ -#ifdef _SC_PAGESIZE -#define PAGE_SIZE sysconf(_SC_PAGESIZE) -#elif defined(_MSC_VER) -# ifdef _M_ALPHA -# define PAGE_SIZE 0x2000 -# else -# define PAGE_SIZE 0x1000 -# endif -#else -#define PAGE_SIZE getpagesize() -#endif - -#define ELIB_EXPAND(need) expand_sbrk(need) -static FUNCTION(int, expand_sbrk, (EWord)); - -#elif defined(ELIB_HEAP_FIXED) - -#define PAGE_SIZE 1024 -#define ELIB_EXPAND(need) -1 -static EWord fix_heap[WORDS(ELIB_HEAP_SIZE)]; - -#elif defined(ELIB_HEAP_USER) - -#define PAGE_SIZE 1024 -#define ELIB_EXPAND(need) -1 - -#else - -#error "ELIB HEAP TYPE NOT SET" - -#endif - - -#define STAT_ALLOCED_BLOCK(SZ) \ -do { \ - tot_allocated += (SZ); \ - if (max_allocated < tot_allocated) \ - max_allocated = tot_allocated; \ -} while (0) - -#define STAT_FREED_BLOCK(SZ) \ -do { \ - tot_allocated -= (SZ); \ -} while (0) - -static int max_allocated = 0; -static int tot_allocated = 0; -static EWord* eheap; /* Align heap start */ -static EWord* eheap_top; /* Point to end of heap */ -EWord page_size = 0; /* Set by elib_init */ - -#if defined(ELIB_DEBUG) || defined(DEBUG) -#define ALIGN_CHECK(a, p) \ - do { \ - if ((EWord)(p) & (a-1)) { \ - elib_printf(stderr, \ - "RUNTIME ERROR: bad alignment (0x%lx:%d:%d)\n", \ - (unsigned long) (p), (int) a, __LINE__); \ - ELIB_FAILURE; \ - } \ - } while(0) -#define ELIB_ALIGN_CHECK(p) ALIGN_CHECK(ELIB_ALIGN, p) -#else -#define ALIGN_CHECK(a, p) -#define ELIB_ALIGN_CHECK(p) -#endif - -#define DYNAMIC 32 - -/* -** Free block layout -** 1 1 30 -** +--------------------------+ -** |F|P| Size | -** +--------------------------+ -** -** Where F is the free bit -** P is the free above bit -** Size is messured in words and does not include the hdr word -** -** If block is on the free list the size is also stored last in the block. -** -*/ -typedef struct _free_block FreeBlock; -struct _free_block { - EWord hdr; - Uint flags; - FreeBlock* parent; - FreeBlock* left; - FreeBlock* right; - EWord v[1]; -}; - -typedef struct _allocated_block { - EWord hdr; - EWord v[5]; -} AllocatedBlock; - - -/* - * Interface to tree routines. - */ -typedef Uint Block_t; - -static Block_t* get_free_block(Uint); -static void link_free_block(Block_t *); -static void unlink_free_block(Block_t *del); - -#define FREE_BIT 0x80000000 -#define FREE_ABOVE_BIT 0x40000000 -#define SIZE_MASK 0x3fffffff /* 2^30 words = 2^32 bytes */ - -/* Work on both FreeBlock and AllocatedBlock */ -#define SIZEOF(p) ((p)->hdr & SIZE_MASK) -#define IS_FREE(p) (((p)->hdr & FREE_BIT) != 0) -#define IS_FREE_ABOVE(p) (((p)->hdr & FREE_ABOVE_BIT) != 0) - -/* Given that we have a free block above find its size */ -#define SIZEOF_ABOVE(p) *(((EWord*) (p)) - 1) - -#define MIN_BLOCK_SIZE (sizeof(FreeBlock)/sizeof(EWord)) -#define MIN_WORD_SIZE (MIN_BLOCK_SIZE-1) -#define MIN_BYTE_SIZE (sizeof(FreeBlock)-sizeof(EWord)) - -#define MIN_ALIGN_SIZE ALIGN_SIZE(MIN_BYTE_SIZE) - - -static AllocatedBlock* heap_head = 0; -static AllocatedBlock* heap_tail = 0; -static EWord eheap_size = 0; - -static int heap_locked; - -static int elib_need_init = 1; -#if THREAD_SAFE_ELIB_MALLOC -static int elib_is_initing = 0; -#endif - -typedef FreeBlock RBTree_t; - -static RBTree_t* root = NULL; - - -static FUNCTION(void, deallocate, (AllocatedBlock*, int)); - -/* - * Unlink a free block - */ - -#define mark_allocated(p, szp) do { \ - (p)->hdr = ((p)->hdr & FREE_ABOVE_BIT) | (szp); \ - (p)->v[szp] &= ~FREE_ABOVE_BIT; \ - } while(0) - -#define mark_free(p, szp) do { \ - (p)->hdr = FREE_BIT | (szp); \ - ((FreeBlock *)p)->v[szp-sizeof(FreeBlock)/sizeof(EWord)+1] = (szp); \ - } while(0) - -#if 0 -/* Help macros to log2 */ -#define LOG_1(x) (((x) > 1) ? 1 : 0) -#define LOG_2(x) (((x) > 3) ? 2+LOG_1((x) >> 2) : LOG_1(x)) -#define LOG_4(x) (((x) > 15) ? 4+LOG_2((x) >> 4) : LOG_2(x)) -#define LOG_8(x) (((x) > 255) ? 8+LOG_4((x)>>8) : LOG_4(x)) -#define LOG_16(x) (((x) > 65535) ? 16+LOG_8((x)>>16) : LOG_8(x)) - -#define log2(x) LOG_16(x) -#endif - -/* - * Split a block to be allocated. - * Mark block as ALLOCATED and clear - * FREE_ABOVE_BIT on next block - * - * nw is SIZE aligned and szp is SIZE aligned + 1 - */ -static void -split_block(FreeBlock* p, EWord nw, EWord szp) -{ - EWord szq; - FreeBlock* q; - - szq = szp - nw; - /* Preserve FREE_ABOVE bit in p->hdr !!! */ - - if (szq >= MIN_ALIGN_SIZE+1) { - szq--; - p->hdr = (p->hdr & FREE_ABOVE_BIT) | nw; - - q = (FreeBlock*) (((EWord*) p) + nw + 1); - mark_free(q, szq); - link_free_block((Block_t *) q); - - q = (FreeBlock*) (((EWord*) q) + szq + 1); - q->hdr |= FREE_ABOVE_BIT; - } - else { - mark_allocated((AllocatedBlock*)p, szp); - } -} - -/* - * Find a free block - */ -static FreeBlock* -alloc_block(EWord nw) -{ - for (;;) { - FreeBlock* p = (FreeBlock *) get_free_block(nw); - - if (p != NULL) { - return p; - } else if (ELIB_EXPAND(nw+MIN_WORD_SIZE)) { - return 0; - } - } -} - - -size_t elib_sizeof(void *p) -{ - AllocatedBlock* pp; - - if (p != 0) { - pp = (AllocatedBlock*) (((char *)p)-1); - return SIZEOF(pp); - } - return 0; -} - -static void locked_elib_init(EWord*, EWord); -static void init_elib_malloc(EWord*, EWord); - -/* -** Initialize the elib -** The addr and sz is only used when compiled with EXPAND_ADDR -*/ -/* Not static, this is used by VxWorks */ -void elib_init(EWord* addr, EWord sz) -{ - if (!elib_need_init) - return; - erts_mtx_lock(&malloc_mutex); - locked_elib_init(addr, sz); - erts_mtx_unlock(&malloc_mutex); -} - -static void locked_elib_init(EWord* addr, EWord sz) -{ - if (!elib_need_init) - return; - -#if THREAD_SAFE_ELIB_MALLOC - -#if !USE_RECURSIVE_MALLOC_MUTEX - { - static erts_tid_t initer_tid; - - if(elib_is_initing) { - - if(erts_equal_tids(initer_tid, erts_thr_self())) - return; - - /* Wait until initializing thread is done with initialization */ - - while(elib_need_init) - erts_cnd_wait(&malloc_cond, &malloc_mutex); - - return; - } - else { - initer_tid = erts_thr_self(); - elib_is_initing = 1; - } - } -#else - if(elib_is_initing) - return; - elib_is_initing = 1; -#endif - -#endif /* #if THREAD_SAFE_ELIB_MALLOC */ - - /* Do the actual initialization of the malloc implementation */ - init_elib_malloc(addr, sz); - -#if THREAD_SAFE_ELIB_MALLOC - -#if !USE_RECURSIVE_MALLOC_MUTEX - erts_mtx_unlock(&malloc_mutex); -#endif - - /* Recursive calls to malloc are allowed here... */ - erts_mtx_set_forksafe(&malloc_mutex); - -#if !USE_RECURSIVE_MALLOC_MUTEX - erts_mtx_lock(&malloc_mutex); - elib_is_initing = 0; -#endif - -#endif /* #if THREAD_SAFE_ELIB_MALLOC */ - - elib_need_init = 0; - -#if THREAD_SAFE_ELIB_MALLOC && !USE_RECURSIVE_MALLOC_MUTEX - erts_cnd_broadcast(&malloc_cond); -#endif - -} - -static void init_elib_malloc(EWord* addr, EWord sz) -{ - int i; - FreeBlock* freep; - EWord tmp_sz; -#ifdef ELIB_HEAP_SBRK - char* top; - EWord n; -#endif - - max_allocated = 0; - tot_allocated = 0; - root = NULL; - - /* Get the page size (may involve system call!!!) */ - page_size = PAGE_SIZE; - -#if defined(ELIB_HEAP_SBRK) - sz = PAGES(ELIB_HEAP_SIZE)*page_size; - - if ((top = (char*) sbrk(0)) == (char*)-1) { - elib_printf(stderr, "could not initialize elib, sbrk(0)"); - ELIB_FAILURE; - } - n = PAGE_ALIGN(top) - top; - if ((top = (char*) sbrk(n)) == (char*)-1) { - elib_printf(stderr, "could not initialize elib, sbrk(n)"); - ELIB_FAILURE; - } - if ((eheap = (EWord*) sbrk(sz)) == (EWord*)-1) { - elib_printf(stderr, "could not initialize elib, sbrk(SIZE)"); - ELIB_FAILURE; - } - sz = WORDS(ELIB_HEAP_SIZE); -#elif defined(ELIB_HEAP_FIXED) - eheap = fix_heap; - sz = WORDS(ELIB_HEAP_SIZE); -#elif defined(ELIB_HEAP_USER) - eheap = addr; - sz = WORDS(sz); -#else - return -1; -#endif - eheap_size = 0; - - /* Make sure that the first word of the heap_head is aligned */ - addr = ALIGN(eheap+1); - sz -= ((addr - 1) - eheap); /* Subtract unusable size */ - eheap_top = eheap = addr - 1; /* Set new aligned heap start */ - - eheap_top[sz-1] = 0; /* Heap stop mark */ - - addr = eheap; - heap_head = (AllocatedBlock*) addr; - heap_head->hdr = MIN_ALIGN_SIZE; - for (i = 0; i < MIN_ALIGN_SIZE; i++) - heap_head->v[i] = 0; - - addr += (MIN_ALIGN_SIZE+1); - freep = (FreeBlock*) addr; - tmp_sz = sz - (((MIN_ALIGN_SIZE+1) + MIN_BLOCK_SIZE) + 1 + 1); - mark_free(freep, tmp_sz); - link_free_block((Block_t *) freep); - - /* No need to align heap tail */ - heap_tail = (AllocatedBlock*) &eheap_top[sz-MIN_BLOCK_SIZE-1]; - heap_tail->hdr = FREE_ABOVE_BIT | MIN_WORD_SIZE; - heap_tail->v[0] = 0; - heap_tail->v[1] = 0; - heap_tail->v[2] = 0; - - eheap_top += sz; - eheap_size += sz; - - heap_locked = 0; -} - -#ifdef ELIB_HEAP_USER -void elib_force_init(EWord* addr, EWord sz) -{ - elib_need_init = 1; - elib_init(addr,sz); -} -#endif - -#ifdef ELIB_HEAP_SBRK - -/* -** need in number of words (should include head and tail words) -*/ -static int expand_sbrk(EWord sz) -{ - EWord* p; - EWord bytes = sz * sizeof(EWord); - EWord size; - AllocatedBlock* tail; - - if (bytes < ELIB_HEAP_SIZE) - size = PAGES(ELIB_HEAP_INCREAMENT)*page_size; - else - size = PAGES(bytes)*page_size; - - if ((p = (EWord*) sbrk(size)) == ((EWord*) -1)) - return -1; - - if (p != eheap_top) { - elib_printf(stderr, "panic: sbrk moved\n"); - ELIB_FAILURE; - } - - sz = WORDS(size); - - /* Set new endof heap marker and a new heap tail */ - eheap_top[sz-1] = 0; - - tail = (AllocatedBlock*) &eheap_top[sz-MIN_BLOCK_SIZE-1]; - tail->hdr = FREE_ABOVE_BIT | MIN_WORD_SIZE; - tail->v[0] = 0; - tail->v[1] = 0; - tail->v[2] = 0; - - /* Patch old tail with new appended size */ - heap_tail->hdr = (heap_tail->hdr & FREE_ABOVE_BIT) | - (MIN_WORD_SIZE+1+(sz-MIN_BLOCK_SIZE-1)); - deallocate(heap_tail, 0); - - heap_tail = tail; - - eheap_size += sz; - eheap_top += sz; - - return 0; -} - -#endif /* ELIB_HEAP_SBRK */ - - -/* -** Scan heap and check for corrupted heap -*/ -int elib_check_heap(void) -{ - AllocatedBlock* p = heap_head; - EWord sz; - - if (heap_locked) { - elib_printf(stderr, "heap is locked no info avaiable\n"); - return 0; - } - - while((sz = SIZEOF(p)) != 0) { - if (IS_FREE(p)) { - if (p->v[sz-1] != sz) { - elib_printf(stderr, "panic: heap corrupted\r\n"); - ELIB_FAILURE; - } - p = (AllocatedBlock*) (p->v + sz); - if (!IS_FREE_ABOVE(p)) { - elib_printf(stderr, "panic: heap corrupted\r\n"); - ELIB_FAILURE; - } - } - else - p = (AllocatedBlock*) (p->v + sz); - } - return 1; -} - -/* -** Load the byte vector pointed to by v of length vsz -** with a heap image -** The scale is defined by vsz and the current heap size -** free = 0, full = 255 -** -** -*/ -int elib_heap_map(EByte* v, int vsz) -{ - AllocatedBlock* p = heap_head; - EWord sz; - int gsz = eheap_size / vsz; /* The granuality used */ - int fsz = 0; - int usz = 0; - - if (gsz == 0) - return -1; /* too good reolution */ - - while((sz = SIZEOF(p)) != 0) { - if (IS_FREE(p)) { - fsz += sz; - if ((fsz + usz) > gsz) { - *v++ = (255*usz)/gsz; - fsz -= (gsz - usz); - usz = 0; - while(fsz >= gsz) { - *v++ = 0; - fsz -= gsz; - } - } - } - else { - usz += sz; - if ((fsz + usz) > gsz) { - *v++ = 255 - (255*fsz)/gsz; - usz -= (gsz - fsz); - fsz = 0; - while(usz >= gsz) { - *v++ = 255; - usz -= gsz; - } - } - } - p = (AllocatedBlock*) (p->v + sz); - } - return 0; -} - -/* -** Generate a histogram of free/allocated blocks -** Count granuality of 10 gives -** (0-10],(10-100],(100-1000],(1000-10000] ... -** (0-2], (2-4], (4-8], (8-16], .... -*/ -static int i_logb(EWord size, int base) -{ - int lg = 0; - while(size >= base) { - size /= base; - lg++; - } - return lg; -} - -int elib_histo(EWord* vf, EWord* va, int vsz, int base) -{ - AllocatedBlock* p = heap_head; - EWord sz; - int i; - int linear; - - if ((vsz <= 1) || (vf == 0 && va == 0)) - return -1; - - if (base < 0) { - linear = 1; - base = -base; - } - else - linear = 0; - - if (base <= 1) - return -1; - - if (vf != 0) { - for (i = 0; i < vsz; i++) - vf[i] = 0; - } - if (va != 0) { - for (i = 0; i < vsz; i++) - va[i] = 0; - } - - while((sz = SIZEOF(p)) != 0) { - if (IS_FREE(p)) { - if (vf != 0) { - int val; - if (linear) - val = sz / base; - else - val = i_logb(sz, base); - if (val >= vsz) - vf[vsz-1]++; - else - vf[val]++; - } - } - else { - if (va != 0) { - int val; - if (linear) - val = sz / base; - else - val = i_logb(sz, base); - if (val >= vsz) - va[vsz-1]++; - else - va[val]++; - } - } - p = (AllocatedBlock*) (p->v + sz); - } - return 0; -} - -/* -** Fill the info structure with actual values -** Total -** Allocated -** Free -** maxMaxFree -*/ -void elib_stat(struct elib_stat* info) -{ - EWord blks = 0; - EWord sz_free = 0; - EWord sz_alloc = 0; - EWord sz_max_free = 0; - EWord sz_min_used = 0x7fffffff; - EWord sz; - EWord num_free = 0; - AllocatedBlock* p = heap_head; - - info->mem_total = eheap_size; - - p = (AllocatedBlock*) (p->v + SIZEOF(p)); - - while((sz = SIZEOF(p)) != 0) { - blks++; - if (IS_FREE(p)) { - if (sz > sz_max_free) - sz_max_free = sz; - sz_free += sz; - ++num_free; - } - else { - if (sz < sz_min_used) - sz_min_used = sz; - sz_alloc += sz; - } - p = (AllocatedBlock*) (p->v + sz); - } - info->mem_blocks = blks; - info->free_blocks = num_free; - info->mem_alloc = sz_alloc; - info->mem_free = sz_free; - info->min_used = sz_min_used; - info->max_free = sz_max_free; - info->mem_max_alloc = max_allocated; - ASSERT(sz_alloc == tot_allocated); -} - -/* -** Dump the heap -*/ -void elib_heap_dump(char* label) -{ - AllocatedBlock* p = heap_head; - EWord sz; - - elib_printf(stderr, "HEAP DUMP (%s)\n", label); - if (!elib_check_heap()) - return; - - while((sz = SIZEOF(p)) != 0) { - if (IS_FREE(p)) { - elib_printf(stderr, "%p: FREE, size = %d\n", p, (int) sz); - } - else { - elib_printf(stderr, "%p: USED, size = %d %s\n", p, (int) sz, - IS_FREE_ABOVE(p)?"(FREE ABOVE)":""); - } - p = (AllocatedBlock*) (p->v + sz); - } -} - -/* -** Scan heaps and count: -** free_size, allocated_size, max_free_block -*/ -void elib_statistics(void* to) -{ - struct elib_stat info; - EWord frag; - - if (!elib_check_heap()) - return; - - elib_stat(&info); - - frag = 1000 - ((1000 * info.max_free) / info.mem_free); - - elib_printf(to, "Heap Statistics: total(%d), blocks(%d), frag(%d.%d%%)\n", - info.mem_total, info.mem_blocks, - (int) frag/10, (int) frag % 10); - - elib_printf(to, " allocated(%d), free(%d), " - "free_blocks(%d)\n", - info.mem_alloc, info.mem_free,info.free_blocks); - elib_printf(to, " max_free(%d), min_used(%d)\n", - info.max_free, info.min_used); -} - -/* -** Allocate a least nb bytes with alignment a -** Algorithm: -** 1) Try locate a block which match exacly among the by direct index. -** 2) Try using a fix block of greater size -** 3) Try locate a block by searching in lists where block sizes -** X may vary between 2^i < X <= 2^(i+1) -** -** Reset memory to zero if clear is true -*/ -static AllocatedBlock* allocate(EWord nb, EWord a, int clear) -{ - FreeBlock* p; - EWord nw; - - if (a == ELIB_ALIGN) { - /* - * Common case: Called by malloc(), realloc(), calloc(). - */ - nw = nb < MIN_BYTE_SIZE ? MIN_ALIGN_SIZE : ALIGN_SIZE(nb); - - if ((p = alloc_block(nw)) == 0) - return NULL; - } else { - /* - * Special case: Called by memalign(). - */ - EWord asz, szp, szq, tmpsz; - FreeBlock *q; - - if ((p = alloc_block((1+MIN_ALIGN_SIZE)*sizeof(EWord)+a-1+nb)) == 0) - return NULL; - - asz = a - ((EWord) ((AllocatedBlock *)p)->v) % a; - - if (asz != a) { - /* Enforce the alignment requirement by cutting of a free - block at the beginning of the block. */ - - if (asz < (1+MIN_ALIGN_SIZE)*sizeof(EWord) && !IS_FREE_ABOVE(p)) { - /* Not enough room to cut of a free block; - increase align size */ - asz += (((1+MIN_ALIGN_SIZE)*sizeof(EWord) + a - 1)/a)*a; - } - - szq = ALIGN_SIZE(asz - sizeof(EWord)); - szp = SIZEOF(p) - szq - 1; - - q = p; - p = (FreeBlock*) (((EWord*) q) + szq + 1); - p->hdr = FREE_ABOVE_BIT | FREE_BIT | szp; - - if (IS_FREE_ABOVE(q)) { /* This should not be possible I think, - but just in case... */ - tmpsz = SIZEOF_ABOVE(q) + 1; - szq += tmpsz; - q = (FreeBlock*) (((EWord*) q) - tmpsz); - unlink_free_block((Block_t *) q); - q->hdr = (q->hdr & FREE_ABOVE_BIT) | FREE_BIT | szq; - } - mark_free(q, szq); - link_free_block((Block_t *) q); - - } /* else already had the correct alignment */ - - nw = nb < MIN_BYTE_SIZE ? MIN_ALIGN_SIZE : ALIGN_SIZE(nb); - } - - split_block(p, nw, SIZEOF(p)); - - STAT_ALLOCED_BLOCK(SIZEOF(p)); - - if (clear) { - EWord* pp = ((AllocatedBlock*)p)->v; - - while(nw--) - *pp++ = 0; - } - - return (AllocatedBlock*) p; -} - - -/* -** Deallocate memory pointed to by p -** 1. Merge with block above if this block is free -** 2. Merge with block below if this block is free -** Link the block to the correct free list -** -** p points to the block header! -** -*/ -static void deallocate(AllocatedBlock* p, int stat_count) -{ - FreeBlock* q; - EWord szq; - EWord szp; - - szp = SIZEOF(p); - - if (stat_count) - STAT_FREED_BLOCK(SIZEOF(p)); - - if (IS_FREE_ABOVE(p)) { - szq = SIZEOF_ABOVE(p); - q = (FreeBlock*) ( ((EWord*) p) - szq - 1); - unlink_free_block((Block_t *) q); - - p = (AllocatedBlock*) q; - szp += (szq + 1); - } - q = (FreeBlock*) (p->v + szp); - if (IS_FREE(q)) { - szq = SIZEOF(q); - unlink_free_block((Block_t *) q); - szp += (szq + 1); - } - else - q->hdr |= FREE_ABOVE_BIT; - - /* The block above p can NEVER be free !!! */ - p->hdr = FREE_BIT | szp; - p->v[szp-1] = szp; - - link_free_block((Block_t *) p); -} - -/* -** Reallocate memory -** If preserve is true then data is moved if neccesary -*/ -static AllocatedBlock* reallocate(AllocatedBlock* p, EWord nb, int preserve) -{ - EWord szp; - EWord szq; - EWord sz; - EWord nw; - FreeBlock* q; - - if (nb < MIN_BYTE_SIZE) - nw = MIN_ALIGN_SIZE; - else - nw = ALIGN_SIZE(nb); - - sz = szp = SIZEOF(p); - - STAT_FREED_BLOCK(szp); - - /* Merge with block below */ - q = (FreeBlock*) (p->v + szp); - if (IS_FREE(q)) { - szq = SIZEOF(q); - unlink_free_block((Block_t *) q); - szp += (szq + 1); - } - - if (nw <= szp) { - split_block((FreeBlock *) p, nw, szp); - STAT_ALLOCED_BLOCK(SIZEOF(p)); - return p; - } - else { - EWord* dp = p->v; - AllocatedBlock* npp; - - if (IS_FREE_ABOVE(p)) { - szq = SIZEOF_ABOVE(p); - if (szq + szp + 1 >= nw) { - q = (FreeBlock*) (((EWord*) p) - szq - 1); - unlink_free_block((Block_t * )q); - szp += (szq + 1); - p = (AllocatedBlock*) q; - - if (preserve) { - EWord* pp = p->v; - while(sz--) - *pp++ = *dp++; - } - split_block((FreeBlock *) p, nw, szp); - STAT_ALLOCED_BLOCK(SIZEOF(p)); - return p; - } - } - - /* - * Update p so that allocate() and deallocate() works. - * (Note that allocate() may call expand_sbrk(), which in - * in turn calls deallocate().) - */ - - p->hdr = (p->hdr & FREE_ABOVE_BIT) | szp; - p->v[szp] &= ~FREE_ABOVE_BIT; - - npp = allocate(nb, ELIB_ALIGN, 0); - if(npp == NULL) - return NULL; - if (preserve) { - EWord* pp = npp->v; - while(sz--) - *pp++ = *dp++; - } - deallocate(p, 0); - return npp; - } -} - -/* -** What malloc() and friends should do (and return) when the heap is -** exhausted. [sverkerw] -*/ -static void* heap_exhausted(void) -{ - /* Choose behaviour */ -#if 0 - /* Crash-and-burn --- leave a usable corpse (hopefully) */ - abort(); -#endif - /* The usual ANSI-compliant behaviour */ - return NULL; -} - -/* -** Allocate size bytes of memory -*/ -void* ELIB_PREFIX(malloc, (size_t nb)) -{ - void *res; - AllocatedBlock* p; - - erts_mtx_lock(&malloc_mutex); - if (elib_need_init) - locked_elib_init(NULL,(EWord)0); - - if (nb == 0) - res = NULL; - else if ((p = allocate(nb, ELIB_ALIGN, 0)) != 0) { - ELIB_ALIGN_CHECK(p->v); - res = p->v; - } - else - res = heap_exhausted(); - - erts_mtx_unlock(&malloc_mutex); - - return res; -} - - -void* ELIB_PREFIX(calloc, (size_t nelem, size_t size)) -{ - void *res; - int nb; - AllocatedBlock* p; - - erts_mtx_lock(&malloc_mutex); - if (elib_need_init) - locked_elib_init(NULL,(EWord)0); - - if ((nb = nelem * size) == 0) - res = NULL; - else if ((p = allocate(nb, ELIB_ALIGN, 1)) != 0) { - ELIB_ALIGN_CHECK(p->v); - res = p->v; - } - else - res = heap_exhausted(); - - erts_mtx_unlock(&malloc_mutex); - - return res; -} - -/* -** Free memory allocated by malloc -*/ - -void ELIB_PREFIX(free, (EWord* p)) -{ - erts_mtx_lock(&malloc_mutex); - if (elib_need_init) - locked_elib_init(NULL,(EWord)0); - - if (p != 0) - deallocate((AllocatedBlock*)(p-1), 1); - - erts_mtx_unlock(&malloc_mutex); -} - -void ELIB_PREFIX(cfree, (EWord* p)) -{ - ELIB_PREFIX(free, (p)); -} - - -/* -** Realloc the memory allocated in p to nb number of bytes -** -*/ - -void* ELIB_PREFIX(realloc, (EWord* p, size_t nb)) -{ - void *res = NULL; - AllocatedBlock* pp; - - erts_mtx_lock(&malloc_mutex); - if (elib_need_init) - locked_elib_init(NULL,(EWord)0); - - if (p != 0) { - pp = (AllocatedBlock*) (p-1); - if (nb > 0) { - if ((pp = reallocate(pp, nb, 1)) != 0) { - ELIB_ALIGN_CHECK(pp->v); - res = pp->v; - } - } - else - deallocate(pp, 1); - } - else if (nb > 0) { - if ((pp = allocate(nb, ELIB_ALIGN, 0)) != 0) { - ELIB_ALIGN_CHECK(pp->v); - res = pp->v; - } - else - res = heap_exhausted(); - } - - erts_mtx_unlock(&malloc_mutex); - - return res; -} - -/* -** Resize the memory area pointed to by p with nb number of bytes -*/ -void* ELIB_PREFIX(memresize, (EWord* p, int nb)) -{ - void *res = NULL; - AllocatedBlock* pp; - - erts_mtx_lock(&malloc_mutex); - if (elib_need_init) - locked_elib_init(NULL,(EWord)0); - - if (p != 0) { - pp = (AllocatedBlock*) (p-1); - if (nb > 0) { - if ((pp = reallocate(pp, nb, 0)) != 0) { - ELIB_ALIGN_CHECK(pp->v); - res = pp->v; - } - } - else - deallocate(pp, 1); - } - else if (nb > 0) { - if ((pp = allocate(nb, ELIB_ALIGN, 0)) != 0) { - ELIB_ALIGN_CHECK(pp->v); - res = pp->v; - } - else - res = heap_exhausted(); - } - - erts_mtx_unlock(&malloc_mutex); - - return res; -} - - -/* Create aligned memory a must be a power of 2 !!! */ - -void* ELIB_PREFIX(memalign, (int a, int nb)) -{ - void *res; - AllocatedBlock* p; - - erts_mtx_lock(&malloc_mutex); - if (elib_need_init) - locked_elib_init(NULL,(EWord)0); - - if (nb == 0 || a <= 0) - res = NULL; - else if ((p = allocate(nb, a, 0)) != 0) { - ALIGN_CHECK(a, p->v); - res = p->v; - } - else - res = heap_exhausted(); - - erts_mtx_unlock(&malloc_mutex); - - return res; -} - -void* ELIB_PREFIX(valloc, (int nb)) -{ - return ELIB_PREFIX(memalign, (page_size, nb)); -} - - -void* ELIB_PREFIX(pvalloc, (int nb)) -{ - return ELIB_PREFIX(memalign, (page_size, PAGES(nb)*page_size)); -} -/* Return memory size for pointer p in bytes */ - -int ELIB_PREFIX(memsize, (p)) -EWord* p; -{ - return SIZEOF((AllocatedBlock*)(p-1))*4; -} - - -/* -** -------------------------------------------------------------------------- -** DEBUG LIBRARY -** -------------------------------------------------------------------------- -*/ - -#ifdef ELIB_DEBUG - -#define IN_HEAP(p) (((p) >= (char*) eheap) && (p) < (char*) eheap_top) -/* -** ptr_to_block: return the pointer to heap block pointed into by ptr -** Returns 0 if not pointing into a block -*/ - -static EWord* ptr_to_block(char* ptr) -{ - AllocatedBlock* p = heap_head; - EWord sz; - - while((sz = SIZEOF(p)) != 0) { - if ((ptr >= (char*) p->v) && (ptr < (char*)(p->v+sz))) - return p->v; - p = (AllocatedBlock*) (p->v + sz); - } - return 0; -} - -/* -** Validate a pointer -** returns: -** 0 - if points to start of a block -** 1 - if points outsize heap -** -1 - if points inside block -** -*/ -static int check_pointer(char* ptr) -{ - if (IN_HEAP(ptr)) { - if (ptr_to_block(ptr) == 0) - return 1; - return 0; - } - return -1; -} - -/* -** Validate a memory area -** returns: -** 0 - if area is included in a block -** -1 - if area overlap a heap block -** 1 - if area is outside heap -*/ -static int check_area(char* ptr, int n) -{ - if (IN_HEAP(ptr)) { - if (IN_HEAP(ptr+n-1)) { - EWord* p1 = ptr_to_block(ptr); - EWord* p2 = ptr_to_block(ptr+n-1); - - if (p1 == p2) - return (p1 == 0) ? -1 : 0; - return -1; - } - } - else if (IN_HEAP(ptr+n-1)) - return -1; - return 1; -} - -/* -** Check if a block write will overwrite heap block -*/ -static void check_write(char* ptr, int n, char* file, int line, char* fun) -{ - if (check_area(ptr, n) == -1) { - elib_printf(stderr, "RUNTIME ERROR: %s heap overwrite\n", fun); - elib_printf(stderr, "File: %s Line: %d\n", file, line); - ELIB_FAILURE; - } -} - -/* -** Check if a pointer is an allocated object -*/ -static void check_allocated_block(char* ptr, char* file, int line, char* fun) -{ - EWord* q; - - if (!IN_HEAP(ptr) || ((q=ptr_to_block(ptr)) == 0) || (ptr != (char*) q)) { - elib_printf(stderr, "RUNTIME ERROR: %s non heap pointer\n", fun); - elib_printf(stderr, "File: %s Line: %d\n", file, line); - ELIB_FAILURE; - } - - if (IS_FREE((AllocatedBlock*)(q-1))) { - elib_printf(stderr, "RUNTIME ERROR: %s free pointer\n", fun); - elib_printf(stderr, "File: %s Line: %d\n", file, line); - ELIB_FAILURE; - } - -} - -/* -** -------------------------------------------------------------------------- -** DEBUG VERSIONS (COMPILED WITH THE ELIB.H) -** -------------------------------------------------------------------------- -*/ - -void* elib_dbg_malloc(int n, char* file, int line) -{ - return elib__malloc(n); -} - -void* elib_dbg_calloc(int n, int s, char* file, int line) -{ - return elib__calloc(n, s); -} - -void* elib_dbg_realloc(EWord* p, int n, char* file, int line) -{ - if (p == 0) - return elib__malloc(n); - check_allocated_block(p, file, line, "elib_realloc"); - return elib__realloc(p, n); -} - -void elib_dbg_free(EWord* p, char* file, int line) -{ - if (p == 0) - return; - check_allocated_block(p, file, line, "elib_free"); - elib__free(p); -} - -void elib_dbg_cfree(EWord* p, char* file, int line) -{ - if (p == 0) - return; - check_allocated_block(p, file, line, "elib_free"); - elib__cfree(p); -} - -void* elib_dbg_memalign(int a, int n, char* file, int line) -{ - return elib__memalign(a, n); -} - -void* elib_dbg_valloc(int n, char* file, int line) -{ - return elib__valloc(n); -} - -void* elib_dbg_pvalloc(int n, char* file, int line) -{ - return elib__pvalloc(n); -} - -void* elib_dbg_memresize(EWord* p, int n, char* file, int line) -{ - if (p == 0) - return elib__malloc(n); - check_allocated_block(p, file, line, "elib_memresize"); - return elib__memresize(p, n); -} - -int elib_dbg_memsize(void* p, char* file, int line) -{ - check_allocated_block(p, file, line, "elib_memsize"); - return elib__memsize(p); -} - -/* -** -------------------------------------------------------------------------- -** LINK TIME FUNCTIONS (NOT COMPILED CALLS) -** -------------------------------------------------------------------------- -*/ - -void* elib_malloc(int n) -{ - return elib_dbg_malloc(n, "", -1); -} - -void* elib_calloc(int n, int s) -{ - return elib_dbg_calloc(n, s, "", -1); -} - -void* elib_realloc(EWord* p, int n) -{ - return elib_dbg_realloc(p, n, "", -1); -} - -void elib_free(EWord* p) -{ - elib_dbg_free(p, "", -1); -} - -void elib_cfree(EWord* p) -{ - elib_dbg_cfree(p, "", -1); -} - -void* elib_memalign(int a, int n) -{ - return elib_dbg_memalign(a, n, "", -1); -} - -void* elib_valloc(int n) -{ - return elib_dbg_valloc(n, "", -1); -} - -void* elib_pvalloc(int n) -{ - return elib_dbg_pvalloc(n, "", -1); -} - -void* elib_memresize(EWord* p, int n) -{ - return elib_dbg_memresize(p, n, "", -1); -} - - -int elib_memsize(EWord* p) -{ - return elib_dbg_memsize(p, "", -1); -} - -#endif /* ELIB_DEBUG */ - -/* -** -------------------------------------------------------------------------- -** Map c library functions to elib -** -------------------------------------------------------------------------- -*/ - -#if defined(ELIB_ALLOC_IS_CLIB) -void* malloc(size_t nb) -{ - return elib_malloc(nb); -} - -void* calloc(size_t nelem, size_t size) -{ - return elib_calloc(nelem, size); -} - - -void free(void *p) -{ - elib_free(p); -} - -void cfree(void *p) -{ - elib_cfree(p); -} - -void* realloc(void* p, size_t nb) -{ - return elib_realloc(p, nb); -} - - -void* memalign(size_t a, size_t s) -{ - return elib_memalign(a, s); -} - -void* valloc(size_t nb) -{ - return elib_valloc(nb); -} - -void* pvalloc(size_t nb) -{ - return elib_pvalloc(nb); -} - -#if 0 -void* memresize(void* p, int nb) -{ - return elib_memresize(p, nb); -} - -int memsize(void* p) -{ - return elib_memsize(p); -} -#endif -#endif /* ELIB_ALLOC_IS_CLIB */ - -#endif /* ENABLE_ELIB_MALLOC */ - -void elib_ensure_initialized(void) -{ -#ifdef ENABLE_ELIB_MALLOC -#ifndef ELIB_DONT_INITIALIZE - elib_init(NULL, 0); -#endif -#endif -} - -#ifdef ENABLE_ELIB_MALLOC -/** - ** A Slightly modified version of the "address order best fit" algorithm - ** used in erl_bestfit_alloc.c. Comments refer to that implementation. - **/ - -/* - * Description: A combined "address order best fit"/"best fit" allocator - * based on a Red-Black (binary search) Tree. The search, - * insert, and delete operations are all O(log n) operations - * on a Red-Black Tree. In the "address order best fit" case - * n equals number of free blocks, and in the "best fit" case - * n equals number of distinct sizes of free blocks. Red-Black - * Trees are described in "Introduction to Algorithms", by - * Thomas H. Cormen, Charles E. Leiserson, and - * Ronald L. Riverest. - * - * This module is a callback-module for erl_alloc_util.c - * - * Author: Rickard Green - */ - -#ifdef DEBUG -#if 0 -#define HARD_DEBUG -#endif -#else -#undef HARD_DEBUG -#endif - -#define SZ_MASK SIZE_MASK -#define FLG_MASK (~(SZ_MASK)) - -#define BLK_SZ(B) (*((Block_t *) (B)) & SZ_MASK) - -#define TREE_NODE_FLG (((Uint) 1) << 0) -#define RED_FLG (((Uint) 1) << 1) -#ifdef HARD_DEBUG -# define LEFT_VISITED_FLG (((Uint) 1) << 2) -# define RIGHT_VISITED_FLG (((Uint) 1) << 3) -#endif - -#define IS_TREE_NODE(N) (((RBTree_t *) (N))->flags & TREE_NODE_FLG) -#define IS_LIST_ELEM(N) (!IS_TREE_NODE(((RBTree_t *) (N)))) - -#define SET_TREE_NODE(N) (((RBTree_t *) (N))->flags |= TREE_NODE_FLG) -#define SET_LIST_ELEM(N) (((RBTree_t *) (N))->flags &= ~TREE_NODE_FLG) - -#define IS_RED(N) (((RBTree_t *) (N)) \ - && ((RBTree_t *) (N))->flags & RED_FLG) -#define IS_BLACK(N) (!IS_RED(((RBTree_t *) (N)))) - -#define SET_RED(N) (((RBTree_t *) (N))->flags |= RED_FLG) -#define SET_BLACK(N) (((RBTree_t *) (N))->flags &= ~RED_FLG) - -#undef ASSERT -#define ASSERT ASSERT_EXPR - -#if 1 -#define RBT_ASSERT ASSERT -#else -#define RBT_ASSERT(x) -#endif - - -#ifdef HARD_DEBUG -static RBTree_t * check_tree(Uint); -#endif - -#ifdef ERTS_INLINE -# ifndef ERTS_CAN_INLINE -# define ERTS_CAN_INLINE 1 -# endif -#else -# if defined(__GNUC__) -# define ERTS_CAN_INLINE 1 -# define ERTS_INLINE __inline__ -# elif defined(__WIN32__) -# define ERTS_CAN_INLINE 1 -# define ERTS_INLINE __inline -# else -# define ERTS_CAN_INLINE 0 -# define ERTS_INLINE -# endif -#endif - -/* Types... */ -#if 0 -typedef struct RBTree_t_ RBTree_t; - -struct RBTree_t_ { - Block_t hdr; - Uint flags; - RBTree_t *parent; - RBTree_t *left; - RBTree_t *right; -}; -#endif - -#if 0 -typedef struct { - RBTree_t t; - RBTree_t *next; -} RBTreeList_t; - -#define LIST_NEXT(N) (((RBTreeList_t *) (N))->next) -#define LIST_PREV(N) (((RBTreeList_t *) (N))->t.parent) -#endif - -#ifdef DEBUG - -/* Destroy all tree fields */ -#define DESTROY_TREE_NODE(N) \ - sys_memset((void *) (((Block_t *) (N)) + 1), \ - 0xff, \ - (sizeof(RBTree_t) - sizeof(Block_t))) - -/* Destroy all tree and list fields */ -#define DESTROY_LIST_ELEM(N) \ - sys_memset((void *) (((Block_t *) (N)) + 1), \ - 0xff, \ - (sizeof(RBTreeList_t) - sizeof(Block_t))) - -#else - -#define DESTROY_TREE_NODE(N) -#define DESTROY_LIST_ELEM(N) - -#endif - - -/* - * Red-Black Tree operations needed - */ - -static ERTS_INLINE void -left_rotate(RBTree_t **root, RBTree_t *x) -{ - RBTree_t *y = x->right; - x->right = y->left; - if (y->left) - y->left->parent = x; - y->parent = x->parent; - if (!y->parent) { - RBT_ASSERT(*root == x); - *root = y; - } - else if (x == x->parent->left) - x->parent->left = y; - else { - RBT_ASSERT(x == x->parent->right); - x->parent->right = y; - } - y->left = x; - x->parent = y; -} - -static ERTS_INLINE void -right_rotate(RBTree_t **root, RBTree_t *x) -{ - RBTree_t *y = x->left; - x->left = y->right; - if (y->right) - y->right->parent = x; - y->parent = x->parent; - if (!y->parent) { - RBT_ASSERT(*root == x); - *root = y; - } - else if (x == x->parent->right) - x->parent->right = y; - else { - RBT_ASSERT(x == x->parent->left); - x->parent->left = y; - } - y->right = x; - x->parent = y; -} - - -/* - * Replace node x with node y - * NOTE: block header of y is not changed - */ -static ERTS_INLINE void -replace(RBTree_t **root, RBTree_t *x, RBTree_t *y) -{ - - if (!x->parent) { - RBT_ASSERT(*root == x); - *root = y; - } - else if (x == x->parent->left) - x->parent->left = y; - else { - RBT_ASSERT(x == x->parent->right); - x->parent->right = y; - } - if (x->left) { - RBT_ASSERT(x->left->parent == x); - x->left->parent = y; - } - if (x->right) { - RBT_ASSERT(x->right->parent == x); - x->right->parent = y; - } - - y->flags = x->flags; - y->parent = x->parent; - y->right = x->right; - y->left = x->left; - - DESTROY_TREE_NODE(x); - -} - -static void -tree_insert_fixup(RBTree_t *blk) -{ - RBTree_t *x = blk, *y; - - /* - * Rearrange the tree so that it satisfies the Red-Black Tree properties - */ - - RBT_ASSERT(x != root && IS_RED(x->parent)); - do { - - /* - * x and its parent are both red. Move the red pair up the tree - * until we get to the root or until we can separate them. - */ - - RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_BLACK(x->parent->parent)); - RBT_ASSERT(x->parent->parent); - - if (x->parent == x->parent->parent->left) { - y = x->parent->parent->right; - if (IS_RED(y)) { - SET_BLACK(y); - x = x->parent; - SET_BLACK(x); - x = x->parent; - SET_RED(x); - } - else { - - if (x == x->parent->right) { - x = x->parent; - left_rotate(&root, x); - } - - RBT_ASSERT(x == x->parent->parent->left->left); - RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_RED(x->parent)); - RBT_ASSERT(IS_BLACK(x->parent->parent)); - RBT_ASSERT(IS_BLACK(y)); - - SET_BLACK(x->parent); - SET_RED(x->parent->parent); - right_rotate(&root, x->parent->parent); - - RBT_ASSERT(x == x->parent->left); - RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_RED(x->parent->right)); - RBT_ASSERT(IS_BLACK(x->parent)); - break; - } - } - else { - RBT_ASSERT(x->parent == x->parent->parent->right); - y = x->parent->parent->left; - if (IS_RED(y)) { - SET_BLACK(y); - x = x->parent; - SET_BLACK(x); - x = x->parent; - SET_RED(x); - } - else { - - if (x == x->parent->left) { - x = x->parent; - right_rotate(&root, x); - } - - RBT_ASSERT(x == x->parent->parent->right->right); - RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_RED(x->parent)); - RBT_ASSERT(IS_BLACK(x->parent->parent)); - RBT_ASSERT(IS_BLACK(y)); - - SET_BLACK(x->parent); - SET_RED(x->parent->parent); - left_rotate(&root, x->parent->parent); - - RBT_ASSERT(x == x->parent->right); - RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_RED(x->parent->left)); - RBT_ASSERT(IS_BLACK(x->parent)); - break; - } - } - } while (x != root && IS_RED(x->parent)); - - SET_BLACK(root); -} - -static void -unlink_free_block(Block_t *del) -{ - Uint spliced_is_black; - RBTree_t *x, *y, *z = (RBTree_t *) del; - RBTree_t null_x; /* null_x is used to get the fixup started when we - splice out a node without children. */ - - null_x.parent = NULL; - -#ifdef HARD_DEBUG - check_tree(0); -#endif - - /* Remove node from tree... */ - - /* Find node to splice out */ - if (!z->left || !z->right) - y = z; - else - /* Set y to z:s successor */ - for(y = z->right; y->left; y = y->left); - /* splice out y */ - x = y->left ? y->left : y->right; - spliced_is_black = IS_BLACK(y); - if (x) { - x->parent = y->parent; - } - else if (!x && spliced_is_black) { - x = &null_x; - x->flags = 0; - SET_BLACK(x); - x->right = x->left = NULL; - x->parent = y->parent; - y->left = x; - } - - if (!y->parent) { - RBT_ASSERT(root == y); - root = x; - } - else if (y == y->parent->left) - y->parent->left = x; - else { - RBT_ASSERT(y == y->parent->right); - y->parent->right = x; - } - if (y != z) { - /* We spliced out the successor of z; replace z by the successor */ - replace(&root, z, y); - } - - if (spliced_is_black) { - /* We removed a black node which makes the resulting tree - violate the Red-Black Tree properties. Fixup tree... */ - - while (IS_BLACK(x) && x->parent) { - - /* - * x has an "extra black" which we move up the tree - * until we reach the root or until we can get rid of it. - * - * y is the sibbling of x - */ - - if (x == x->parent->left) { - y = x->parent->right; - RBT_ASSERT(y); - if (IS_RED(y)) { - RBT_ASSERT(y->right); - RBT_ASSERT(y->left); - SET_BLACK(y); - RBT_ASSERT(IS_BLACK(x->parent)); - SET_RED(x->parent); - left_rotate(&root, x->parent); - y = x->parent->right; - } - RBT_ASSERT(y); - RBT_ASSERT(IS_BLACK(y)); - if (IS_BLACK(y->left) && IS_BLACK(y->right)) { - SET_RED(y); - x = x->parent; - } - else { - if (IS_BLACK(y->right)) { - SET_BLACK(y->left); - SET_RED(y); - right_rotate(&root, y); - y = x->parent->right; - } - RBT_ASSERT(y); - if (IS_RED(x->parent)) { - - SET_BLACK(x->parent); - SET_RED(y); - } - RBT_ASSERT(y->right); - SET_BLACK(y->right); - left_rotate(&root, x->parent); - x = root; - break; - } - } - else { - RBT_ASSERT(x == x->parent->right); - y = x->parent->left; - RBT_ASSERT(y); - if (IS_RED(y)) { - RBT_ASSERT(y->right); - RBT_ASSERT(y->left); - SET_BLACK(y); - RBT_ASSERT(IS_BLACK(x->parent)); - SET_RED(x->parent); - right_rotate(&root, x->parent); - y = x->parent->left; - } - RBT_ASSERT(y); - RBT_ASSERT(IS_BLACK(y)); - if (IS_BLACK(y->right) && IS_BLACK(y->left)) { - SET_RED(y); - x = x->parent; - } - else { - if (IS_BLACK(y->left)) { - SET_BLACK(y->right); - SET_RED(y); - left_rotate(&root, y); - y = x->parent->left; - } - RBT_ASSERT(y); - if (IS_RED(x->parent)) { - SET_BLACK(x->parent); - SET_RED(y); - } - RBT_ASSERT(y->left); - SET_BLACK(y->left); - right_rotate(&root, x->parent); - x = root; - break; - } - } - } - SET_BLACK(x); - - if (null_x.parent) { - if (null_x.parent->left == &null_x) - null_x.parent->left = NULL; - else { - RBT_ASSERT(null_x.parent->right == &null_x); - null_x.parent->right = NULL; - } - RBT_ASSERT(!null_x.left); - RBT_ASSERT(!null_x.right); - } - else if (root == &null_x) { - root = NULL; - RBT_ASSERT(!null_x.left); - RBT_ASSERT(!null_x.right); - } - } - - - DESTROY_TREE_NODE(del); - -#ifdef HARD_DEBUG - check_tree(0); -#endif - -} - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ - * "Address order best fit" specific callbacks. * -\* */ - -static void -link_free_block(Block_t *block) -{ - RBTree_t *blk = (RBTree_t *) block; - Uint blk_sz = BLK_SZ(blk); - - blk->flags = 0; - blk->left = NULL; - blk->right = NULL; - - if (!root) { - blk->parent = NULL; - SET_BLACK(blk); - root = blk; - } else { - RBTree_t *x = root; - while (1) { - Uint size; - - size = BLK_SZ(x); - - if (blk_sz < size || (blk_sz == size && blk < x)) { - if (!x->left) { - blk->parent = x; - x->left = blk; - break; - } - x = x->left; - } - else { - if (!x->right) { - blk->parent = x; - x->right = blk; - break; - } - x = x->right; - } - - } - - /* Insert block into size tree */ - RBT_ASSERT(blk->parent); - - SET_RED(blk); - if (IS_RED(blk->parent)) { - tree_insert_fixup(blk); - } - } - -#ifdef HARD_DEBUG - check_tree(0); -#endif -} - - -static Block_t * -get_free_block(Uint size) -{ - RBTree_t *x = root; - RBTree_t *blk = NULL; - Uint blk_sz; - - while (x) { - blk_sz = BLK_SZ(x); - if (blk_sz < size) { - x = x->right; - } - else { - blk = x; - x = x->left; - } - } - - if (!blk) - return NULL; - -#ifdef HARD_DEBUG - ASSERT(blk == check_tree(size)); -#endif - - unlink_free_block((Block_t *) blk); - - return (Block_t *) blk; -} - - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ - * Debug functions * -\* */ - - -#ifdef HARD_DEBUG - -#define IS_LEFT_VISITED(FB) ((FB)->flags & LEFT_VISITED_FLG) -#define IS_RIGHT_VISITED(FB) ((FB)->flags & RIGHT_VISITED_FLG) - -#define SET_LEFT_VISITED(FB) ((FB)->flags |= LEFT_VISITED_FLG) -#define SET_RIGHT_VISITED(FB) ((FB)->flags |= RIGHT_VISITED_FLG) - -#define UNSET_LEFT_VISITED(FB) ((FB)->flags &= ~LEFT_VISITED_FLG) -#define UNSET_RIGHT_VISITED(FB) ((FB)->flags &= ~RIGHT_VISITED_FLG) - - -#if 0 -# define PRINT_TREE -#else -# undef PRINT_TREE -#endif - -#ifdef PRINT_TREE -static void print_tree(void); -#endif - -/* - * Checks that the order between parent and children are correct, - * and that the Red-Black Tree properies are satisfied. if size > 0, - * check_tree() returns a node that satisfies "best fit" resp. - * "address order best fit". - * - * The Red-Black Tree properies are: - * 1. Every node is either red or black. - * 2. Every leaf (NIL) is black. - * 3. If a node is red, then both its children are black. - * 4. Every simple path from a node to a descendant leaf - * contains the same number of black nodes. - */ - -static RBTree_t * -check_tree(Uint size) -{ - RBTree_t *res = NULL; - Sint blacks; - Sint curr_blacks; - RBTree_t *x; - -#ifdef PRINT_TREE - print_tree(); -#endif - - if (!root) - return res; - - x = root; - ASSERT(IS_BLACK(x)); - ASSERT(!x->parent); - curr_blacks = 1; - blacks = -1; - - while (x) { - if (!IS_LEFT_VISITED(x)) { - SET_LEFT_VISITED(x); - if (x->left) { - x = x->left; - if (IS_BLACK(x)) - curr_blacks++; - continue; - } - else { - if (blacks < 0) - blacks = curr_blacks; - ASSERT(blacks == curr_blacks); - } - } - - if (!IS_RIGHT_VISITED(x)) { - SET_RIGHT_VISITED(x); - if (x->right) { - x = x->right; - if (IS_BLACK(x)) - curr_blacks++; - continue; - } - else { - if (blacks < 0) - blacks = curr_blacks; - ASSERT(blacks == curr_blacks); - } - } - - - if (IS_RED(x)) { - ASSERT(IS_BLACK(x->right)); - ASSERT(IS_BLACK(x->left)); - } - - ASSERT(x->parent || x == root); - - if (x->left) { - ASSERT(x->left->parent == x); - ASSERT(BLK_SZ(x->left) < BLK_SZ(x) - || (BLK_SZ(x->left) == BLK_SZ(x) && x->left < x)); - } - - if (x->right) { - ASSERT(x->right->parent == x); - ASSERT(BLK_SZ(x->right) > BLK_SZ(x) - || (BLK_SZ(x->right) == BLK_SZ(x) && x->right > x)); - } - - if (size && BLK_SZ(x) >= size) { - if (!res - || BLK_SZ(x) < BLK_SZ(res) - || (BLK_SZ(x) == BLK_SZ(res) && x < res)) - res = x; - } - - UNSET_LEFT_VISITED(x); - UNSET_RIGHT_VISITED(x); - if (IS_BLACK(x)) - curr_blacks--; - x = x->parent; - - } - - ASSERT(curr_blacks == 0); - - UNSET_LEFT_VISITED(root); - UNSET_RIGHT_VISITED(root); - - return res; - -} - - -#ifdef PRINT_TREE -#define INDENT_STEP 2 - -#include <stdio.h> - -static void -print_tree_aux(RBTree_t *x, int indent) -{ - int i; - - if (!x) { - for (i = 0; i < indent; i++) { - putc(' ', stderr); - } - fprintf(stderr, "BLACK: nil\r\n"); - } - else { - print_tree_aux(x->right, indent + INDENT_STEP); - for (i = 0; i < indent; i++) { - putc(' ', stderr); - } - fprintf(stderr, "%s: sz=%lu addr=0x%lx\r\n", - IS_BLACK(x) ? "BLACK" : "RED", - BLK_SZ(x), - (Uint) x); - print_tree_aux(x->left, indent + INDENT_STEP); - } -} - - -static void -print_tree(void) -{ - fprintf(stderr, " --- Size-Adress tree begin ---\r\n"); - print_tree_aux(root, 0); - fprintf(stderr, " --- Size-Adress tree end ---\r\n"); -} - -#endif - -#endif - -#endif /* ENABLE_ELIB_MALLOC */ diff --git a/erts/emulator/beam/elib_stat.h b/erts/emulator/beam/elib_stat.h deleted file mode 100644 index d8c7f31737..0000000000 --- a/erts/emulator/beam/elib_stat.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - -/* -** Interface to elib statistics -** -*/ -#ifndef __ELIB_STAT_H__ -#define __ELIB_STAT_H__ - -struct elib_stat { - int mem_total; /* Number of heap words */ - int mem_blocks; /* Number of block */ - int mem_alloc; /* Number of words in use */ - int mem_free; /* Number of words free */ - int min_used; /* Size of the smallest block used */ - int max_free; /* Size of the largest free block */ - int free_blocks; /* Number of fragments in free list */ - int mem_max_alloc;/* Max number of words in use */ -}; - -EXTERN_FUNCTION(void, elib_statistics, (void*)); -EXTERN_FUNCTION(int, elib_check_heap, (_VOID_)); -EXTERN_FUNCTION(void, elib_heap_dump, (char*)); -EXTERN_FUNCTION(void, elib_stat, (struct elib_stat*)); -EXTERN_FUNCTION(int, elib_heap_map, (unsigned char*, int)); -EXTERN_FUNCTION(int, elib_histo, (unsigned long*, unsigned long*, int, int)); - -#endif diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index b853ec0f01..07b4167b27 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2002-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2002-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -38,9 +38,6 @@ #include "erl_bits.h" #include "erl_instrument.h" #include "erl_mseg.h" -#ifdef ELIB_ALLOC_IS_CLIB -#include "erl_version.h" -#endif #include "erl_monitors.h" #include "erl_bif_timer.h" #if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE) @@ -64,8 +61,15 @@ #ifdef DEBUG static Uint install_debug_functions(void); +#if 0 +#define HARD_DEBUG +#ifdef __GNUC__ +#warning "* * * * * * * * * * * * * *" +#warning "* HARD DEBUG IS ENABLED! *" +#warning "* * * * * * * * * * * * * *" +#endif +#endif #endif -extern void elib_ensure_initialized(void); ErtsAllocatorFunctions_t erts_allctrs[ERTS_ALC_A_MAX+1]; ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1]; @@ -220,7 +224,7 @@ set_default_ll_alloc_opts(struct au_init *ip) ip->init.util.ramv = 0; ip->init.util.mmsbc = 0; ip->init.util.mmmbc = 0; - ip->init.util.sbct = ~((Uint) 0); + ip->init.util.sbct = ~((UWord) 0); ip->init.util.name_prefix = "ll_"; ip->init.util.alloc_no = ERTS_ALC_A_LONG_LIVED; #ifndef SMALL_MEMORY @@ -391,10 +395,14 @@ refuse_af_strategy(struct au_init *init) static void init_thr_ix(int static_ixs); +#ifdef HARD_DEBUG +static void hdbg_init(void); +#endif + void erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) { - Uint extra_block_size = 0; + UWord extra_block_size = 0; int i; erts_alc_hndl_args_init_t init = { 0, @@ -406,6 +414,10 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) ERTS_DEFAULT_ALCU_INIT }; +#ifdef HARD_DEBUG + hdbg_init(); +#endif + erts_sys_alloc_init(); init_thr_ix(erts_no_schedulers); erts_init_utils_mem(); @@ -542,7 +554,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) sys_alloc_opt(SYS_ALLOC_OPT_TRIM_THRESHOLD, init.trim_threshold); sys_alloc_opt(SYS_ALLOC_OPT_TOP_PAD, init.top_pad); - if (erts_allctrs_info[ERTS_FIX_CORE_ALLOCATOR].enabled) erts_fix_core_allocator_ix = ERTS_FIX_CORE_ALLOCATOR; else @@ -719,8 +730,8 @@ start_au_allocator(ErtsAlcType_t alctr_n, init->init.util.name_prefix); tspec->allctr = (Allctr_t **) states; states = ((char *) states) + sizeof(Allctr_t *) * (tspec->size + 1); - states = ((((Uint) states) & ERTS_CACHE_LINE_MASK) - ? (void *) ((((Uint) states) & ~ERTS_CACHE_LINE_MASK) + states = ((((UWord) states) & ERTS_CACHE_LINE_MASK) + ? (void *) ((((UWord) states) & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE) : (void *) states); tspec->allctr[0] = init->thr_spec > 0 ? (Allctr_t *) state : (Allctr_t *) NULL; @@ -1605,12 +1616,13 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) } else { - Eterm tmp_heap[2]; + DeclareTmpHeapNoproc(tmp_heap,2); Eterm wanted_list; if (is_nil(earg)) return NIL; + UseTmpHeapNoproc(2); if (is_not_atom(earg)) wanted_list = earg; else { @@ -1690,15 +1702,18 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) atoms[length] = am_maximum; uintps[length++] = &size.maximum; } - } - else + } else { + UnUseTmpHeapNoproc(2); return am_badarg; + } break; default: + UnUseTmpHeapNoproc(2); return am_badarg; } wanted_list = CDR(list_val(wanted_list)); } + UnUseTmpHeapNoproc(2); if (is_not_nil(wanted_list)) return am_badarg; } @@ -2285,8 +2300,8 @@ erts_allocator_info_term(void *proc, Eterm which_alloc, int only_sz) SysAllocStat sas; Eterm opts_am; Eterm opts; - Eterm as[4]; - Eterm ts[4]; + Eterm as[4]; /* Ok even if !HEAP_ON_C_STACK, not really heap data on stack */ + Eterm ts[4]; /* Ok even if !HEAP_ON_C_STACK, not really heap data on stack */ int l; if (only_sz) @@ -2302,13 +2317,8 @@ erts_allocator_info_term(void *proc, Eterm which_alloc, int only_sz) l = 0; as[l] = am_atom_put("e", 1); ts[l++] = am_true; -#ifdef ELIB_ALLOC_IS_CLIB - as[l] = am_atom_put("m", 1); - ts[l++] = am_atom_put("elib", 4); -#else as[l] = am_atom_put("m", 1); ts[l++] = am_atom_put("libc", 4); -#endif if(sas.trim_threshold >= 0) { as[l] = am_atom_put("tt", 2); ts[l++] = erts_bld_uint(hpp, szp, @@ -2462,11 +2472,7 @@ erts_allocator_info(int to, void *arg) case ERTS_ALC_A_SYSTEM: { SysAllocStat sas; erts_print(to, arg, "option e: true\n"); -#ifdef ELIB_ALLOC_IS_CLIB - erts_print(to, arg, "option m: elib\n"); -#else erts_print(to, arg, "option m: libc\n"); -#endif sys_alloc_stat(&sas); if(sas.trim_threshold >= 0) erts_print(to, arg, "option tt: %d\n", sas.trim_threshold); @@ -2570,13 +2576,8 @@ erts_allocator_options(void *proc) switch (a) { case ERTS_ALC_A_SYSTEM: -#ifdef ELIB_ALLOC_IS_CLIB - as[l] = am_atom_put("m", 1); - ts[l++] = am_atom_put("elib", 4); -#else as[l] = am_atom_put("m", 1); ts[l++] = am_atom_put("libc", 4); -#endif if(sas.trim_threshold >= 0) { as[l] = am_atom_put("tt", 2); ts[l++] = erts_bld_uint(hpp, szp, @@ -2647,23 +2648,7 @@ erts_allocator_options(void *proc) features = length ? erts_bld_list(hpp, szp, length, terms) : NIL; -#if defined(ELIB_ALLOC_IS_CLIB) - { - Eterm version; - int i; - int ver[5]; - i = sscanf(ERLANG_VERSION, - "%d.%d.%d.%d.%d", - &ver[0], &ver[1], &ver[2], &ver[3], &ver[4]); - - version = NIL; - for(i--; i >= 0; i--) - version = erts_bld_cons(hpp, szp, make_small(ver[i]), version); - - res = erts_bld_tuple(hpp, szp, 4, - am_elib_malloc, version, features, settings); - } -#elif defined(__GLIBC__) +#if defined(__GLIBC__) { Eterm AM_glibc = am_atom_put("glibc", 5); Eterm version; @@ -2859,12 +2844,10 @@ unsigned long erts_alc_test(unsigned long op, break; } case 0xf0a: - if (ethr_mutex_lock((ethr_mutex *) a1) != 0) - ERTS_ALC_TEST_ABORT; + ethr_mutex_lock((ethr_mutex *) a1); break; case 0xf0b: - if (ethr_mutex_unlock((ethr_mutex *) a1) != 0) - ERTS_ALC_TEST_ABORT; + ethr_mutex_unlock((ethr_mutex *) a1); break; case 0xf0c: { ethr_cond *cnd = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_cond)); @@ -2880,31 +2863,21 @@ unsigned long erts_alc_test(unsigned long op, break; } case 0xf0e: - if (ethr_cond_broadcast((ethr_cond *) a1) != 0) - ERTS_ALC_TEST_ABORT; + ethr_cond_broadcast((ethr_cond *) a1); break; case 0xf0f: { int res; do { res = ethr_cond_wait((ethr_cond *) a1, (ethr_mutex *) a2); } while (res == EINTR); - if (res != 0) - ERTS_ALC_TEST_ABORT; break; } case 0xf10: { ethr_tid *tid = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_tid)); -#ifdef ERTS_ENABLE_LOCK_COUNT - if (erts_lcnt_thr_create(tid, - (void * (*)(void *)) a1, - (void *) a2, - NULL) != 0) -#else if (ethr_thr_create(tid, (void * (*)(void *)) a1, (void *) a2, NULL) != 0) -#endif ERTS_ALC_TEST_ABORT; return (unsigned long) tid; } @@ -2943,10 +2916,13 @@ unsigned long erts_alc_test(unsigned long op, #undef PRINT_OPS #endif +#ifdef HARD_DEBUG +#define FENCE_SZ (4*sizeof(UWord)) +#else +#define FENCE_SZ (3*sizeof(UWord)) +#endif -#define FENCE_SZ (3*sizeof(Uint)) - -#ifdef ARCH_64 +#if defined(ARCH_64) #define FENCE_PATTERN 0xABCDEF97ABCDEF97 #else #define FENCE_PATTERN 0xABCDEF97 @@ -2956,7 +2932,7 @@ unsigned long erts_alc_test(unsigned long op, #define TYPE_PATTERN_SHIFT 16 #define FIXED_FENCE_PATTERN_MASK \ - (~((Uint) (TYPE_PATTERN_MASK << TYPE_PATTERN_SHIFT))) + (~((UWord) (TYPE_PATTERN_MASK << TYPE_PATTERN_SHIFT))) #define FIXED_FENCE_PATTERN \ (FENCE_PATTERN & FIXED_FENCE_PATTERN_MASK) @@ -2966,22 +2942,170 @@ unsigned long erts_alc_test(unsigned long op, #define GET_TYPE_OF_PATTERN(P) \ (((P) >> TYPE_PATTERN_SHIFT) & TYPE_PATTERN_MASK) +#ifdef HARD_DEBUG + +#define ERL_ALC_HDBG_MAX_MBLK 100000 +#define ERTS_ALC_O_CHECK -1 + +typedef struct hdbg_mblk_ hdbg_mblk; +struct hdbg_mblk_ { + hdbg_mblk *next; + hdbg_mblk *prev; + void *p; + Uint s; + ErtsAlcType_t n; +}; + +static hdbg_mblk hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK]; + +static hdbg_mblk *free_hdbg_mblks; +static hdbg_mblk *used_hdbg_mblks; +static erts_mtx_t hdbg_mblk_mtx; + +static void +hdbg_init(void) +{ + int i; + for (i = 0; i < ERL_ALC_HDBG_MAX_MBLK-1; i++) + hdbg_mblks[i].next = &hdbg_mblks[i+1]; + hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK-1].next = NULL; + free_hdbg_mblks = &hdbg_mblks[0]; + used_hdbg_mblks = NULL; + erts_mtx_init(&hdbg_mblk_mtx, "erts_alloc_hard_debug"); +} + +static void *check_memory_fence(void *ptr, + Uint *size, + ErtsAlcType_t n, + int func); +void erts_hdbg_chk_blks(void); + +void +erts_hdbg_chk_blks(void) +{ + hdbg_mblk *mblk; + + erts_mtx_lock(&hdbg_mblk_mtx); + for (mblk = used_hdbg_mblks; mblk; mblk = mblk->next) { + Uint sz; + check_memory_fence(mblk->p, &sz, mblk->n, ERTS_ALC_O_CHECK); + ASSERT(sz == mblk->s); + } + erts_mtx_unlock(&hdbg_mblk_mtx); +} + +static hdbg_mblk * +hdbg_alloc(void *p, Uint s, ErtsAlcType_t n) +{ + hdbg_mblk *mblk; + + erts_mtx_lock(&hdbg_mblk_mtx); + mblk = free_hdbg_mblks; + if (!mblk) { + erts_fprintf(stderr, + "Ran out of debug blocks; please increase " + "ERL_ALC_HDBG_MAX_MBLK=%d and recompile!\n", + ERL_ALC_HDBG_MAX_MBLK); + abort(); + } + free_hdbg_mblks = mblk->next; + + mblk->p = p; + mblk->s = s; + mblk->n = n; + + mblk->next = used_hdbg_mblks; + mblk->prev = NULL; + if (used_hdbg_mblks) + used_hdbg_mblks->prev = mblk; + used_hdbg_mblks = mblk; + erts_mtx_unlock(&hdbg_mblk_mtx); + return (void *) mblk; +} + +static void +hdbg_free(hdbg_mblk *mblk) +{ + erts_mtx_lock(&hdbg_mblk_mtx); + if (mblk->next) + mblk->next->prev = mblk->prev; + if (mblk->prev) + mblk->prev->next = mblk->next; + else + used_hdbg_mblks = mblk->next; + + mblk->next = free_hdbg_mblks; + free_hdbg_mblks = mblk; + erts_mtx_unlock(&hdbg_mblk_mtx); +} + +#endif + +#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG +static void *check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func); + +void check_allocated_block( Uint type, void *blk) +{ + Uint dummy; + check_memory_fence(blk, &dummy, ERTS_ALC_T2N(type), ERTS_ALC_O_FREE); +} + +void check_allocators(void) +{ + int i; + if (!erts_initialized) + return; + for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; ++i) { + if (erts_allctrs_info[i].alloc_util) { + ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) erts_allctrs[i].extra; + Allctr_t *allctr = real_af->extra; + Carrier_t *ct; +#ifdef USE_THREADS + if (allctr->thread_safe) + erts_mtx_lock(&allctr->mutex); +#endif + + if (allctr->check_mbc) { + for (ct = allctr->mbc_list.first; ct; ct = ct->next) { + fprintf(stderr,"Checking allocator %d\r\n",i); + allctr->check_mbc(allctr,ct); + } + } +#ifdef USE_THREADS + if (allctr->thread_safe) + erts_mtx_unlock(&allctr->mutex); +#endif + } + } +} +#endif static void * set_memory_fence(void *ptr, Uint sz, ErtsAlcType_t n) { - Uint *ui_ptr; - Uint pattern; + UWord *ui_ptr; + UWord pattern; +#ifdef HARD_DEBUG + hdbg_mblk **mblkpp; +#endif if (!ptr) return NULL; - ui_ptr = (Uint *) ptr; + ui_ptr = (UWord *) ptr; pattern = MK_PATTERN(n); - + +#ifdef HARD_DEBUG + mblkpp = (hdbg_mblk **) ui_ptr++; +#endif + *(ui_ptr++) = sz; *(ui_ptr++) = pattern; - memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(Uint)); + memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(UWord)); + +#ifdef HARD_DEBUG + *mblkpp = hdbg_alloc((void *) ui_ptr, sz, n); +#endif return (void *) ui_ptr; } @@ -2991,16 +3115,22 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) { Uint sz; Uint found_type; - Uint pre_pattern; - Uint post_pattern; - Uint *ui_ptr; + UWord pre_pattern; + UWord post_pattern; + UWord *ui_ptr; +#ifdef HARD_DEBUG + hdbg_mblk *mblk; +#endif if (!ptr) return NULL; - ui_ptr = (Uint *) ptr; + ui_ptr = (UWord *) ptr; pre_pattern = *(--ui_ptr); *size = sz = *(--ui_ptr); +#ifdef HARD_DEBUG + mblk = (hdbg_mblk *) *(--ui_ptr); +#endif found_type = GET_TYPE_OF_PATTERN(pre_pattern); if (pre_pattern != MK_PATTERN(n)) { @@ -3011,7 +3141,7 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) (unsigned long) ptr); } - memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(Uint)); + memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(UWord)); if (post_pattern != MK_PATTERN(n) || pre_pattern != post_pattern) { @@ -3056,6 +3186,17 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) (unsigned long) ptr, (unsigned long) sz, ftype, op_str, otype); } +#ifdef HARD_DEBUG + switch (func) { + case ERTS_ALC_O_REALLOC: + case ERTS_ALC_O_FREE: + hdbg_free(mblk); + break; + default: + break; + } +#endif + return (void *) ui_ptr; } @@ -3068,6 +3209,10 @@ debug_alloc(ErtsAlcType_t n, void *extra, Uint size) Uint dsize; void *res; +#ifdef HARD_DEBUG + erts_hdbg_chk_blks(); +#endif + ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); dsize = size + FENCE_SZ; res = (*real_af->alloc)(n, real_af->extra, dsize); @@ -3097,13 +3242,17 @@ debug_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) dsize = size + FENCE_SZ; dptr = check_memory_fence(ptr, &old_size, n, ERTS_ALC_O_REALLOC); +#ifdef HARD_DEBUG + erts_hdbg_chk_blks(); +#endif + if (old_size > size) sys_memset((void *) (((char *) ptr) + size), 0xf, sizeof(Uint) + old_size - size); res = (*real_af->realloc)(n, real_af->extra, dptr, dsize); - + res = set_memory_fence(res, size, n); #ifdef PRINT_OPS @@ -3133,6 +3282,10 @@ debug_free(ErtsAlcType_t n, void *extra, void *ptr) fprintf(stderr, "free(%s, 0x%lx)\r\n", ERTS_ALC_N2TD(n), (Uint) ptr); #endif +#ifdef HARD_DEBUG + erts_hdbg_chk_blks(); +#endif + } static Uint diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index e7a203002f..3e96c76dbf 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2002-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2002-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -238,7 +238,7 @@ void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size) #ifndef ERTS_CACHE_LINE_SIZE /* Assume a cache line size of 64 bytes */ -# define ERTS_CACHE_LINE_SIZE ((Uint) 64) +# define ERTS_CACHE_LINE_SIZE ((UWord) 64) # define ERTS_CACHE_LINE_MASK (ERTS_CACHE_LINE_SIZE - 1) #endif @@ -486,9 +486,9 @@ init_##NAME##_alloc(void) \ qa_data_##NAME##__ = erts_alloc(ERTS_ALC_T_PRE_ALLOC_DATA,tot_size);\ chunk_start = (((char *) qa_data_##NAME##__) \ + sizeof(erts_sched_pref_quick_alloc_data_t)); \ - if ((((Uint) chunk_start) & ERTS_CACHE_LINE_MASK) != ((Uint) 0)) \ + if ((((UWord) chunk_start) & ERTS_CACHE_LINE_MASK) != ((UWord) 0)) \ chunk_start = ((char *) \ - ((((Uint) chunk_start) & ~ERTS_CACHE_LINE_MASK) \ + ((((UWord) chunk_start) & ~ERTS_CACHE_LINE_MASK) \ + ERTS_CACHE_LINE_SIZE)); \ qa_data_##NAME##__->chunks_mem_size = chunk_mem_size; \ qa_data_##NAME##__->start = (void *) chunk_start; \ @@ -553,7 +553,7 @@ NAME##_free(TYPE *p) \ } #ifdef DEBUG -#define ERTS_ALC_DBG_BLK_SZ(PTR) (*(((Uint *) (PTR)) - 2)) +#define ERTS_ALC_DBG_BLK_SZ(PTR) (*(((UWord *) (PTR)) - 2)) #endif /* #ifdef DEBUG */ #undef ERTS_ALC_INLINE diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index f701f71c7d..7df9f19af0 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -1,19 +1,19 @@ # # %CopyrightBegin% -# -# Copyright Ericsson AB 2003-2009. All Rights Reserved. -# +# +# Copyright Ericsson AB 2003-2010. All Rights Reserved. +# # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in # compliance with the License. You should have received a copy of the # Erlang Public License along with this software. If not, it can be # retrieved online at http://www.erlang.org/. -# +# # Software distributed under the License is distributed on an "AS IS" # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See # the License for the specific language governing rights and limitations # under the License. -# +# # %CopyrightEnd% # @@ -138,6 +138,7 @@ type PEND_SUSPEND SHORT_LIVED PROCESSES pending_suspend type PROC_LIST SHORT_LIVED PROCESSES proc_list type FUN_ENTRY FIXED_SIZE CODE fun_entry type ATOM_TXT LONG_LIVED ATOM atom_text +type BEAM_REGISTER EHEAP PROCESSES beam_register type HEAP EHEAP PROCESSES heap type OLD_HEAP EHEAP PROCESSES old_heap type HEAP_FRAG EHEAP PROCESSES heap_frag @@ -211,7 +212,8 @@ type LOGGER_DSBUF TEMPORARY SYSTEM logger_dsbuf type TMP_DSBUF TEMPORARY SYSTEM tmp_dsbuf type INFO_DSBUF SYSTEM SYSTEM info_dsbuf # INFO_DSBUF have to use the SYSTEM allocator; otherwise, a deadlock might occur -type SCHDLR_DATA LONG_LIVED PROCESSES scheduler_data +type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data +type SCHDLR_SLP_INFO LONG_LIVED SYSTEM scheduler_sleep_info type RUNQS LONG_LIVED SYSTEM run_queues type DDLL_PROCESS STANDARD SYSTEM ddll_processes type DDLL_HANDLE STANDARD SYSTEM ddll_handle @@ -231,6 +233,7 @@ type RE_SUBJECT SHORT_LIVED SYSTEM re_subject type RE_HEAP STANDARD SYSTEM re_heap type RE_STACK SHORT_LIVED SYSTEM re_stack type UNICODE_BUFFER SHORT_LIVED SYSTEM unicode_buffer +type BINARY_BUFFER SHORT_LIVED SYSTEM binary_buffer type PRE_ALLOC_DATA LONG_LIVED SYSTEM pre_alloc_data type DRV_THR_OPTS DRIVER SYSTEM driver_thread_opts type DRV_TID DRIVER SYSTEM driver_tid @@ -244,6 +247,7 @@ type CPUDATA LONG_LIVED SYSTEM cpu_data type TMP_CPU_IDS SHORT_LIVED SYSTEM tmp_cpu_ids type EXT_TERM_DATA SHORT_LIVED PROCESSES external_term_data type ZLIB STANDARD SYSTEM zlib +type RDR_GRPS_MAP LONG_LIVED SYSTEM reader_groups_map +if smp type ASYNC SHORT_LIVED SYSTEM async @@ -267,7 +271,9 @@ type RUNQ_BLNS LONG_LIVED SYSTEM run_queue_balancing +if threads -type ETHR_INTERNAL SYSTEM SYSTEM ethread_internal +type ETHR_STD STANDARD SYSTEM ethread_standard +type ETHR_SL SHORT_LIVED SYSTEM ethread_short_lived +type ETHR_LL LONG_LIVED SYSTEM ethread_long_lived +ifnot smp @@ -322,6 +328,9 @@ type SSB SHORT_LIVED PROCESSES ssb # Types used by system specific code # +type TEMP_TERM TEMPORARY SYSTEM temp_term +type LL_TEMP_TERM LONG_LIVED SYSTEM ll_temp_term + type DRV_TAB LONG_LIVED SYSTEM drv_tab type DRV_EV_STATE LONG_LIVED SYSTEM driver_event_state type DRV_EV_D_STATE FIXED_SIZE SYSTEM driver_event_data_state diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 9b7bc24c1c..8b184899c9 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2002-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2002-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -69,14 +69,14 @@ static int initialized = 0; #if HAVE_ERTS_MSEG -#define INV_MSEG_UNIT_MASK ((Uint) (mseg_unit_size - 1)) +#define INV_MSEG_UNIT_MASK ((UWord) (mseg_unit_size - 1)) #define MSEG_UNIT_MASK (~INV_MSEG_UNIT_MASK) #define MSEG_UNIT_FLOOR(X) ((X) & MSEG_UNIT_MASK) #define MSEG_UNIT_CEILING(X) MSEG_UNIT_FLOOR((X) + INV_MSEG_UNIT_MASK) #endif -#define INV_SYS_ALLOC_CARRIER_MASK ((Uint) (sys_alloc_carrier_size - 1)) +#define INV_SYS_ALLOC_CARRIER_MASK ((UWord) (sys_alloc_carrier_size - 1)) #define SYS_ALLOC_CARRIER_MASK (~INV_SYS_ALLOC_CARRIER_MASK) #define SYS_ALLOC_CARRIER_FLOOR(X) ((X) & SYS_ALLOC_CARRIER_MASK) #define SYS_ALLOC_CARRIER_CEILING(X) \ @@ -85,7 +85,7 @@ static int initialized = 0; #undef ASSERT #define ASSERT ASSERT_EXPR -#define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE ((Uint) 1) +#define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE ((UWord) 1) #if 0 /* Can be useful for debugging */ @@ -114,12 +114,12 @@ static Uint mseg_unit_size; /* Blocks ... */ -#define SBC_BLK_FTR_FLG (((Uint) 1) << 0) -#define UNUSED1_BLK_FTR_FLG (((Uint) 1) << 1) -#define UNUSED2_BLK_FTR_FLG (((Uint) 1) << 2) +#define SBC_BLK_FTR_FLG (((UWord) 1) << 0) +#define UNUSED1_BLK_FTR_FLG (((UWord) 1) << 1) +#define UNUSED2_BLK_FTR_FLG (((UWord) 1) << 2) #define ABLK_HDR_SZ (sizeof(Block_t)) -#define FBLK_FTR_SZ (sizeof(Uint)) +#define FBLK_FTR_SZ (sizeof(UWord)) #define UMEMSZ2BLKSZ(AP, SZ) \ (ABLK_HDR_SZ + (SZ) <= (AP)->min_block_size \ @@ -130,14 +130,14 @@ static Uint mseg_unit_size; #define BLK2UMEM(P) ((void *) (((char *) (P)) + ABLK_HDR_SZ)) #define PREV_BLK_SZ(B) \ - ((Uint) (*(((Uint *) (B)) - 1) & SZ_MASK)) + ((UWord) (*(((UWord *) (B)) - 1) & SZ_MASK)) #define SET_BLK_SZ_FTR(B, SZ) \ - (*((Uint *) (((char *) (B)) + (SZ) - sizeof(Uint))) = (SZ)) + (*((UWord *) (((char *) (B)) + (SZ) - sizeof(UWord))) = (SZ)) -#define THIS_FREE_BLK_HDR_FLG (((Uint) 1) << 0) -#define PREV_FREE_BLK_HDR_FLG (((Uint) 1) << 1) -#define LAST_BLK_HDR_FLG (((Uint) 1) << 2) +#define THIS_FREE_BLK_HDR_FLG (((UWord) 1) << 0) +#define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1) +#define LAST_BLK_HDR_FLG (((UWord) 1) << 2) #define SET_BLK_SZ(B, SZ) \ (ASSERT(((SZ) & FLG_MASK) == 0), \ @@ -156,11 +156,11 @@ static Uint mseg_unit_size; (*((Block_t *) (B)) &= ~LAST_BLK_HDR_FLG) #define SBH_THIS_FREE THIS_FREE_BLK_HDR_FLG -#define SBH_THIS_ALLOCED ((Uint) 0) +#define SBH_THIS_ALLOCED ((UWord) 0) #define SBH_PREV_FREE PREV_FREE_BLK_HDR_FLG -#define SBH_PREV_ALLOCED ((Uint) 0) +#define SBH_PREV_ALLOCED ((UWord) 0) #define SBH_LAST_BLK LAST_BLK_HDR_FLG -#define SBH_NOT_LAST_BLK ((Uint) 0) +#define SBH_NOT_LAST_BLK ((UWord) 0) #define SET_BLK_HDR(B, Sz, F) \ (ASSERT(((Sz) & FLG_MASK) == 0), *((Block_t *) (B)) = ((Sz) | (F))) @@ -200,7 +200,7 @@ static Uint mseg_unit_size; ((FTR) = 0) #define IS_SBC_BLK(B) \ - (IS_PREV_BLK_FREE((B)) && (((Uint *) (B))[-1] & SBC_BLK_FTR_FLG)) + (IS_PREV_BLK_FREE((B)) && (((UWord *) (B))[-1] & SBC_BLK_FTR_FLG)) #define IS_MBC_BLK(B) \ (!IS_SBC_BLK((B))) @@ -211,8 +211,8 @@ static Uint mseg_unit_size; /* Carriers ... */ -#define MSEG_CARRIER_HDR_FLAG (((Uint) 1) << 0) -#define SBC_CARRIER_HDR_FLAG (((Uint) 1) << 1) +#define MSEG_CARRIER_HDR_FLAG (((UWord) 1) << 0) +#define SBC_CARRIER_HDR_FLAG (((UWord) 1) << 1) #define SCH_SYS_ALLOC 0 #define SCH_MSEG MSEG_CARRIER_HDR_FLAG @@ -407,18 +407,18 @@ do { \ /* Debug stuff... */ #ifdef DEBUG -static Uint carrier_alignment; +static UWord carrier_alignment; #define DEBUG_SAVE_ALIGNMENT(C) \ do { \ - Uint algnmnt__ = sizeof(Unit_t) - (((Uint) (C)) % sizeof(Unit_t)); \ + UWord algnmnt__ = sizeof(Unit_t) - (((UWord) (C)) % sizeof(Unit_t)); \ carrier_alignment = MIN(carrier_alignment, algnmnt__); \ - ASSERT(((Uint) (C)) % sizeof(Uint) == 0); \ + ASSERT(((UWord) (C)) % sizeof(UWord) == 0); \ } while (0) #define DEBUG_CHECK_ALIGNMENT(P) \ do { \ - ASSERT(sizeof(Unit_t) - (((Uint) (P)) % sizeof(Unit_t)) \ + ASSERT(sizeof(Unit_t) - (((UWord) (P)) % sizeof(Unit_t)) \ >= carrier_alignment); \ - ASSERT(((Uint) (P)) % sizeof(Uint) == 0); \ + ASSERT(((UWord) (P)) % sizeof(UWord) == 0); \ } while (0) #else @@ -610,7 +610,7 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr) } -static Block_t *create_carrier(Allctr_t *, Uint, Uint); +static Block_t *create_carrier(Allctr_t *, Uint, UWord); static void destroy_carrier(Allctr_t *, Block_t *); /* Multi block carrier alloc/realloc/free ... */ @@ -630,6 +630,11 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp) blk = (*allctr->get_free_block)(allctr, *blk_szp, NULL, 0); +#if HALFWORD_HEAP + if (!blk) { + blk = create_carrier(allctr, *blk_szp, CFLG_MBC|CFLG_FORCE_MSEG); + } +#else if (!blk) { blk = create_carrier(allctr, *blk_szp, CFLG_MBC); if (!blk) { @@ -640,6 +645,7 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp) CFLG_SBC|CFLG_FORCE_SIZE|CFLG_FORCE_SYS_ALLOC); } } +#endif #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG if (IS_MBC_BLK(blk)) { @@ -656,14 +662,14 @@ static ERTS_INLINE void mbc_alloc_finalize(Allctr_t *allctr, Block_t *blk, Uint org_blk_sz, - Uint flags, + UWord flags, Uint want_blk_sz, int valid_blk_info) { Uint blk_sz; Uint nxt_blk_sz; Block_t *nxt_blk; - Uint prev_free_flg = flags & PREV_FREE_BLK_HDR_FLG; + UWord prev_free_flg = flags & PREV_FREE_BLK_HDR_FLG; ASSERT(org_blk_sz >= want_blk_sz); ASSERT(blk); @@ -853,7 +859,7 @@ mbc_free(Allctr_t *allctr, void *p) } static void * -mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs) +mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) { void *new_p; Uint old_blk_sz; @@ -1144,7 +1150,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs) } else { Uint new_blk_sz; - Uint new_blk_flgs; + UWord new_blk_flgs; Uint prev_blk_sz; Uint blk_cpy_sz; @@ -1239,7 +1245,7 @@ do { \ static Block_t * -create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags) +create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) { Block_t *blk; Carrier_t *crr; @@ -1283,8 +1289,8 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags) if (crr_sz < allctr->mbc_header_size + blk_sz) crr_sz = allctr->mbc_header_size + blk_sz; #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (sizeof(Unit_t) == sizeof(Uint)) - crr_sz += sizeof(Uint); + if (sizeof(Unit_t) == sizeof(UWord)) + crr_sz += sizeof(UWord); #endif } crr_sz = MSEG_UNIT_CEILING(crr_sz); @@ -1324,8 +1330,8 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags) && bcrr_sz < allctr->smallest_mbc_size) bcrr_sz = allctr->smallest_mbc_size; #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (sizeof(Unit_t) == sizeof(Uint)) - bcrr_sz += sizeof(Uint); + if (sizeof(Unit_t) == sizeof(UWord)) + bcrr_sz += sizeof(UWord); #endif } @@ -1360,7 +1366,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags) blk = SBC2BLK(allctr, crr); - SET_SBC_BLK_FTR(((Uint *) blk)[-1]); + SET_SBC_BLK_FTR(((UWord *) blk)[-1]); SET_BLK_HDR(blk, blk_sz, SBH_THIS_ALLOCED|SBH_PREV_FREE|SBH_LAST_BLK); link_carrier(&allctr->sbc_list, crr); @@ -1379,13 +1385,13 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags) blk = MBC2FBLK(allctr, crr); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (sizeof(Unit_t) == sizeof(Uint)) - crr_sz -= sizeof(Uint); + if (sizeof(Unit_t) == sizeof(UWord)) + crr_sz -= sizeof(UWord); #endif blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size); - SET_MBC_BLK_FTR(((Uint *) blk)[-1]); + SET_MBC_BLK_FTR(((UWord *) blk)[-1]); SET_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_PREV_FREE|SBH_LAST_BLK); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -1400,13 +1406,13 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags) link_carrier(&allctr->mbc_list, crr); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (sizeof(Unit_t) == sizeof(Uint)) - crr_sz += sizeof(Uint); + if (sizeof(Unit_t) == sizeof(UWord)) + crr_sz += sizeof(UWord); #endif CHECK_1BLK_CARRIER(allctr, 0, is_mseg, crr, crr_sz, blk, blk_sz); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (sizeof(Unit_t) == sizeof(Uint)) - crr_sz -= sizeof(Uint); + if (sizeof(Unit_t) == sizeof(UWord)) + crr_sz -= sizeof(UWord); #endif if (allctr->creating_mbc) (*allctr->creating_mbc)(allctr, crr); @@ -1418,11 +1424,12 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags) } static Block_t * -resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, Uint flags) +resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) { Block_t *new_blk; Carrier_t *new_crr, *old_crr; - Uint create_flags, old_crr_sz, old_blk_sz, new_blk_sz, new_crr_sz; + UWord create_flags; + Uint old_crr_sz, old_blk_sz, new_blk_sz, new_crr_sz; Uint new_bcrr_sz; if (flags & CFLG_MBC) { @@ -2522,7 +2529,12 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) INC_CC(allctr->calls.this_alloc); if (size >= allctr->sbc_threshold) { +#if HALFWORD_HEAP + Block_t *blk = create_carrier(allctr, size, + CFLG_SBC | CFLG_FORCE_MSEG); +#else Block_t *blk = create_carrier(allctr, size, CFLG_SBC); +#endif res = blk ? BLK2UMEM(blk) : NULL; } else @@ -2594,16 +2606,16 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size) Allctr_t *allctr; void *res; - ASSERT(sizeof(Uint) == sizeof(Allctr_t *)); + ASSERT(sizeof(UWord) == sizeof(Allctr_t *)); ASSERT(ix > 0); if (ix >= tspec->size) ix = (ix % (tspec->size - 1)) + 1; allctr = tspec->allctr[ix]; erts_mtx_lock(&allctr->mutex); - res = do_erts_alcu_alloc(type, allctr, size + sizeof(Uint)); + res = do_erts_alcu_alloc(type, allctr, size + sizeof(UWord)); if (res) { *((Allctr_t **) res) = allctr; - res = (void *) (((char *) res) + sizeof(Uint)); + res = (void *) (((char *) res) + sizeof(UWord)); } erts_mtx_unlock(&allctr->mutex); DEBUG_CHECK_ALIGNMENT(res); @@ -2681,7 +2693,7 @@ void erts_alcu_free_thr_pref(ErtsAlcType_t type, void *unused, void *p) { if (p) { - void *ptr = (void *) (((char *) p) - sizeof(Uint)); + void *ptr = (void *) (((char *) p) - sizeof(UWord)); Allctr_t *allctr = *((Allctr_t **) ptr); erts_mtx_lock(&allctr->mutex); do_erts_alcu_free(type, allctr, ptr); @@ -2698,7 +2710,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, void *extra, void *p, Uint size, - Uint flgs) + UWord flgs) { Allctr_t *allctr = (Allctr_t *) extra; Block_t *blk; @@ -2780,13 +2792,21 @@ do_erts_alcu_realloc(ErtsAlcType_t type, Block_t *new_blk; if(IS_SBC_BLK(blk)) { do_carrier_resize: +#if HALFWORD_HEAP + new_blk = resize_carrier(allctr, blk, size, CFLG_SBC | CFLG_FORCE_MSEG); +#else new_blk = resize_carrier(allctr, blk, size, CFLG_SBC); +#endif res = new_blk ? BLK2UMEM(new_blk) : NULL; } else if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) return NULL; else { +#if HALFWORD_HEAP + new_blk = create_carrier(allctr, size, CFLG_SBC | CFLG_FORCE_MSEG); +#else new_blk = create_carrier(allctr, size, CFLG_SBC); +#endif if (new_blk) { res = BLK2UMEM(new_blk); sys_memcpy((void *) res, @@ -2962,7 +2982,7 @@ erts_alcu_realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size) if (!p) return erts_alcu_alloc_thr_pref(type, extra, size); - ptr = (void *) (((char *) p) - sizeof(Uint)); + ptr = (void *) (((char *) p) - sizeof(UWord)); used_allctr = *((Allctr_t **) ptr); ix = erts_alc_get_thr_ix(); @@ -2976,32 +2996,32 @@ erts_alcu_realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size) res = do_erts_alcu_realloc(type, used_allctr, ptr, - size + sizeof(Uint), + size + sizeof(UWord), (pref_allctr != used_allctr ? ERTS_ALCU_FLG_FAIL_REALLOC_MOVE : 0)); erts_mtx_unlock(&used_allctr->mutex); if (res) { ASSERT(used_allctr == *((Allctr_t **) res)); - res = (void *) (((char *) res) + sizeof(Uint)); + res = (void *) (((char *) res) + sizeof(UWord)); DEBUG_CHECK_ALIGNMENT(res); } else { erts_mtx_lock(&pref_allctr->mutex); - res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(Uint)); + res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(UWord)); erts_mtx_unlock(&pref_allctr->mutex); if (res) { Block_t *blk; size_t cpy_size; *((Allctr_t **) res) = pref_allctr; - res = (void *) (((char *) res) + sizeof(Uint)); + res = (void *) (((char *) res) + sizeof(UWord)); DEBUG_CHECK_ALIGNMENT(res); erts_mtx_lock(&used_allctr->mutex); blk = UMEM2BLK(ptr); - cpy_size = BLK_SZ(blk) - ABLK_HDR_SZ - sizeof(Uint); + cpy_size = BLK_SZ(blk) - ABLK_HDR_SZ - sizeof(UWord); if (cpy_size > size) cpy_size = size; sys_memcpy(res, p, cpy_size); @@ -3026,7 +3046,7 @@ erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra, if (!p) return erts_alcu_alloc_thr_pref(type, extra, size); - ptr = (void *) (((char *) p) - sizeof(Uint)); + ptr = (void *) (((char *) p) - sizeof(UWord)); used_allctr = *((Allctr_t **) ptr); ix = erts_alc_get_thr_ix(); @@ -3037,7 +3057,7 @@ erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra, ASSERT(used_allctr && pref_allctr); erts_mtx_lock(&pref_allctr->mutex); - res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(Uint)); + res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(UWord)); if (!res) { erts_mtx_unlock(&pref_allctr->mutex); res = erts_alcu_realloc_thr_pref(type, extra, p, size); @@ -3048,7 +3068,7 @@ erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra, Allctr_t *allctr; *((Allctr_t **) res) = pref_allctr; - res = (void *) (((char *) res) + sizeof(Uint)); + res = (void *) (((char *) res) + sizeof(UWord)); DEBUG_CHECK_ALIGNMENT(res); @@ -3061,7 +3081,7 @@ erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra, } blk = UMEM2BLK(ptr); - cpy_size = BLK_SZ(blk) - ABLK_HDR_SZ - sizeof(Uint); + cpy_size = BLK_SZ(blk) - ABLK_HDR_SZ - sizeof(UWord); if (cpy_size > size) cpy_size = size; sys_memcpy(res, p, cpy_size); @@ -3138,11 +3158,11 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (allctr->min_block_size < ABLK_HDR_SZ) goto error; allctr->min_block_size = UNIT_CEILING(allctr->min_block_size - + sizeof(Uint)); + + sizeof(UWord)); #if HAVE_ERTS_MSEG - if (allctr->mseg_opt.abs_shrink_th > ~((Uint) 0) / 100) - allctr->mseg_opt.abs_shrink_th = ~((Uint) 0) / 100; + if (allctr->mseg_opt.abs_shrink_th > ~((UWord) 0) / 100) + allctr->mseg_opt.abs_shrink_th = ~((UWord) 0) / 100; #endif #ifdef USE_THREADS @@ -3182,15 +3202,15 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->mbc_header_size = (UNIT_CEILING(allctr->mbc_header_size + FBLK_FTR_SZ + ABLK_HDR_SZ - + sizeof(Uint)) + + sizeof(UWord)) - ABLK_HDR_SZ - - sizeof(Uint)); + - sizeof(UWord)); allctr->sbc_header_size = (UNIT_CEILING(sizeof(Carrier_t) + FBLK_FTR_SZ + ABLK_HDR_SZ - + sizeof(Uint)) + + sizeof(UWord)) - ABLK_HDR_SZ - - sizeof(Uint)); + - sizeof(UWord)); } else #endif @@ -3208,12 +3228,21 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (allctr->main_carrier_size) { Block_t *blk; +#if HALFWORD_HEAP + blk = create_carrier(allctr, + allctr->main_carrier_size, + CFLG_MBC + | CFLG_FORCE_SIZE + | CFLG_FORCE_MSEG + | CFLG_MAIN_CARRIER); +#else blk = create_carrier(allctr, allctr->main_carrier_size, CFLG_MBC | CFLG_FORCE_SIZE | CFLG_FORCE_SYS_ALLOC | CFLG_MAIN_CARRIER); +#endif if (!blk) goto error; @@ -3409,7 +3438,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) is_free_blk = (int) IS_FREE_BLK(blk); if(is_free_blk) { if (IS_NOT_LAST_BLK(blk)) - ASSERT(*((Uint *) (((char *) blk)+blk_sz-sizeof(Uint))) + ASSERT(*((UWord *) (((char *) blk)+blk_sz-sizeof(UWord))) == blk_sz); } @@ -3417,7 +3446,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) (*allctr->check_block)(allctr, blk, (int) is_free_blk); if (IS_LAST_BLK(blk)) { - carrier_end = ((char *) NXT_BLK(blk)) + sizeof(Uint); + carrier_end = ((char *) NXT_BLK(blk)) + sizeof(UWord); mbc = *((Carrier_t **) NXT_BLK(blk)); prev_blk = NULL; blk = MBC2FBLK(allctr, mbc); @@ -3433,7 +3462,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) ASSERT((((char *) mbc) + allctr->mbc_header_size + tot_blk_sz - + sizeof(Uint)) == carrier_end); + + sizeof(UWord)) == carrier_end); ASSERT(((char *) mbc) + CARRIER_SZ(mbc) == carrier_end); if (allctr->check_mbc) diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index 10b11661e6..f2b951bca6 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2002-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2002-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -27,8 +27,8 @@ typedef struct Allctr_t_ Allctr_t; typedef struct { - Uint ycs; - Uint mmc; + UWord ycs; + UWord mmc; } AlcUInit_t; typedef struct { @@ -38,22 +38,22 @@ typedef struct { int tspec; int tpref; int ramv; - Uint sbct; - Uint asbcst; - Uint rsbcst; - Uint rsbcmt; - Uint rmbcmt; - Uint mmbcs; - Uint mmsbc; - Uint mmmbc; - Uint lmbcs; - Uint smbcs; - Uint mbcgs; + UWord sbct; + UWord asbcst; + UWord rsbcst; + UWord rsbcmt; + UWord rmbcmt; + UWord mmbcs; + UWord mmsbc; + UWord mmmbc; + UWord lmbcs; + UWord smbcs; + UWord mbcgs; } AllctrInit_t; typedef struct { - Uint blocks; - Uint carriers; + UWord blocks; + UWord carriers; } AllctrSize_t; #ifndef SMALL_MEMORY @@ -163,19 +163,19 @@ void erts_alcu_current_size(Allctr_t *, AllctrSize_t *); #define CEILING(X, I) ((((X) - 1)/(I) + 1)*(I)) #undef WORD_MASK -#define INV_WORD_MASK ((Uint) (sizeof(Uint) - 1)) +#define INV_WORD_MASK ((UWord) (sizeof(UWord) - 1)) #define WORD_MASK (~INV_WORD_MASK) #define WORD_FLOOR(X) ((X) & WORD_MASK) #define WORD_CEILING(X) WORD_FLOOR((X) + INV_WORD_MASK) #undef UNIT_MASK -#define INV_UNIT_MASK ((Uint) (sizeof(Unit_t) - 1)) +#define INV_UNIT_MASK ((UWord) (sizeof(Unit_t) - 1)) #define UNIT_MASK (~INV_UNIT_MASK) #define UNIT_FLOOR(X) ((X) & UNIT_MASK) #define UNIT_CEILING(X) UNIT_FLOOR((X) + INV_UNIT_MASK) -#define SZ_MASK (~((Uint) 0) << 3) +#define SZ_MASK (~((UWord) 0) << 3) #define FLG_MASK (~(SZ_MASK)) @@ -189,7 +189,7 @@ typedef union {char c[8]; long l; double d;} Unit_t; typedef struct Carrier_t_ Carrier_t; struct Carrier_t_ { - Uint chdr; + UWord chdr; Carrier_t *next; Carrier_t *prev; }; @@ -199,17 +199,17 @@ typedef struct { Carrier_t *last; } CarrierList_t; -typedef Uint Block_t; -typedef Uint FreeBlkFtr_t; +typedef UWord Block_t; +typedef UWord FreeBlkFtr_t; typedef struct { - Uint giga_no; - Uint no; + UWord giga_no; + UWord no; } CallCounter_t; typedef struct { - Uint no; - Uint size; + UWord no; + UWord size; } StatValues_t; typedef struct { diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c index 126ec7cc73..64fad9fe0e 100644 --- a/erts/emulator/beam/erl_arith.c +++ b/erts/emulator/beam/erl_arith.c @@ -41,6 +41,16 @@ # define MAX(x, y) (((x) > (y)) ? (x) : (y)) #endif +#if !HEAP_ON_C_STACK +# define DECLARE_TMP(VariableName,N,P) \ + Eterm *VariableName = ((ERTS_PROC_GET_SCHDATA(P)->erl_arith_tmp_heap) + (2 * N)) +#else +# define DECLARE_TMP(VariableName,N,P) \ + Eterm VariableName[2] +#endif +# define ARG_IS_NOT_TMP(Arg,Tmp) ((Arg) != make_big((Tmp))) + + static Eterm shift(Process* p, Eterm arg1, Eterm arg2, int right); static ERTS_INLINE void maybe_shrink(Process* p, Eterm* hp, Eterm res, Uint alloc) @@ -169,7 +179,7 @@ shift(Process* p, Eterm arg1, Eterm arg2, int right) { Sint i; Sint ires; - Eterm tmp_big1[2]; + DECLARE_TMP(tmp_big1,0,p); Eterm* bigp; Uint need; @@ -312,8 +322,8 @@ BIF_RETTYPE bnot_1(BIF_ALIST_1) Eterm erts_mixed_plus(Process* p, Eterm arg1, Eterm arg2) { - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); Eterm res; Eterm hdr; FloatDef f1, f2; @@ -458,8 +468,8 @@ erts_mixed_plus(Process* p, Eterm arg1, Eterm arg2) Eterm erts_mixed_minus(Process* p, Eterm arg1, Eterm arg2) { - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); Eterm hdr; Eterm res; FloatDef f1, f2; @@ -602,8 +612,8 @@ erts_mixed_minus(Process* p, Eterm arg1, Eterm arg2) Eterm erts_mixed_times(Process* p, Eterm arg1, Eterm arg2) { - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); Eterm hdr; Eterm res; FloatDef f1, f2; @@ -627,8 +637,8 @@ erts_mixed_times(Process* p, Eterm arg1, Eterm arg2) } else if (arg2 == SMALL_ONE) { return(arg1); } else { - Eterm big_res[3]; - + DeclareTmpHeap(big_res,3,p); + UseTmpHeap(3,p); /* * The following code is optimized for the case that * result is small (which should be the most common case @@ -636,6 +646,7 @@ erts_mixed_times(Process* p, Eterm arg1, Eterm arg2) */ res = small_times(signed_val(arg1), signed_val(arg2), big_res); if (is_small(res)) { + UnUseTmpHeap(3,p); return res; } else { /* @@ -657,6 +668,7 @@ erts_mixed_times(Process* p, Eterm arg1, Eterm arg2) if (arity > 1) { *hp = big_res[2]; } + UnUseTmpHeap(3,p); return res; } } @@ -915,8 +927,8 @@ erts_mixed_div(Process* p, Eterm arg1, Eterm arg2) Eterm erts_int_div(Process* p, Eterm arg1, Eterm arg2) { - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); int ires; switch (NUMBER_CODE(arg1, arg2)) { @@ -967,8 +979,8 @@ erts_int_div(Process* p, Eterm arg1, Eterm arg2) Eterm erts_int_rem(Process* p, Eterm arg1, Eterm arg2) { - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); int ires; switch (NUMBER_CODE(arg1, arg2)) { @@ -979,7 +991,8 @@ erts_int_rem(Process* p, Eterm arg1, Eterm arg2) if (arg1 != make_small(MIN_SMALL)) { return arg1; } else { - Eterm tmp = small_to_big(signed_val(arg1), tmp_big1); + Eterm tmp; + tmp = small_to_big(signed_val(arg1), tmp_big1); if ((ires = big_ucomp(tmp, arg2)) == 0) { return SMALL_ZERO; } else { @@ -1013,8 +1026,8 @@ erts_int_rem(Process* p, Eterm arg1, Eterm arg2) Eterm erts_band(Process* p, Eterm arg1, Eterm arg2) { - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); Eterm* hp; int need; @@ -1041,8 +1054,8 @@ Eterm erts_band(Process* p, Eterm arg1, Eterm arg2) Eterm erts_bor(Process* p, Eterm arg1, Eterm arg2) { - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); Eterm* hp; int need; @@ -1069,8 +1082,8 @@ Eterm erts_bor(Process* p, Eterm arg1, Eterm arg2) Eterm erts_bxor(Process* p, Eterm arg1, Eterm arg2) { - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); Eterm* hp; int need; @@ -1148,8 +1161,8 @@ erts_gc_mixed_plus(Process* p, Eterm* reg, Uint live) { Eterm arg1; Eterm arg2; - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); Eterm res; Eterm hdr; FloatDef f1, f2; @@ -1237,10 +1250,10 @@ erts_gc_mixed_plus(Process* p, Eterm* reg, Uint live) need_heap = BIG_NEED_SIZE(sz); if (ERTS_NEED_GC(p, need_heap)) { erts_garbage_collect(p, need_heap, reg, live+2); - if (arg1 != make_big(tmp_big1)) { + if (ARG_IS_NOT_TMP(arg1,tmp_big1)) { arg1 = reg[live]; } - if (arg2 != make_big(tmp_big2)) { + if (ARG_IS_NOT_TMP(arg2,tmp_big2)) { arg2 = reg[live+1]; } } @@ -1316,8 +1329,8 @@ erts_gc_mixed_minus(Process* p, Eterm* reg, Uint live) { Eterm arg1; Eterm arg2; - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); Eterm hdr; Eterm res; FloatDef f1, f2; @@ -1394,10 +1407,10 @@ erts_gc_mixed_minus(Process* p, Eterm* reg, Uint live) need_heap = BIG_NEED_SIZE(sz); if (ERTS_NEED_GC(p, need_heap)) { erts_garbage_collect(p, need_heap, reg, live+2); - if (arg1 != make_big(tmp_big1)) { + if (ARG_IS_NOT_TMP(arg1,tmp_big1)) { arg1 = reg[live]; } - if (arg2 != make_big(tmp_big2)) { + if (ARG_IS_NOT_TMP(arg2,tmp_big2)) { arg2 = reg[live+1]; } } @@ -1482,8 +1495,8 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live) { Eterm arg1; Eterm arg2; - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); Eterm hdr; Eterm res; FloatDef f1, f2; @@ -1509,7 +1522,8 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live) } else if (arg2 == SMALL_ONE) { return(arg1); } else { - Eterm big_res[3]; + DeclareTmpHeap(big_res,3,p); + UseTmpHeap(3,p); /* * The following code is optimized for the case that @@ -1519,6 +1533,7 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live) res = small_times(signed_val(arg1), signed_val(arg2), big_res); if (is_small(res)) { + UnUseTmpHeap(3,p); return res; } else { /* @@ -1546,6 +1561,7 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live) if (arity > 1) { *hp = big_res[2]; } + UnUseTmpHeap(3,p); return res; } } @@ -1609,17 +1625,17 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live) need_heap = BIG_NEED_SIZE(sz); if (ERTS_NEED_GC(p, need_heap)) { erts_garbage_collect(p, need_heap, reg, live+2); - if (arg1 != make_big(tmp_big1)) { + if (ARG_IS_NOT_TMP(arg1,tmp_big1)) { arg1 = reg[live]; } - if (arg2 != make_big(tmp_big2)) { + if (ARG_IS_NOT_TMP(arg2,tmp_big2)) { arg2 = reg[live+1]; } } hp = p->htop; p->htop += need_heap; res = big_times(arg1, arg2, hp); - trim_heap(p, hp, res); + trim_heap(p, hp, res); /* * Note that the result must be big in this case, since @@ -1828,8 +1844,8 @@ erts_gc_int_div(Process* p, Eterm* reg, Uint live) { Eterm arg1; Eterm arg2; - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); int ires; arg1 = reg[live]; @@ -1866,10 +1882,10 @@ erts_gc_int_div(Process* p, Eterm* reg, Uint live) need = BIG_NEED_SIZE(i-ires+1) + BIG_NEED_SIZE(i); if (ERTS_NEED_GC(p, need)) { erts_garbage_collect(p, need, reg, live+2); - if (arg1 != make_big(tmp_big1)) { + if (ARG_IS_NOT_TMP(arg1,tmp_big1)) { arg1 = reg[live]; } - if (arg2 != make_big(tmp_big2)) { + if (ARG_IS_NOT_TMP(arg2,tmp_big2)) { arg2 = reg[live+1]; } } @@ -1894,8 +1910,8 @@ erts_gc_int_rem(Process* p, Eterm* reg, Uint live) { Eterm arg1; Eterm arg2; - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); int ires; arg1 = reg[live]; @@ -1908,7 +1924,8 @@ erts_gc_int_rem(Process* p, Eterm* reg, Uint live) if (arg1 != make_small(MIN_SMALL)) { return arg1; } else { - Eterm tmp = small_to_big(signed_val(arg1), tmp_big1); + Eterm tmp; + tmp = small_to_big(signed_val(arg1), tmp_big1); if ((ires = big_ucomp(tmp, arg2)) == 0) { return SMALL_ZERO; } else { @@ -1928,10 +1945,10 @@ erts_gc_int_rem(Process* p, Eterm* reg, Uint live) if (ERTS_NEED_GC(p, need)) { erts_garbage_collect(p, need, reg, live+2); - if (arg1 != make_big(tmp_big1)) { + if (ARG_IS_NOT_TMP(arg1,tmp_big1)) { arg1 = reg[live]; } - if (arg2 != make_big(tmp_big2)) { + if (ARG_IS_NOT_TMP(arg2,tmp_big2)) { arg2 = reg[live+1]; } } @@ -1956,8 +1973,8 @@ Eterm erts_gc_##func(Process* p, Eterm* reg, Uint live) \ { \ Eterm arg1; \ Eterm arg2; \ - Eterm tmp_big1[2]; \ - Eterm tmp_big2[2]; \ + DECLARE_TMP(tmp_big1,0,p); \ + DECLARE_TMP(tmp_big2,1,p); \ Eterm* hp; \ int need; \ \ diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index b090564649..12c7631448 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2000-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ #ifdef HAVE_CONFIG_H @@ -70,7 +70,6 @@ static ErlAsync* async_ready_list = NULL; /* Detach from driver */ static void async_detach(DE_Handle* dh) { - /* XXX:PaN what should happen here? we want to unload the driver or??? */ return; } @@ -176,7 +175,6 @@ int exit_async() static void async_add(ErlAsync* a, AsyncQueue* q) { - /* XXX:PaN Is this still necessary when ports lock drivers? */ if (is_internal_port(a->port)) { ERTS_LC_ASSERT(erts_drvportid2port(a->port)); /* make sure the driver will stay around */ diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c new file mode 100644 index 0000000000..024ff2a684 --- /dev/null +++ b/erts/emulator/beam/erl_bif_binary.c @@ -0,0 +1,2931 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * NOTE: This file contains the BIF's for the *module* binary in stdlib. + * other BIF's concerning binaries are in binary.c. + */ + + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "erl_vm.h" +#include "global.h" +#include "erl_process.h" +#include "error.h" +#include "bif.h" +#include "big.h" +#include "erl_binary.h" +#include "erl_bits.h" + + +/* + * The native implementation functions for the module binary. + * Searching is implemented using either Boyer-Moore or Aho-Corasick + * depending on number of searchstrings (BM if one, AC if more than one). + * Native implementation is mostly for efficiency, nothing + * (except binary:referenced_byte_size) really *needs* to be implemented + * in native code. + */ + +/* #define HARDDEBUG */ + +/* Init and local variables */ + +static Export binary_match_trap_export; +static BIF_RETTYPE binary_match_trap(BIF_ALIST_3); +static Export binary_matches_trap_export; +static BIF_RETTYPE binary_matches_trap(BIF_ALIST_3); +static Export binary_longest_prefix_trap_export; +static BIF_RETTYPE binary_longest_prefix_trap(BIF_ALIST_3); +static Export binary_longest_suffix_trap_export; +static BIF_RETTYPE binary_longest_suffix_trap(BIF_ALIST_3); +static Export binary_bin_to_list_trap_export; +static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3); +static Export binary_copy_trap_export; +static BIF_RETTYPE binary_copy_trap(BIF_ALIST_2); +static Uint max_loop_limit; + + +void erts_init_bif_binary(void) +{ + sys_memset((void *) &binary_match_trap_export, 0, sizeof(Export)); + binary_match_trap_export.address = &binary_match_trap_export.code[3]; + binary_match_trap_export.code[0] = am_erlang; + binary_match_trap_export.code[1] = am_binary_match_trap; + binary_match_trap_export.code[2] = 3; + binary_match_trap_export.code[3] = (BeamInstr) em_apply_bif; + binary_match_trap_export.code[4] = (BeamInstr) &binary_match_trap; + + sys_memset((void *) &binary_matches_trap_export, 0, sizeof(Export)); + binary_matches_trap_export.address = &binary_matches_trap_export.code[3]; + binary_matches_trap_export.code[0] = am_erlang; + binary_matches_trap_export.code[1] = am_binary_matches_trap; + binary_matches_trap_export.code[2] = 3; + binary_matches_trap_export.code[3] = (BeamInstr) em_apply_bif; + binary_matches_trap_export.code[4] = (BeamInstr) &binary_matches_trap; + + sys_memset((void *) &binary_longest_prefix_trap_export, 0, sizeof(Export)); + binary_longest_prefix_trap_export.address = &binary_longest_prefix_trap_export.code[3]; + binary_longest_prefix_trap_export.code[0] = am_erlang; + binary_longest_prefix_trap_export.code[1] = am_binary_longest_prefix_trap; + binary_longest_prefix_trap_export.code[2] = 3; + binary_longest_prefix_trap_export.code[3] = (BeamInstr) em_apply_bif; + binary_longest_prefix_trap_export.code[4] = (BeamInstr) &binary_longest_prefix_trap; + + sys_memset((void *) &binary_longest_suffix_trap_export, 0, sizeof(Export)); + binary_longest_suffix_trap_export.address = &binary_longest_suffix_trap_export.code[3]; + binary_longest_suffix_trap_export.code[0] = am_erlang; + binary_longest_suffix_trap_export.code[1] = am_binary_longest_suffix_trap; + binary_longest_suffix_trap_export.code[2] = 3; + binary_longest_suffix_trap_export.code[3] = (BeamInstr) em_apply_bif; + binary_longest_suffix_trap_export.code[4] = (BeamInstr) &binary_longest_suffix_trap; + + sys_memset((void *) &binary_bin_to_list_trap_export, 0, sizeof(Export)); + binary_bin_to_list_trap_export.address = &binary_bin_to_list_trap_export.code[3]; + binary_bin_to_list_trap_export.code[0] = am_erlang; + binary_bin_to_list_trap_export.code[1] = am_binary_bin_to_list_trap; + binary_bin_to_list_trap_export.code[2] = 3; + binary_bin_to_list_trap_export.code[3] = (BeamInstr) em_apply_bif; + binary_bin_to_list_trap_export.code[4] = (BeamInstr) &binary_bin_to_list_trap; + sys_memset((void *) &binary_copy_trap_export, 0, sizeof(Export)); + binary_copy_trap_export.address = &binary_copy_trap_export.code[3]; + binary_copy_trap_export.code[0] = am_erlang; + binary_copy_trap_export.code[1] = am_binary_copy_trap; + binary_copy_trap_export.code[2] = 2; + binary_copy_trap_export.code[3] = (BeamInstr) em_apply_bif; + binary_copy_trap_export.code[4] = (BeamInstr) &binary_copy_trap; + + max_loop_limit = 0; + return; +} + +/* + * Setting the loop_limit for searches for debugging + */ +Sint erts_binary_set_loop_limit(Sint limit) +{ + Sint save = (Sint) max_loop_limit; + if (limit <= 0) { + max_loop_limit = 0; + } else { + max_loop_limit = (Uint) limit; + } + + return save; +} + +static Uint get_reds(Process *p, int loop_factor) +{ + Uint reds = ERTS_BIF_REDS_LEFT(p) * loop_factor; + Uint tmp = max_loop_limit; + if (tmp != 0 && tmp < reds) { + return tmp; + } + if (!reds) { + reds = 1; + } + return reds; +} + +/* + * A micro allocator used when building search structures, just a convenience + * for building structures inside a pre-allocated magic binary using + * conventional malloc-like interface. + */ + +#define MYALIGN(Size) (SIZEOF_VOID_P * (((Size) / SIZEOF_VOID_P) + \ + !!(((Size) % SIZEOF_VOID_P)))) + +#ifdef DEBUG +#define CHECK_ALLOCATOR(My) ASSERT((My).current <= ((My).mem + (My).size)) +#else +#define CHECK_ALLOCATOR(My) /* nothing */ +#endif + +typedef struct _my_allocator { + Uint size; + byte *current; + byte *mem; +} MyAllocator; + +static void init_my_allocator(MyAllocator *my, Uint siz, byte *array) +{ + ASSERT((siz % SIZEOF_VOID_P) == 0); + my->size = siz; + my->mem = array; + my->current = my->mem; +} + +static void *my_alloc(MyAllocator *my, Uint size) +{ + void *ptr = my->current; + my->current += MYALIGN(size); + return ptr; +} + +/* + * The search functionality. + * + * The search is byte oriented, which works nicely for UTF-8 as well as + * latin1 data + */ + +#define ALPHABET_SIZE 256 + +typedef struct _ac_node { +#ifdef HARDDEBUG + Uint32 id; /* To identify h pointer targets when + dumping */ +#endif + Uint32 d; /* Depth in trie, also represents the + length (-1) of the matched string if + in final set */ + Sint32 final; /* Members in final set represent + * matches. + * The set representation is scattered + * among the nodes in this way: + * >0 -> this represents a member of + * the final set, <0 -> member of + * final set somewhere in the failure + * chain, + * 0 -> not member of the final set */ + struct _ac_node *h; /* h(Hode) is the failure function */ + struct _ac_node *g[ALPHABET_SIZE]; /* g(Node,Character) is the + transition function */ +} ACNode; + +typedef struct _ac_trie { +#ifdef HARDDEBUG + Uint32 idc; +#endif + Uint32 counter; /* Number of added patterns */ + ACNode *root; /* pointer to the root state */ +} ACTrie; + +typedef struct _bm_data { + byte *x; + Sint len; + Sint *goodshift; + Sint badshift[ALPHABET_SIZE]; +} BMData; + +#ifdef HARDDEBUG +static void dump_bm_data(BMData *bm); +static void dump_ac_trie(ACTrie *act); +static void dump_ac_node(ACNode *node, int indent, int ch); +#endif + +/* + * The needed size of binary data for a search structure - given the + * accumulated string lengths. + */ +#define BM_SIZE(StrLen) /* StrLen: length of searchstring */ \ +((MYALIGN(sizeof(Sint) * (StrLen))) + /* goodshift array */ \ + MYALIGN(StrLen) + /* searchstring saved */ \ + (MYALIGN(sizeof(BMData)))) /* Structure */ + +#define AC_SIZE(StrLens) /* StrLens: sum of all searchstring lengths */ \ +((MYALIGN(sizeof(ACNode)) * \ +((StrLens)+1)) + /* The actual nodes (including rootnode) */ \ + MYALIGN(sizeof(ACTrie))) /* Structure */ + + +#ifndef MAX +#define MAX(A,B) (((A) > (B)) ? (A) : (B)) +#endif + +#ifndef MIN +#define MIN(A,B) (((A) > (B)) ? (B) : (A)) +#endif +/* + * Callback for the magic binary + */ +static void cleanup_my_data_ac(Binary *bp) +{ + return; +} +static void cleanup_my_data_bm(Binary *bp) +{ + return; +} + +/* + * Initiate a (allocated) micro allocator and fill in the base + * for an Aho-Corasick search trie, given the accumulated length of the search + * strings. + */ +static ACTrie *create_acdata(MyAllocator *my, Uint len, + ACNode ***qbuff /* out */, + Binary **the_bin /* out */) +{ + Uint datasize = AC_SIZE(len); + ACTrie *act; + ACNode *acn; + Binary *mb = erts_create_magic_binary(datasize,cleanup_my_data_ac); + byte *data = ERTS_MAGIC_BIN_DATA(mb); + + init_my_allocator(my, datasize, data); + act = my_alloc(my, sizeof(ACTrie)); /* Important that this is the first + allocation */ + act->counter = 0; + act->root = acn = my_alloc(my, sizeof(ACNode)); + acn->d = 0; + acn->final = 0; + acn->h = NULL; + memset(acn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE); +#ifdef HARDDEBUG + act->idc = 0; + acn->id = 0; +#endif + *qbuff = erts_alloc(ERTS_ALC_T_TMP, sizeof(ACNode *) * len); + *the_bin = mb; + return act; +} + +/* + * The same initialization of allocator and basic data for Boyer-Moore. + */ +static BMData *create_bmdata(MyAllocator *my, byte *x, Uint len, + Binary **the_bin /* out */) +{ + Uint datasize = BM_SIZE(len); + BMData *bmd; + Binary *mb = erts_create_magic_binary(datasize,cleanup_my_data_bm); + byte *data = ERTS_MAGIC_BIN_DATA(mb); + init_my_allocator(my, datasize, data); + bmd = my_alloc(my, sizeof(BMData)); + bmd->x = my_alloc(my,len); + memcpy(bmd->x,x,len); + bmd->len = len; + bmd->goodshift = my_alloc(my,sizeof(Uint) * len); + *the_bin = mb; + return bmd; +} + +/* + * Compilation of search structures + */ + +/* + * Aho Corasick - Build a Trie and fill in the failure functions + * when all strings are added. + * The algorithm is nicely described by Dieter B�hler of University of + * T�bingen: + * http://www-sr.informatik.uni-tuebingen.de/~buehler/AC/AC.html + */ + +/* + * Helper called once for each search pattern + */ +static void ac_add_one_pattern(MyAllocator *my, ACTrie *act, byte *x, Uint len) +{ + ACNode *acn = act->root; + Uint32 n = ++act->counter; /* Always increase counter, even if it's a + duplicate as this may identify the pattern + in the final set (not in current interface + though) */ + Uint i = 0; + + while(i < len) { + if (acn->g[x[i]] != NULL) { + /* node exists, continue */ + acn = acn->g[x[i]]; + ++i; + } else { + /* allocate a new node */ + ACNode *nn = my_alloc(my,sizeof(ACNode)); +#ifdef HARDDEBUG + nn->id = ++(act->idc); +#endif + nn->d = i+1; + nn->h = act->root; + nn->final = 0; + memset(nn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE); + acn->g[x[i]] = nn; + ++i; + acn = nn; + } + } + if (acn->final == 0) { /* New pattern, add to final set */ + acn->final = n; + } +} + +/* + * Called when all search patterns are added. + */ +static void ac_compute_failure_functions(ACTrie *act, ACNode **qbuff) +{ + ACNode *root = act->root; + ACNode *parent; + int i; + int qh = 0,qt = 0; + ACNode *child, *r; + + /* Set all children of the root to have the root as failure function */ + for (i = 0; i < ALPHABET_SIZE; ++i) { + if (root->g[i] != NULL) { + root->g[i]->h = root; + /* Add to que for later traversal */ + qbuff[qt++] = root->g[i]; + } + } + + /* So, now we've handled children of the root state, traverse the + rest of the trie BF... */ + while (qh < qt) { + parent = qbuff[qh++]; + for (i = 0; i < ALPHABET_SIZE; ++ i) { + if ((child = parent->g[i]) != NULL) { + /* Visit this node to */ + qbuff[qt++] = child; + /* Search for correct failure function, follow the parent's + failure function until you find a similar transition + funtion to this child's */ + r = parent->h; + while (r != NULL && r->g[i] == NULL) { + r = r->h; + } + if (r == NULL) { + /* Replace NULL failures with the root as we go */ + child->h = (root->g[i] == NULL) ? root : root->g[i]; + } else { + child->h = r->g[i]; + /* + * The "final" set is scattered among the nodes. When + * the failure function points to a member of the final + * set, we have a match, but we might not see it in the + * current node if we dont mark it as a special type of + * final, i.e. foolow the failure function and you will + * find a real member of final set. This is marked with + * a negative string id and only done if this node does + * not represent a member in the final set. + */ + if (!(child->final) && (child->h->final)) { + child->final = -1; + } + } + } + } + } + /* Finally the failure function of the root should point to itself */ + root->h = root; +} + + +/* + * The actual searching for needles in the haystack... + * Find first match using Aho-Coracick Trie + * return pattern number and fill in mpos + mlen if found, otherwise return 0 + * Return the matching pattern that *starts* first, and ends + * last (difference when overlapping), hence the candidate thing. + * Basic AC finds the first end before the first start... + * + */ +typedef struct { + ACNode *q; + Uint pos; + Uint len; + ACNode *candidate; + Uint candidate_start; +} ACFindFirstState; + + +static void ac_init_find_first_match(ACFindFirstState *state, ACTrie *act, Sint startpos, Uint len) +{ + state->q = act->root; + state->pos = startpos; + state->len = len; + state->candidate = NULL; + state->candidate_start = 0; +} +#define AC_OK 0 +#define AC_NOT_FOUND -1 +#define AC_RESTART -2 + +#define AC_LOOP_FACTOR 10 + +static int ac_find_first_match(ACFindFirstState *state, byte *haystack, + Uint *mpos, Uint *mlen, Uint *reductions) +{ + ACNode *q = state->q; + Uint i = state->pos; + ACNode *candidate = state->candidate, *r; + Uint len = state->len; + Uint candidate_start = state->candidate_start; + Uint rstart; + register Uint reds = *reductions; + + while (i < len) { + if (--reds == 0) { + state->q = q; + state->pos = i; + state->len = len; + state->candidate = candidate; + state->candidate_start = candidate_start; + return AC_RESTART; + } + + while (q->g[haystack[i]] == NULL && q->h != q) { + q = q->h; + } + if (q->g[haystack[i]] != NULL) { + q = q->g[haystack[i]]; + } +#ifdef HARDDEBUG + erts_printf("ch = %c, Current: %u\n", (int) haystack[i], (unsigned) q->id); +#endif + ++i; + if (candidate != NULL && (i - q->d) > candidate_start) { + break; + } + if (q->final) { + r = q; + while (r->final < 0) + r = r->h; + rstart = i - r->d; + if (candidate == NULL || rstart < candidate_start || + (rstart == candidate_start && candidate->d < q->d)) { + candidate_start = rstart; + candidate = r; + } + } + } + *reductions = reds; + if (!candidate) { + return AC_NOT_FOUND; + } +#ifdef HARDDEBUG + dump_ac_node(candidate,0,'?'); +#endif + *mpos = candidate_start; + *mlen = candidate->d; + return AC_OK; +} + +typedef struct _findall_data { + Uint pos; + Uint len; +#ifdef HARDDEBUG + Uint id; +#endif + Eterm epos; + Eterm elen; +} FindallData; + +typedef struct { + ACNode *q; + Uint pos; + Uint len; + Uint m; + Uint allocated; + FindallData *out; +} ACFindAllState; + +static void ac_init_find_all(ACFindAllState *state, ACTrie *act, Sint startpos, Uint len) +{ + state->q = act->root; + state->pos = startpos; + state->len = len; + state->m = 0; + state->allocated = 0; + state->out = NULL; +} + +static void ac_restore_find_all(ACFindAllState *state, char *buff) +{ + memcpy(state,buff,sizeof(ACFindAllState)); + state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * (state->allocated)); + memcpy(state->out,buff+sizeof(ACFindAllState),sizeof(FindallData)*state->m); +} + +static void ac_serialize_find_all(ACFindAllState *state, char *buff) +{ + memcpy(buff,state,sizeof(ACFindAllState)); + memcpy(buff+sizeof(ACFindAllState),state->out,sizeof(FindallData)*state->m); +} + +static void ac_clean_find_all(ACFindAllState *state) +{ + if (state->out != NULL) { + erts_free(ERTS_ALC_T_TMP, state->out); + } +#ifdef HARDDEBUG + state->out = NULL; + state->allocated = 0; +#endif +} + +#define SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(S) \ + (sizeof(ACFindAllState)+(sizeof(FindallData)*(S).m)) + +/* + * Differs to the find_first function in that it stores all matches and the values + * arte returned only in the state. + */ +static int ac_find_all_non_overlapping(ACFindAllState *state, byte *haystack, + Uint *reductions) +{ + ACNode *q = state->q; + Uint i = state->pos; + Uint rstart; + ACNode *r; + Uint len = state->len; + Uint m = state->m, save_m; + Uint allocated = state->allocated; + FindallData *out = state->out; + register Uint reds = *reductions; + + + while (i < len) { + if (--reds == 0) { + state->q = q; + state->pos = i; + state->len = len; + state->m = m; + state->allocated = allocated; + state->out = out; + return AC_RESTART; + } + while (q->g[haystack[i]] == NULL && q->h != q) { + q = q->h; + } + if (q->g[haystack[i]] != NULL) { + q = q->g[haystack[i]]; + } + ++i; + if (q->final) { + r = q; + while (r->final) { + while (r->final < 0) + r = r->h; +#ifdef HARDDEBUG + erts_printf("Trying to add %u\n",(unsigned) r->final); +#endif + rstart = i - r->d; + save_m = m; + while (m > 0 && (out[m-1].pos > rstart || + (out[m-1].pos == rstart && + out[m-1].len < r->d))) { +#ifdef HARDDEBUG + erts_printf("Popping %u\n",(unsigned) out[m-1].id); +#endif + --m; + } +#ifdef HARDDEBUG + if (m > 0) { + erts_printf("Pos %u\n",out[m-1].pos); + erts_printf("Len %u\n",out[m-1].len); + } + erts_printf("Rstart %u\n",rstart); +#endif + if (m == 0 || out[m-1].pos + out[m-1].len <= rstart) { + if (m >= allocated) { + if (!allocated) { + allocated = 10; + out = erts_alloc(ERTS_ALC_T_TMP, + sizeof(FindallData) * allocated); + } else { + allocated *= 2; + out = erts_realloc(ERTS_ALC_T_TMP, out, + sizeof(FindallData) * + allocated); + } + } + out[m].pos = rstart; + out[m].len = r->d; +#ifdef HARDDEBUG + out[m].id = r->final; +#endif + ++m; +#ifdef HARDDEBUG + erts_printf("Pushing %u\n",(unsigned) out[m-1].id); +#endif + } else { +#ifdef HARDDEBUG + erts_printf("Backtracking %d steps\n",save_m - m); +#endif + m = save_m; + } + r = r->h; + } + } + } + *reductions = reds; + state->m = m; + state->out = out; + return (m == 0) ? AC_NOT_FOUND : AC_OK; +} + +/* + * Boyer Moore - most obviously implemented more or less exactly as + * Christian Charras and Thierry Lecroq describe it in "Handbook of + * Exact String-Matching Algorithms" + * http://www-igm.univ-mlv.fr/~lecroq/string/ + */ + +/* + * Call this to compute badshifts array + */ +static void compute_badshifts(BMData *bmd) +{ + Sint i; + Sint m = bmd->len; + + for (i = 0; i < ALPHABET_SIZE; ++i) { + bmd->badshift[i] = m; + } + for (i = 0; i < m - 1; ++i) { + bmd->badshift[bmd->x[i]] = m - i - 1; + } +} + +/* Helper for "compute_goodshifts" */ +static void compute_suffixes(byte *x, Sint m, Sint *suffixes) +{ + int f,g,i; + + suffixes[m - 1] = m; + + f = 0; /* To avoid use before set warning */ + + g = m - 1; + + for (i = m - 2; i >= 0; --i) { + if (i > g && suffixes[i + m - 1 - f] < i - g) { + suffixes[i] = suffixes[i + m - 1 - f]; + } else { + if (i < g) { + g = i; + } + f = i; + while ( g >= 0 && x[g] == x[g + m - 1 - f] ) { + --g; + } + suffixes[i] = f - g; + } + } +} + +/* + * Call this to compute goodshift array + */ +static void compute_goodshifts(BMData *bmd) +{ + Sint m = bmd->len; + byte *x = bmd->x; + Sint i, j; + Sint *suffixes = erts_alloc(ERTS_ALC_T_TMP, m * sizeof(Sint)); + + compute_suffixes(x, m, suffixes); + + for (i = 0; i < m; ++i) { + bmd->goodshift[i] = m; + } + + j = 0; + + for (i = m - 1; i >= -1; --i) { + if (i == -1 || suffixes[i] == i + 1) { + while (j < m - 1 - i) { + if (bmd->goodshift[j] == m) { + bmd->goodshift[j] = m - 1 - i; + } + ++j; + } + } + } + for (i = 0; i <= m - 2; ++i) { + bmd->goodshift[m - 1 - suffixes[i]] = m - 1 - i; + } + erts_free(ERTS_ALC_T_TMP, suffixes); +} + +typedef struct { + Sint pos; + Sint len; +} BMFindFirstState; + +#define BM_OK 0 /* used only for find_all */ +#define BM_NOT_FOUND -1 +#define BM_RESTART -2 +#define BM_LOOP_FACTOR 10 /* Should we have a higher value? */ + +static void bm_init_find_first_match(BMFindFirstState *state, Sint startpos, + Uint len) +{ + state->pos = startpos; + state->len = (Sint) len; +} + + +static Sint bm_find_first_match(BMFindFirstState *state, BMData *bmd, + byte *haystack, Uint *reductions) +{ + Sint blen = bmd->len; + Sint len = state->len; + Sint *gs = bmd->goodshift; + Sint *bs = bmd->badshift; + byte *needle = bmd->x; + Sint i; + Sint j = state->pos; + register Uint reds = *reductions; + + while (j <= len - blen) { + if (--reds == 0) { + state->pos = j; + return BM_RESTART; + } + for (i = blen - 1; i >= 0 && needle[i] == haystack[i + j]; --i) + ; + if (i < 0) { /* found */ + *reductions = reds; + return j; + } + j += MAX(gs[i],bs[haystack[i+j]] - blen + 1 + i); + } + *reductions = reds; + return BM_NOT_FOUND; +} + +typedef struct { + Sint pos; + Sint len; + Uint m; + Uint allocated; + FindallData *out; +} BMFindAllState; + +static void bm_init_find_all(BMFindAllState *state, Sint startpos, Uint len) +{ + state->pos = startpos; + state->len = (Sint) len; + state->m = 0; + state->allocated = 0; + state->out = NULL; +} + +static void bm_restore_find_all(BMFindAllState *state, char *buff) +{ + memcpy(state,buff,sizeof(BMFindAllState)); + state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * + (state->allocated)); + memcpy(state->out,buff+sizeof(BMFindAllState), + sizeof(FindallData)*state->m); +} + +static void bm_serialize_find_all(BMFindAllState *state, char *buff) +{ + memcpy(buff,state,sizeof(BMFindAllState)); + memcpy(buff+sizeof(BMFindAllState),state->out, + sizeof(FindallData)*state->m); +} + +static void bm_clean_find_all(BMFindAllState *state) +{ + if (state->out != NULL) { + erts_free(ERTS_ALC_T_TMP, state->out); + } +#ifdef HARDDEBUG + state->out = NULL; + state->allocated = 0; +#endif +} + +#define SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(S) \ + (sizeof(BMFindAllState)+(sizeof(FindallData)*(S).m)) + +/* + * Differs to the find_first function in that it stores all matches and the + * values are returned only in the state. + */ +static Sint bm_find_all_non_overlapping(BMFindAllState *state, + BMData *bmd, byte *haystack, + Uint *reductions) +{ + Sint blen = bmd->len; + Sint len = state->len; + Sint *gs = bmd->goodshift; + Sint *bs = bmd->badshift; + byte *needle = bmd->x; + Sint i; + Sint j = state->pos; + Uint m = state->m; + Uint allocated = state->allocated; + FindallData *out = state->out; + register Uint reds = *reductions; + + while (j <= len - blen) { + if (--reds == 0) { + state->pos = j; + state->m = m; + state->allocated = allocated; + state->out = out; + return BM_RESTART; + } + for (i = blen - 1; i >= 0 && needle[i] == haystack[i + j]; --i) + ; + if (i < 0) { /* found */ + if (m >= allocated) { + if (!allocated) { + allocated = 10; + out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * allocated); + } else { + allocated *= 2; + out = erts_realloc(ERTS_ALC_T_TMP, out, + sizeof(FindallData) * allocated); + } + } + out[m].pos = j; + out[m].len = blen; + ++m; + j += blen; + } else { + j += MAX(gs[i],bs[haystack[i+j]] - blen + 1 + i); + } + } + state->m = m; + state->out = out; + *reductions = reds; + return (m == 0) ? BM_NOT_FOUND : BM_OK; +} + +/* + * Interface functions (i.e. "bif's") + */ + +/* + * Search functionality interfaces + */ + +static int do_binary_match_compile(Eterm argument, Eterm *tag, Binary **binp) +{ + Eterm t, b, comp_term = NIL; + Uint characters; + Uint words; + + characters = 0; + words = 0; + + if (is_list(argument)) { + t = argument; + while (is_list(t)) { + b = CAR(list_val(t)); + t = CDR(list_val(t)); + if (!is_binary(b)) { + goto badarg; + } + if (binary_bitsize(b) != 0) { + goto badarg; + } + ++words; + characters += binary_size(b); + } + if (is_not_nil(t)) { + goto badarg; + } + if (words > 1) { + comp_term = argument; + } else { + comp_term = CAR(list_val(argument)); + } + } else if (is_binary(argument)) { + if (binary_bitsize(argument) != 0) { + goto badarg; + } + words = 1; + comp_term = argument; + characters = binary_size(argument); + } + + if (characters == 0) { + goto badarg; + } + ASSERT(words > 0); + + if (words == 1) { + byte *bytes; + Uint bitoffs, bitsize; + byte *temp_alloc = NULL; + MyAllocator my; + BMData *bmd; + Binary *bin; + + ERTS_GET_BINARY_BYTES(comp_term, bytes, bitoffs, bitsize); + if (bitoffs != 0) { + bytes = erts_get_aligned_binary_bytes(comp_term, &temp_alloc); + } + bmd = create_bmdata(&my, bytes, characters, &bin); + compute_badshifts(bmd); + compute_goodshifts(bmd); + erts_free_aligned_binary_bytes(temp_alloc); + CHECK_ALLOCATOR(my); + *tag = am_bm; + *binp = bin; + return 0; + } else { + ACTrie *act; + MyAllocator my; + ACNode **qbuff; + Binary *bin; + + act = create_acdata(&my, characters, &qbuff, &bin); + t = comp_term; + while (is_list(t)) { + byte *bytes; + Uint bitoffs, bitsize; + byte *temp_alloc = NULL; + b = CAR(list_val(t)); + t = CDR(list_val(t)); + ERTS_GET_BINARY_BYTES(b, bytes, bitoffs, bitsize); + if (bitoffs != 0) { + bytes = erts_get_aligned_binary_bytes(b, &temp_alloc); + } + ac_add_one_pattern(&my,act,bytes,binary_size(b)); + erts_free_aligned_binary_bytes(temp_alloc); + } + ac_compute_failure_functions(act,qbuff); + CHECK_ALLOCATOR(my); + erts_free(ERTS_ALC_T_TMP,qbuff); + *tag = am_ac; + *binp = bin; + return 0; + } + badarg: + return -1; +} + +BIF_RETTYPE binary_compile_pattern_1(BIF_ALIST_1) +{ + Binary *bin; + Eterm tag, ret; + Eterm *hp; + + if (do_binary_match_compile(BIF_ARG_1,&tag,&bin)) { + BIF_ERROR(BIF_P,BADARG); + } + hp = HAlloc(BIF_P, PROC_BIN_SIZE+3); + ret = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), bin); + ret = TUPLE2(hp, tag, ret); + BIF_RET(ret); +} + +#define DO_BIN_MATCH_OK 0 +#define DO_BIN_MATCH_BADARG -1 +#define DO_BIN_MATCH_RESTART -2 + +static int do_binary_match(Process *p, Eterm subject, Uint hsstart, Uint hsend, + Eterm type, Binary *bin, Eterm state_term, + Eterm *res_term) +{ + byte *bytes; + Uint bitoffs, bitsize; + byte *temp_alloc = NULL; + + ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize); + if (bitsize != 0) { + goto badarg; + } + if (bitoffs != 0) { + bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc); + } + if (state_term != NIL) { + Eterm *ptr = big_val(state_term); + type = ptr[1]; + } + + if (type == am_bm) { + BMData *bm; + Sint pos; + Eterm ret; + Eterm *hp; + BMFindFirstState state; + Uint reds = get_reds(p, BM_LOOP_FACTOR); + Uint save_reds = reds; + + bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin); +#ifdef HARDDEBUG + dump_bm_data(bm); +#endif + if (state_term == NIL) { + bm_init_find_first_match(&state, hsstart, hsend); + } else { + Eterm *ptr = big_val(state_term); + memcpy(&state,ptr+2,sizeof(state)); + } +#ifdef HARDDEBUG + erts_printf("(bm) state->pos = %ld, state->len = %lu\n",state.pos, + state.len); +#endif + pos = bm_find_first_match(&state, bm, bytes, &reds); + if (pos == BM_NOT_FOUND) { + ret = am_nomatch; + } else if (pos == BM_RESTART) { + int x = (sizeof(BMFindFirstState) / sizeof(Eterm)) + + !!(sizeof(BMFindFirstState) % sizeof(Eterm)); +#ifdef HARDDEBUG + erts_printf("Trap bm!\n"); +#endif + hp = HAlloc(p,x+2); + hp[0] = make_pos_bignum_header(x+1); + hp[1] = type; + memcpy(hp+2,&state,sizeof(state)); + *res_term = make_big(hp); + erts_free_aligned_binary_bytes(temp_alloc); + return DO_BIN_MATCH_RESTART; + } else { + Eterm erlen = erts_make_integer((Uint) bm->len, p); + ret = erts_make_integer(pos,p); + hp = HAlloc(p,3); + ret = TUPLE2(hp, ret, erlen); + } + erts_free_aligned_binary_bytes(temp_alloc); + BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR); + *res_term = ret; + return DO_BIN_MATCH_OK; + } else if (type == am_ac) { + ACTrie *act; + Uint pos, rlen; + int acr; + ACFindFirstState state; + Eterm ret; + Eterm *hp; + Uint reds = get_reds(p, AC_LOOP_FACTOR); + Uint save_reds = reds; + + act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin); +#ifdef HARDDEBUG + dump_ac_trie(act); +#endif + if (state_term == NIL) { + ac_init_find_first_match(&state, act, hsstart, hsend); + } else { + Eterm *ptr = big_val(state_term); + memcpy(&state,ptr+2,sizeof(state)); + } + acr = ac_find_first_match(&state, bytes, &pos, &rlen, &reds); + if (acr == AC_NOT_FOUND) { + ret = am_nomatch; + } else if (acr == AC_RESTART) { + int x = (sizeof(state) / sizeof(Eterm)) + + !!(sizeof(BMFindFirstState) % sizeof(Eterm)); +#ifdef HARDDEBUG + erts_printf("Trap ac!\n"); +#endif + hp = HAlloc(p,x+2); + hp[0] = make_pos_bignum_header(x+1); + hp[1] = type; + memcpy(hp+2,&state,sizeof(state)); + *res_term = make_big(hp); + erts_free_aligned_binary_bytes(temp_alloc); + return DO_BIN_MATCH_RESTART; + } else { + Eterm epos = erts_make_integer(pos+hsstart,p); + Eterm erlen = erts_make_integer(rlen,p); + hp = HAlloc(p,3); + ret = TUPLE2(hp, epos, erlen); + } + erts_free_aligned_binary_bytes(temp_alloc); + BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR); + *res_term = ret; + return DO_BIN_MATCH_OK; + } + badarg: + return DO_BIN_MATCH_BADARG; +} + +static int do_binary_matches(Process *p, Eterm subject, Uint hsstart, + Uint hsend, Eterm type, Binary *bin, + Eterm state_term, Eterm *res_term) +{ + byte *bytes; + Uint bitoffs, bitsize; + byte *temp_alloc = NULL; + + ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize); + if (bitsize != 0) { + goto badarg; + } + if (bitoffs != 0) { + bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc); + } + if (state_term != NIL) { + Eterm *ptr = big_val(state_term); + type = ptr[1]; + } + + if (type == am_bm) { + BMData *bm; + Sint pos; + Eterm ret,tpl; + Eterm *hp; + BMFindAllState state; + Uint reds = get_reds(p, BM_LOOP_FACTOR); + Uint save_reds = reds; + + bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin); +#ifdef HARDDEBUG + dump_bm_data(bm); +#endif + if (state_term == NIL) { + bm_init_find_all(&state, hsstart, hsend); + } else { + Eterm *ptr = big_val(state_term); + bm_restore_find_all(&state,(char *) (ptr+2)); + } + + pos = bm_find_all_non_overlapping(&state, bm, bytes, &reds); + if (pos == BM_NOT_FOUND) { + ret = NIL; + } else if (pos == BM_RESTART) { + int x = + (SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) + + !!(SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(state) % sizeof(Eterm)); +#ifdef HARDDEBUG + erts_printf("Trap bm!\n"); +#endif + hp = HAlloc(p,x+2); + hp[0] = make_pos_bignum_header(x+1); + hp[1] = type; + bm_serialize_find_all(&state, (char *) (hp+2)); + *res_term = make_big(hp); + erts_free_aligned_binary_bytes(temp_alloc); + bm_clean_find_all(&state); + return DO_BIN_MATCH_RESTART; + } else { + FindallData *fad = state.out; + int i; + for (i = 0; i < state.m; ++i) { + fad[i].epos = erts_make_integer(fad[i].pos,p); + fad[i].elen = erts_make_integer(fad[i].len,p); + } + hp = HAlloc(p,state.m * (3 + 2)); + ret = NIL; + for (i = state.m - 1; i >= 0; --i) { + tpl = TUPLE2(hp, fad[i].epos, fad[i].elen); + hp +=3; + ret = CONS(hp,tpl,ret); + hp += 2; + } + } + erts_free_aligned_binary_bytes(temp_alloc); + bm_clean_find_all(&state); + BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR); + *res_term = ret; + return DO_BIN_MATCH_OK; + } else if (type == am_ac) { + ACTrie *act; + int acr; + ACFindAllState state; + Eterm ret,tpl; + Eterm *hp; + Uint reds = get_reds(p, AC_LOOP_FACTOR); + Uint save_reds = reds; + + act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin); +#ifdef HARDDEBUG + dump_ac_trie(act); +#endif + if (state_term == NIL) { + ac_init_find_all(&state, act, hsstart, hsend); + } else { + Eterm *ptr = big_val(state_term); + ac_restore_find_all(&state,(char *) (ptr+2)); + } + acr = ac_find_all_non_overlapping(&state, bytes, &reds); + if (acr == AC_NOT_FOUND) { + ret = NIL; + } else if (acr == AC_RESTART) { + int x = + (SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) + + !!(SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) % sizeof(Eterm)); +#ifdef HARDDEBUG + erts_printf("Trap ac!\n"); +#endif + hp = HAlloc(p,x+2); + hp[0] = make_pos_bignum_header(x+1); + hp[1] = type; + ac_serialize_find_all(&state, (char *) (hp+2)); + *res_term = make_big(hp); + erts_free_aligned_binary_bytes(temp_alloc); + ac_clean_find_all(&state); + return DO_BIN_MATCH_RESTART; + } else { + FindallData *fad = state.out; + int i; + for (i = 0; i < state.m; ++i) { + fad[i].epos = erts_make_integer(fad[i].pos,p); + fad[i].elen = erts_make_integer(fad[i].len,p); + } + hp = HAlloc(p,state.m * (3 + 2)); + ret = NIL; + for (i = state.m - 1; i >= 0; --i) { + tpl = TUPLE2(hp, fad[i].epos, fad[i].elen); + hp +=3; + ret = CONS(hp,tpl,ret); + hp += 2; + } + } + erts_free_aligned_binary_bytes(temp_alloc); + ac_clean_find_all(&state); + BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR); + *res_term = ret; + return DO_BIN_MATCH_OK; + } + badarg: + return DO_BIN_MATCH_BADARG; +} + +static int parse_match_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp) +{ + Eterm *tp; + Uint pos; + Sint len; + if (l == ((Eterm) 0) || l == NIL) { + /* Invalid term or NIL, we're called from binary_match(es)_2 or + have no options*/ + *posp = 0; + *endp = binary_size(bin); + return 0; + } else if (is_list(l)) { + while(is_list(l)) { + Eterm t = CAR(list_val(l)); + Uint orig_size; + if (!is_tuple(t)) { + goto badarg; + } + tp = tuple_val(t); + if (arityval(*tp) != 2) { + goto badarg; + } + if (tp[1] != am_scope || is_not_tuple(tp[2])) { + goto badarg; + } + tp = tuple_val(tp[2]); + if (arityval(*tp) != 2) { + goto badarg; + } + if (!term_to_Uint(tp[1], &pos)) { + goto badarg; + } + if (!term_to_Sint(tp[2], &len)) { + goto badarg; + } + if (len < 0) { + Sint lentmp = -len; + /* overflow */ + if (lentmp == len || lentmp < 0 || -lentmp != len) { + goto badarg; + } + len = lentmp; + pos -= len; + } + /* overflow */ + if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) { + goto badarg; + } + *endp = len + pos; + *posp = pos; + if ((orig_size = binary_size(bin)) < pos || + orig_size < (*endp)) { + goto badarg; + } + l = CDR(list_val(l)); + } + return 0; + } else { + badarg: + return 1; + } +} + +static BIF_RETTYPE binary_match_trap(BIF_ALIST_3) +{ + int runres; + Eterm result; + Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val; + runres = do_binary_match(BIF_P,BIF_ARG_1,0,0,NIL,bin,BIF_ARG_2,&result); + if (runres == DO_BIN_MATCH_OK) { + BIF_RET(result); + } else { + BUMP_ALL_REDS(BIF_P); + BIF_TRAP3(&binary_match_trap_export, BIF_P, BIF_ARG_1, result, + BIF_ARG_3); + } +} + +static BIF_RETTYPE binary_matches_trap(BIF_ALIST_3) +{ + int runres; + Eterm result; + Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val; + runres = do_binary_matches(BIF_P,BIF_ARG_1,0,0,NIL,bin,BIF_ARG_2,&result); + if (runres == DO_BIN_MATCH_OK) { + BIF_RET(result); + } else { + BUMP_ALL_REDS(BIF_P); + BIF_TRAP3(&binary_matches_trap_export, BIF_P, BIF_ARG_1, result, + BIF_ARG_3); + } +} + +BIF_RETTYPE binary_match_3(BIF_ALIST_3) +{ + Uint hsstart; + Uint hsend; + Eterm *tp; + Eterm type; + Binary *bin; + Eterm bin_term = NIL; + int runres; + Eterm result; + + if (is_not_binary(BIF_ARG_1)) { + goto badarg; + } + if (parse_match_opts_list(BIF_ARG_3,BIF_ARG_1,&hsstart,&hsend)) { + goto badarg; + } + if (hsend == 0) { + BIF_RET(am_nomatch); + } + if (is_tuple(BIF_ARG_2)) { + tp = tuple_val(BIF_ARG_2); + if (arityval(*tp) != 2 || is_not_atom(tp[1])) { + goto badarg; + } + if (((tp[1] != am_bm) && (tp[1] != am_ac)) || + !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) { + goto badarg; + } + type = tp[1]; + bin = ((ProcBin *) binary_val(tp[2]))->val; + if (type == am_bm && + ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) { + goto badarg; + } + if (type == am_ac && + ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) { + goto badarg; + } + bin_term = tp[2]; + } else if (do_binary_match_compile(BIF_ARG_2,&type,&bin)) { + goto badarg; + } + runres = do_binary_match(BIF_P,BIF_ARG_1,hsstart,hsend,type,bin,NIL,&result); + if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) { + Eterm *hp = HAlloc(BIF_P, PROC_BIN_SIZE); + bin_term = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), bin); + } else if (bin_term == NIL) { + erts_bin_free(bin); + } + switch (runres) { + case DO_BIN_MATCH_OK: + BIF_RET(result); + case DO_BIN_MATCH_RESTART: + BUMP_ALL_REDS(BIF_P); + BIF_TRAP3(&binary_match_trap_export, BIF_P, BIF_ARG_1, result, bin_term); + default: + goto badarg; + } + badarg: + BIF_ERROR(BIF_P,BADARG); +} + +BIF_RETTYPE binary_matches_3(BIF_ALIST_3) +{ + Uint hsstart, hsend; + Eterm *tp; + Eterm type; + Binary *bin; + Eterm bin_term = NIL; + int runres; + Eterm result; + + if (is_not_binary(BIF_ARG_1)) { + goto badarg; + } + if (parse_match_opts_list(BIF_ARG_3,BIF_ARG_1,&hsstart,&hsend)) { + goto badarg; + } + if (hsend == 0) { + BIF_RET(am_nomatch); + } + if (is_tuple(BIF_ARG_2)) { + tp = tuple_val(BIF_ARG_2); + if (arityval(*tp) != 2 || is_not_atom(tp[1])) { + goto badarg; + } + if (((tp[1] != am_bm) && (tp[1] != am_ac)) || + !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) { + goto badarg; + } + type = tp[1]; + bin = ((ProcBin *) binary_val(tp[2]))->val; + if (type == am_bm && + ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) { + goto badarg; + } + if (type == am_ac && + ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) { + goto badarg; + } + bin_term = tp[2]; + } else if (do_binary_match_compile(BIF_ARG_2,&type,&bin)) { + goto badarg; + } + runres = do_binary_matches(BIF_P,BIF_ARG_1,hsstart,hsend,type,bin, + NIL,&result); + if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) { + Eterm *hp = HAlloc(BIF_P, PROC_BIN_SIZE); + bin_term = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), bin); + } else if (bin_term == NIL) { + erts_bin_free(bin); + } + switch (runres) { + case DO_BIN_MATCH_OK: + BIF_RET(result); + case DO_BIN_MATCH_RESTART: + BUMP_ALL_REDS(BIF_P); + BIF_TRAP3(&binary_matches_trap_export, BIF_P, BIF_ARG_1, result, + bin_term); + default: + goto badarg; + } + badarg: + BIF_ERROR(BIF_P,BADARG); +} + + +BIF_RETTYPE binary_match_2(BIF_ALIST_2) +{ + return binary_match_3(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0)); +} + + +BIF_RETTYPE binary_matches_2(BIF_ALIST_2) +{ + return binary_matches_3(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0)); +} + + +BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen) +{ + Uint pos; + Sint len; + size_t orig_size; + Eterm orig; + Uint offset; + Uint bit_offset; + Uint bit_size; + Eterm* hp; + ErlSubBin* sb; + + if (is_not_binary(binary)) { + goto badarg; + } + if (!term_to_Uint(epos, &pos)) { + goto badarg; + } + if (!term_to_Sint(elen, &len)) { + goto badarg; + } + if (len < 0) { + Sint lentmp = -len; + /* overflow */ + if (lentmp == len || lentmp < 0 || -lentmp != len) { + goto badarg; + } + len = lentmp; + if (len > pos) { + goto badarg; + } + pos -= len; + } + /* overflow */ + if ((pos + len) < pos || (len > 0 && (pos + len) == pos)){ + goto badarg; + } + if ((orig_size = binary_size(binary)) < pos || + orig_size < (pos + len)) { + goto badarg; + } + + + + hp = HAlloc(p, ERL_SUB_BIN_SIZE); + + ERTS_GET_REAL_BIN(binary, orig, offset, bit_offset, bit_size); + sb = (ErlSubBin *) hp; + sb->thing_word = HEADER_SUB_BIN; + sb->size = len; + sb->offs = offset + pos; + sb->orig = orig; + sb->bitoffs = bit_offset; + sb->bitsize = 0; + sb->is_writable = 0; + + BIF_RET(make_binary(sb)); + + badarg: + BIF_ERROR(p, BADARG); +} + +#define ERTS_NEED_GC(p, need) ((HEAP_LIMIT((p)) - HEAP_TOP((p))) <= (need)) + +BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is_tuple) +{ + Uint pos; + Sint len; + size_t orig_size; + Eterm orig; + Uint offset; + Uint bit_offset; + Uint bit_size; + Eterm* hp; + ErlSubBin* sb; + Eterm binary; + Eterm *tp; + Eterm epos, elen; + int extra_args; + + + if (range_is_tuple) { + Eterm tpl = reg[live]; + extra_args = 1; + if (is_not_tuple(tpl)) { + goto badarg; + } + tp = tuple_val(tpl); + if (arityval(*tp) != 2) { + goto badarg; + } + + epos = tp[1]; + elen = tp[2]; + } else { + extra_args = 2; + epos = reg[live-1]; + elen = reg[live]; + } + binary = reg[live-extra_args]; + + if (is_not_binary(binary)) { + goto badarg; + } + if (!term_to_Uint(epos, &pos)) { + goto badarg; + } + if (!term_to_Sint(elen, &len)) { + goto badarg; + } + if (len < 0) { + Sint lentmp = -len; + /* overflow */ + if (lentmp == len || lentmp < 0 || -lentmp != len) { + goto badarg; + } + len = lentmp; + if (len > pos) { + goto badarg; + } + pos -= len; + } + /* overflow */ + if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) { + goto badarg; + } + if ((orig_size = binary_size(binary)) < pos || + orig_size < (pos + len)) { + goto badarg; + } + + if (ERTS_NEED_GC(p, ERL_SUB_BIN_SIZE)) { + erts_garbage_collect(p, ERL_SUB_BIN_SIZE, reg, live+1-extra_args); /* I don't need the tuple + or indices any more */ + binary = reg[live-extra_args]; + } + + hp = p->htop; + p->htop += ERL_SUB_BIN_SIZE; + + ERTS_GET_REAL_BIN(binary, orig, offset, bit_offset, bit_size); + + sb = (ErlSubBin *) hp; + sb->thing_word = HEADER_SUB_BIN; + sb->size = len; + sb->offs = offset + pos; + sb->orig = orig; + sb->bitoffs = bit_offset; + sb->bitsize = 0; + sb->is_writable = 0; + + BIF_RET(make_binary(sb)); + + badarg: + BIF_ERROR(p, BADARG); +} +/************************************************************* + * The actual guard BIFs are in erl_bif_guard.c + * but the implementation of both the non-gc and the gc + * variants are here. Note that the functions are named so that they do + * not clash with the guard bif's erlang:binary_part/2,3 + *************************************************************/ + +BIF_RETTYPE binary_binary_part_3(BIF_ALIST_3) +{ + return erts_binary_part(BIF_P,BIF_ARG_1,BIF_ARG_2, BIF_ARG_3); +} + +BIF_RETTYPE binary_binary_part_2(BIF_ALIST_2) +{ + Eterm *tp; + if (is_not_tuple(BIF_ARG_2)) { + goto badarg; + } + tp = tuple_val(BIF_ARG_2); + if (arityval(*tp) != 2) { + goto badarg; + } + return erts_binary_part(BIF_P,BIF_ARG_1,tp[1], tp[2]); + badarg: + BIF_ERROR(BIF_P,BADARG); +} + +typedef struct { + int type; /* CL_TYPE_XXX */ + byte *temp_alloc; /* Used for erts_get/free_aligned, i.e. CL_TYPE_ALIGNED */ + unsigned char *buff; /* Used for all types, malloced if CL_TYPE_HEAP */ + Uint bufflen; /* The length (in bytes) of buffer */ +} CommonData; + +#define COMMON_LOOP_FACTOR 10 + +#define DIRECTION_PREFIX 0 +#define DIRECTION_SUFFIX 1 + +#define CL_OK 0 +#define CL_RESTART 1 + +/* The type field in the above structure */ +#define CL_TYPE_EMPTY 0 /* End of array */ +#define CL_TYPE_HEAP 1 +#define CL_TYPE_ALIGNED 2 +#define CL_TYPE_COMMON 3 /* emacsulated */ +#define CL_TYPE_HEAP_NOALLOC 4 /* Will need allocating when trapping */ + + +static int do_search_forward(CommonData *cd, Uint *posp, Uint *redsp) +{ + Uint pos = *posp; + Sint reds = (Sint) *redsp; + int i; + unsigned char current = 0; + + for(;;) { + for(i = 0; cd[i].type != CL_TYPE_EMPTY; ++i) { + if (pos >= cd[i].bufflen) { + *posp = pos; + if (reds > 0) { + *redsp = (Uint) reds; + } else { + *redsp = 0; + } + return CL_OK; + } + if (i == 0) { + current = cd[i].buff[pos]; + } else { + if (cd[i].buff[pos] != current) { + *posp = pos; + if (reds > 0) { + *redsp = (Uint) reds; + } else { + *redsp = 0; + } + return CL_OK; + } + } + --reds; + } + ++pos; + if (reds <= 0) { + *posp = pos; + *redsp = 0; + return CL_RESTART; + } + } +} +static int do_search_backward(CommonData *cd, Uint *posp, Uint *redsp) +{ + Uint pos = *posp; + Sint reds = (Sint) *redsp; + int i; + unsigned char current = 0; + + for(;;) { + for(i = 0; cd[i].type != CL_TYPE_EMPTY; ++i) { + if (pos >= cd[i].bufflen) { + *posp = pos; + if (reds > 0) { + *redsp = (Uint) reds; + } else { + *redsp = 0; + } + return CL_OK; + } + if (i == 0) { + current = cd[i].buff[cd[i].bufflen - 1 - pos]; + } else { + if (cd[i].buff[cd[i].bufflen - 1 - pos] != current) { + *posp = pos; + if (reds > 0) { + *redsp = (Uint) reds; + } else { + *redsp = 0; + } + return CL_OK; + } + } + --reds; + } + ++pos; + if (reds <= 0) { + *posp = pos; + *redsp = 0; + return CL_RESTART; + } + } +} + +static void cleanup_common_data(Binary *bp) +{ + int i; + CommonData *cd; + cd = (CommonData *) ERTS_MAGIC_BIN_DATA(bp); + for (i=0;cd[i].type != CL_TYPE_EMPTY;++i) { + switch (cd[i].type) { + case CL_TYPE_HEAP: + erts_free(ERTS_ALC_T_BINARY_BUFFER,cd[i].buff); + break; + case CL_TYPE_ALIGNED: + erts_free_aligned_binary_bytes_extra(cd[i].temp_alloc, ERTS_ALC_T_BINARY_BUFFER); + break; + default: + break; + } + } + return; +} + +static BIF_RETTYPE do_longest_common(Process *p, Eterm list, int direction) +{ + Eterm l = list; + int n = 0; + Binary *mb; + CommonData *cd; + int i = 0; + Uint reds = get_reds(p, COMMON_LOOP_FACTOR); + Uint save_reds = reds; + int res; + Export *trapper; + Uint pos; + Eterm epos; + Eterm *hp; + Eterm bin_term; + Eterm b; + + /* First just count the number of binaries */ + while (is_list(l)) { + b = CAR(list_val(l)); + if (!is_binary(b)) { + goto badarg; + } + ++n; + l = CDR(list_val(l)); + } + if (l != NIL || n == 0) { + goto badarg; + } + + /* OK, now create a buffer of the right size, we can do a magic binary right away, + that's not too costly. */ + mb = erts_create_magic_binary((n+1)*sizeof(CommonData),cleanup_common_data); + cd = (CommonData *) ERTS_MAGIC_BIN_DATA(mb); + l = list; + while (is_list(l)) { + Uint bitoffs; + Uint bitsize; + Uint offset; + Eterm real_bin; + ProcBin* pb; + + cd[i].type = CL_TYPE_EMPTY; + b = CAR(list_val(l)); + ERTS_GET_REAL_BIN(b, real_bin, offset, bitoffs, bitsize); + if (bitsize != 0) { + erts_bin_free(mb); + goto badarg; + } + cd[i].bufflen = binary_size(b); + cd[i].temp_alloc = NULL; + if (*(binary_val(real_bin)) == HEADER_PROC_BIN) { + pb = (ProcBin *) binary_val(real_bin); + if (pb->flags) { + erts_emasculate_writable_binary(pb); + } + cd[i].buff = erts_get_aligned_binary_bytes_extra(b, &(cd[i].temp_alloc), + ERTS_ALC_T_BINARY_BUFFER,0); + cd[i].type = (cd[i].temp_alloc != NULL) ? CL_TYPE_ALIGNED : CL_TYPE_COMMON; + } else { /* Heap binary */ + cd[i].buff = erts_get_aligned_binary_bytes_extra(b, &(cd[i].temp_alloc), + ERTS_ALC_T_BINARY_BUFFER,0); + /* CL_TYPE_HEAP_NOALLOC means you have to copy if trapping */ + cd[i].type = (cd[i].temp_alloc != NULL) ? CL_TYPE_ALIGNED : CL_TYPE_HEAP_NOALLOC; + } + ++i; + l = CDR(list_val(l)); + } + cd[i].type = CL_TYPE_EMPTY; +#if defined(DEBUG) || defined(VALGRIND) + cd[i].temp_alloc = NULL; + cd[i].buff = NULL; + cd[i].bufflen = 0; +#endif + + pos = 0; + if (direction == DIRECTION_PREFIX) { + trapper = &binary_longest_prefix_trap_export; + res = do_search_forward(cd,&pos,&reds); + } else { + ASSERT(direction == DIRECTION_SUFFIX); + trapper = &binary_longest_suffix_trap_export; + res = do_search_backward(cd,&pos,&reds); + } + epos = erts_make_integer(pos,p); + if (res == CL_OK) { + erts_bin_free(mb); + BUMP_REDS(p, (save_reds - reds) / COMMON_LOOP_FACTOR); + BIF_RET(epos); + } else { + ASSERT(res == CL_RESTART); + /* Copy all heap binaries that are not already copied (aligned) */ + for(i = 0; i < n; ++i) { + if (cd[i].type == CL_TYPE_HEAP_NOALLOC) { + unsigned char *tmp = cd[i].buff; + cd[i].buff = erts_alloc(ERTS_ALC_T_BINARY_BUFFER, cd[i].bufflen); + memcpy(cd[i].buff,tmp,cd[i].bufflen); + cd[i].type = CL_TYPE_HEAP; + } + } + hp = HAlloc(p, PROC_BIN_SIZE); + bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), mb); + BUMP_ALL_REDS(p); + BIF_TRAP3(trapper, p, bin_term, epos,list); + } + badarg: + BIF_ERROR(p,BADARG); +} + +static BIF_RETTYPE do_longest_common_trap(Process *p, Eterm bin_term, Eterm current_pos, + Eterm orig_list, int direction) +{ + Uint reds = get_reds(p, COMMON_LOOP_FACTOR); + Uint save_reds = reds; + Uint pos; + Binary *bin; + CommonData *cd; + int res; + Eterm epos; + Export *trapper; + +#ifdef DEBUG + int r; + r = term_to_Uint(current_pos, &pos); + ASSERT(r != 0); +#else + term_to_Uint(current_pos, &pos); +#endif + ASSERT(ERTS_TERM_IS_MAGIC_BINARY(bin_term)); + bin = ((ProcBin *) binary_val(bin_term))->val; + cd = (CommonData *) ERTS_MAGIC_BIN_DATA(bin); + if (direction == DIRECTION_PREFIX) { + trapper = &binary_longest_prefix_trap_export; + res = do_search_forward(cd,&pos,&reds); + } else { + ASSERT(direction == DIRECTION_SUFFIX); + trapper = &binary_longest_suffix_trap_export; + res = do_search_backward(cd,&pos,&reds); + } + epos = erts_make_integer(pos,p); + if (res == CL_OK) { + BUMP_REDS(p, (save_reds - reds) / COMMON_LOOP_FACTOR); + BIF_RET(epos); + } else { + ASSERT(res == CL_RESTART); + /* Copy all heap binaries that are not already copied (aligned) */ + BUMP_ALL_REDS(p); + BIF_TRAP3(trapper, p, bin_term, epos, orig_list); + } +} + +static BIF_RETTYPE binary_longest_prefix_trap(BIF_ALIST_3) +{ + return do_longest_common_trap(BIF_P,BIF_ARG_1,BIF_ARG_2,BIF_ARG_3,DIRECTION_PREFIX); +} + +static BIF_RETTYPE binary_longest_suffix_trap(BIF_ALIST_3) +{ + return do_longest_common_trap(BIF_P,BIF_ARG_1,BIF_ARG_2,BIF_ARG_3,DIRECTION_SUFFIX); +} + +BIF_RETTYPE binary_longest_common_prefix_1(BIF_ALIST_1) +{ + return do_longest_common(BIF_P,BIF_ARG_1,DIRECTION_PREFIX); +} + +BIF_RETTYPE binary_longest_common_suffix_1(BIF_ALIST_1) +{ + return do_longest_common(BIF_P,BIF_ARG_1,DIRECTION_SUFFIX); +} + +BIF_RETTYPE binary_first_1(BIF_ALIST_1) +{ + byte* bytes; + Uint byte_size; + Uint bit_offs; + Uint bit_size; + Uint res; + + if (is_not_binary(BIF_ARG_1)) { + goto badarg; + } + byte_size = binary_size(BIF_ARG_1); + if (!byte_size) { + goto badarg; + } + ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); + if (bit_size) { + goto badarg; + } + if (bit_offs) { + res = ((((Uint) bytes[0]) << bit_offs) | (((Uint) bytes[1]) >> (8-bit_offs))) & 0xFF; + } else { + res = bytes[0]; + } + BIF_RET(make_small(res)); + badarg: + BIF_ERROR(BIF_P,BADARG); +} + +BIF_RETTYPE binary_last_1(BIF_ALIST_1) +{ + byte* bytes; + Uint byte_size; + Uint bit_offs; + Uint bit_size; + Uint res; + + if (is_not_binary(BIF_ARG_1)) { + goto badarg; + } + byte_size = binary_size(BIF_ARG_1); + if (!byte_size) { + goto badarg; + } + ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); + if (bit_size) { + goto badarg; + } + if (bit_offs) { + res = ((((Uint) bytes[byte_size-1]) << bit_offs) | + (((Uint) bytes[byte_size]) >> (8-bit_offs))) & 0xFF; + } else { + res = bytes[byte_size-1]; + } + BIF_RET(make_small(res)); + badarg: + BIF_ERROR(BIF_P,BADARG); +} + +BIF_RETTYPE binary_at_2(BIF_ALIST_2) +{ + byte* bytes; + Uint byte_size; + Uint bit_offs; + Uint bit_size; + Uint res; + Uint index; + + if (is_not_binary(BIF_ARG_1)) { + goto badarg; + } + byte_size = binary_size(BIF_ARG_1); + if (!byte_size) { + goto badarg; + } + if (!term_to_Uint(BIF_ARG_2, &index)) { + goto badarg; + } + if (index >= byte_size) { + goto badarg; + } + ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); + if (bit_size) { + goto badarg; + } + if (bit_offs) { + res = ((((Uint) bytes[index]) << bit_offs) | + (((Uint) bytes[index+1]) >> (8-bit_offs))) & 0xFF; + } else { + res = bytes[index]; + } + BIF_RET(make_small(res)); + badarg: + BIF_ERROR(BIF_P,BADARG); +} + +#define BIN_TO_LIST_OK 0 +#define BIN_TO_LIST_TRAP 1 +/* No badarg, checked before call */ + +#define BIN_TO_LIST_LOOP_FACTOR 10 + +static int do_bin_to_list(Process *p, byte *bytes, Uint bit_offs, + Uint start, Sint *lenp, Eterm *termp) +{ + Uint reds = get_reds(p, BIN_TO_LIST_LOOP_FACTOR); /* reds can never be 0 */ + Uint len = *lenp; + Uint loops; + Eterm *hp; + Eterm term = *termp; + Uint n; + + ASSERT(reds > 0); + + loops = MIN(reds,len); + + BUMP_REDS(p, loops / BIN_TO_LIST_LOOP_FACTOR); + + hp = HAlloc(p,2*loops); + while (loops--) { + --len; + if (bit_offs) { + n = ((((Uint) bytes[start+len]) << bit_offs) | + (((Uint) bytes[start+len+1]) >> (8-bit_offs))) & 0xFF; + } else { + n = bytes[start+len]; + } + + term = CONS(hp,make_small(n),term); + hp +=2; + } + *termp = term; + *lenp = len; + if (len) { + BUMP_ALL_REDS(p); + return BIN_TO_LIST_TRAP; + } + return BIN_TO_LIST_OK; +} + + +static BIF_RETTYPE do_trap_bin_to_list(Process *p, Eterm binary, + Uint start, Sint len, Eterm sofar) +{ + Eterm *hp; + Eterm blob; + + hp = HAlloc(p,3); + hp[0] = make_pos_bignum_header(2); + hp[1] = start; + hp[2] = (Uint) len; + blob = make_big(hp); + BIF_TRAP3(&binary_bin_to_list_trap_export, p, binary, blob, sofar); +} + +static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3) +{ + Eterm *ptr; + Uint start; + Sint len; + byte *bytes; + Uint bit_offs; + Uint bit_size; + Eterm res = BIF_ARG_3; + + ptr = big_val(BIF_ARG_2); + start = ptr[1]; + len = (Sint) ptr[2]; + + ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); + if (do_bin_to_list(BIF_P, bytes, bit_offs, start, &len, &res) == + BIN_TO_LIST_OK) { + BIF_RET(res); + } + return do_trap_bin_to_list(BIF_P,BIF_ARG_1,start,len,res); +} + +static BIF_RETTYPE binary_bin_to_list_common(Process *p, + Eterm bin, + Eterm epos, + Eterm elen) +{ + Uint pos; + Sint len; + size_t sz; + byte *bytes; + Uint bit_offs; + Uint bit_size; + Eterm res = NIL; + + if (is_not_binary(bin)) { + goto badarg; + } + if (!term_to_Uint(epos, &pos)) { + goto badarg; + } + if (!term_to_Sint(elen, &len)) { + goto badarg; + } + if (len < 0) { + Sint lentmp = -len; + /* overflow */ + if (lentmp == len || lentmp < 0 || -lentmp != len) { + goto badarg; + } + len = lentmp; + if (len > pos) { + goto badarg; + } + pos -= len; + } + /* overflow */ + if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) { + goto badarg; + } + sz = binary_size(bin); + + if (pos+len > sz) { + goto badarg; + } + ERTS_GET_BINARY_BYTES(bin,bytes,bit_offs,bit_size); + if (bit_size != 0) { + goto badarg; + } + if(do_bin_to_list(p, bytes, bit_offs, pos, &len, &res) == + BIN_TO_LIST_OK) { + BIF_RET(res); + } + return do_trap_bin_to_list(p,bin,pos,len,res); + + badarg: + BIF_ERROR(p,BADARG); +} + +BIF_RETTYPE binary_bin_to_list_3(BIF_ALIST_3) +{ + return binary_bin_to_list_common(BIF_P,BIF_ARG_1,BIF_ARG_2,BIF_ARG_3); +} + +BIF_RETTYPE binary_bin_to_list_2(BIF_ALIST_2) +{ + Eterm *tp; + + if (is_not_tuple(BIF_ARG_2)) { + goto badarg; + } + tp = tuple_val(BIF_ARG_2); + if (arityval(*tp) != 2) { + goto badarg; + } + return binary_bin_to_list_common(BIF_P,BIF_ARG_1,tp[1],tp[2]); + badarg: + BIF_ERROR(BIF_P,BADARG); +} + +BIF_RETTYPE binary_bin_to_list_1(BIF_ALIST_1) +{ + Uint pos = 0; + Sint len; + byte *bytes; + Uint bit_offs; + Uint bit_size; + Eterm res = NIL; + + if (is_not_binary(BIF_ARG_1)) { + goto badarg; + } + len = binary_size(BIF_ARG_1); + ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); + if (bit_size != 0) { + goto badarg; + } + if(do_bin_to_list(BIF_P, bytes, bit_offs, pos, &len, &res) == + BIN_TO_LIST_OK) { + BIF_RET(res); + } + return do_trap_bin_to_list(BIF_P,BIF_ARG_1,pos,len,res); + badarg: + BIF_ERROR(BIF_P,BADARG); +} + +/* + * Ok, erlang:list_to_binary does not interrupt, and we really don't want + * an alternative implementation for the exact same thing, why we + * have descided to use the old non-restarting implementation for now. + * In reality, there are seldom many iterations involved in doing this, so the + * problem of long-running bifs is not really that big in this case. + * So, for now we use the old implementation also in the module binary. + */ + +BIF_RETTYPE binary_list_to_bin_1(BIF_ALIST_1) +{ + return erts_list_to_binary_bif(BIF_P, BIF_ARG_1); +} + +typedef struct { + Uint times_left; + Uint source_size; + int source_type; + byte *source; + byte *temp_alloc; + Uint result_pos; + Binary *result; +} CopyBinState; + +#define BC_TYPE_EMPTY 0 +#define BC_TYPE_HEAP 1 +#define BC_TYPE_ALIGNED 2 /* May or may not point to (emasculated) binary, temp_alloc field is set + so that erts_free_aligned_binary_bytes_extra can handle either */ + + +#define BINARY_COPY_LOOP_FACTOR 100 + +static void cleanup_copy_bin_state(Binary *bp) +{ + CopyBinState *cbs = (CopyBinState *) ERTS_MAGIC_BIN_DATA(bp); + if (cbs->result != NULL) { + erts_bin_free(cbs->result); + cbs->result = NULL; + } + switch (cbs->source_type) { + case BC_TYPE_HEAP: + erts_free(ERTS_ALC_T_BINARY_BUFFER,cbs->source); + break; + case BC_TYPE_ALIGNED: + erts_free_aligned_binary_bytes_extra(cbs->temp_alloc, + ERTS_ALC_T_BINARY_BUFFER); + break; + default: + /* otherwise do nothing */ + break; + } + cbs->source_type = BC_TYPE_EMPTY; +} + +/* + * Binary *erts_bin_nrml_alloc(Uint size); + * Binary *erts_bin_realloc(Binary *bp, Uint size); + * void erts_bin_free(Binary *bp); + */ +static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en) +{ + Uint n; + byte *bytes; + Uint bit_offs; + Uint bit_size; + size_t size; + Uint reds = get_reds(p, BINARY_COPY_LOOP_FACTOR); + Uint target_size; + byte *t; + Uint pos; + + + if (is_not_binary(bin)) { + goto badarg; + } + if (!term_to_Uint(en, &n)) { + goto badarg; + } + if (!n) { + Eterm res_term = erts_new_heap_binary(p,NULL,0,&bytes); + BIF_RET(res_term); + } + ERTS_GET_BINARY_BYTES(bin,bytes,bit_offs,bit_size); + if (bit_size != 0) { + goto badarg; + } + + size = binary_size(bin); + target_size = size * n; + + if ((target_size - size) >= reds) { + Eterm orig; + Uint offset; + Uint bit_offset; + Uint bit_size; + CopyBinState *cbs; + Eterm *hp; + Eterm trap_term; + int i; + + /* We will trap, set up the structure for trapping right away */ + Binary *mb = erts_create_magic_binary(sizeof(CopyBinState), + cleanup_copy_bin_state); + cbs = ERTS_MAGIC_BIN_DATA(mb); + + cbs->temp_alloc = NULL; + cbs->source = NULL; + + ERTS_GET_REAL_BIN(bin, orig, offset, bit_offset, bit_size); + if (*(binary_val(orig)) == HEADER_PROC_BIN) { + ProcBin* pb = (ProcBin *) binary_val(orig); + if (pb->flags) { + erts_emasculate_writable_binary(pb); + } + cbs->source = + erts_get_aligned_binary_bytes_extra(bin, + &(cbs->temp_alloc), + ERTS_ALC_T_BINARY_BUFFER, + 0); + cbs->source_type = BC_TYPE_ALIGNED; + } else { /* Heap binary */ + cbs->source = + erts_get_aligned_binary_bytes_extra(bin, + &(cbs->temp_alloc), + ERTS_ALC_T_BINARY_BUFFER, + 0); + if (!(cbs->temp_alloc)) { /* alignment not needed, need to copy */ + byte *tmp = erts_alloc(ERTS_ALC_T_BINARY_BUFFER,size); + memcpy(tmp,cbs->source,size); + cbs->source = tmp; + cbs->source_type = BC_TYPE_HEAP; + } else { + cbs->source_type = BC_TYPE_ALIGNED; + } + } + cbs->result = erts_bin_nrml_alloc(target_size); /* Always offheap + if trapping */ + cbs->result->flags = 0; + cbs->result->orig_size = target_size; + erts_refc_init(&(cbs->result->refc), 1); + t = (byte *) cbs->result->orig_bytes; /* No offset or anything */ + pos = 0; + i = 0; + while (pos < reds) { + memcpy(t+pos,cbs->source, size); + pos += size; + ++i; + } + cbs->source_size = size; + cbs->result_pos = pos; + cbs->times_left = n-i; + hp = HAlloc(p,PROC_BIN_SIZE); + trap_term = erts_mk_magic_binary_term(&hp, &MSO(p), mb); + BUMP_ALL_REDS(p); + BIF_TRAP2(&binary_copy_trap_export, p, bin, trap_term); + } else { + Eterm res_term; + byte *temp_alloc = NULL; + byte *source = + erts_get_aligned_binary_bytes(bin, + &temp_alloc); + if (target_size <= ERL_ONHEAP_BIN_LIMIT) { + res_term = erts_new_heap_binary(p,NULL,target_size,&t); + } else { + res_term = erts_new_mso_binary(p,NULL,target_size); + t = ((ProcBin *) binary_val(res_term))->bytes; + } + pos = 0; + while (pos < target_size) { + memcpy(t+pos,source, size); + pos += size; + } + erts_free_aligned_binary_bytes(temp_alloc); + BUMP_REDS(p,pos / BINARY_COPY_LOOP_FACTOR); + BIF_RET(res_term); + } + badarg: + BIF_ERROR(p,BADARG); +} + +BIF_RETTYPE binary_copy_trap(BIF_ALIST_2) +{ + Uint n; + size_t size; + Uint reds = get_reds(BIF_P, BINARY_COPY_LOOP_FACTOR); + byte *t; + Uint pos; + Binary *mb = ((ProcBin *) binary_val(BIF_ARG_2))->val; + CopyBinState *cbs = (CopyBinState *) ERTS_MAGIC_BIN_DATA(mb); + Uint opos; + + /* swapout... */ + n = cbs->times_left; + size = cbs->source_size; + opos = pos = cbs->result_pos; + t = (byte *) cbs->result->orig_bytes; /* "well behaved" binary */ + if ((n-1) * size >= reds) { + Uint i = 0; + while ((pos - opos) < reds) { + memcpy(t+pos,cbs->source, size); + pos += size; + ++i; + } + cbs->result_pos = pos; + cbs->times_left -= i; + BUMP_ALL_REDS(BIF_P); + BIF_TRAP2(&binary_copy_trap_export, BIF_P, BIF_ARG_1, BIF_ARG_2); + } else { + Binary *save; + ProcBin* pb; + Uint target_size = cbs->result->orig_size; + while (pos < target_size) { + memcpy(t+pos,cbs->source, size); + pos += size; + } + save = cbs->result; + cbs->result = NULL; + cleanup_copy_bin_state(mb); /* now cbs is dead */ + pb = (ProcBin *) HAlloc(BIF_P, PROC_BIN_SIZE); + pb->thing_word = HEADER_PROC_BIN; + pb->size = target_size; + pb->next = MSO(BIF_P).first; + MSO(BIF_P).first = (struct erl_off_heap_header*) pb; + pb->val = save; + pb->bytes = t; + pb->flags = 0; + + OH_OVERHEAD(&(MSO(BIF_P)), target_size / sizeof(Eterm)); + BUMP_REDS(BIF_P,(pos - opos) / BINARY_COPY_LOOP_FACTOR); + + BIF_RET(make_binary(pb)); + } +} + + +BIF_RETTYPE binary_copy_1(BIF_ALIST_1) +{ + return do_binary_copy(BIF_P,BIF_ARG_1,make_small(1)); +} + +BIF_RETTYPE binary_copy_2(BIF_ALIST_2) +{ + return do_binary_copy(BIF_P,BIF_ARG_1,BIF_ARG_2); +} + +BIF_RETTYPE binary_referenced_byte_size_1(BIF_ALIST_1) +{ + ErlSubBin *sb; + ProcBin *pb; + Eterm res; + Eterm bin = BIF_ARG_1; + + if (is_not_binary(BIF_ARG_1)) { + BIF_ERROR(BIF_P,BADARG); + } + sb = (ErlSubBin *) binary_val(bin); + if (sb->thing_word == HEADER_SUB_BIN) { + bin = sb->orig; + } + pb = (ProcBin *) binary_val(bin); + if (pb->thing_word == HEADER_PROC_BIN) { + /* XXX:PaN - Halfword - orig_size is a long, we should handle that */ + res = erts_make_integer((Uint) pb->val->orig_size, BIF_P); + } else { /* heap binary */ + res = erts_make_integer((Uint) ((ErlHeapBin *) pb)->size, BIF_P); + } + BIF_RET(res); +} + +#define END_BIG 0 +#define END_SMALL 1 + +#ifdef WORDS_BIGENDIAN +#define END_NATIVE END_BIG +#else +#define END_NATIVE END_SMALL +#endif + +static int get_need(Uint u) { +#if defined(ARCH_64) && !HALFWORD_HEAP + if (u > 0xFFFFFFFFUL) { + if (u > 0xFFFFFFFFFFFFUL) { + if (u > 0xFFFFFFFFFFFFFFUL) { + return 8; + } + return 7; + } + if (u > 0xFFFFFFFFFFUL) { + return 6; + } + return 5; + } +#endif + if (u > 0xFFFFUL) { + if (u > 0xFFFFFFUL) { + return 4; + } + return 3; + } + if (u > 0xFFUL) { + return 2; + } + return 1; +} + +static BIF_RETTYPE do_encode_unsigned(Process *p, Eterm uns, Eterm endianess) +{ + Eterm res; + if ((is_not_small(uns) && is_not_big(uns)) || is_not_atom(endianess) || + (endianess != am_big && endianess != am_little)) { + goto badarg; + } + if (is_small(uns)) { + Sint x = signed_val(uns); + Uint u; + int n,i; + byte *b; + + if (x < 0) { + goto badarg; + } + + u = (Uint) x; + n = get_need(u); + ASSERT(n <= ERL_ONHEAP_BIN_LIMIT); + res = erts_new_heap_binary(p, NULL, n, &b); + if (endianess == am_big) { + for(i=n-1;i>=0;--i) { + b[i] = u & 0xFF; + u >>= 8; + } + } else { + for(i=0;i<n;++i) { + b[i] = u & 0xFF; + u >>= 8; + } + } + BIF_RET(res); + } else { + /* Big */ + Eterm *bigp = big_val(uns); + Uint n; + dsize_t num_parts = BIG_SIZE(bigp); + Eterm res; + byte *b; + ErtsDigit d; + + if(BIG_SIGN(bigp)) { + goto badarg; + } + n = (num_parts-1)*sizeof(ErtsDigit)+get_need(BIG_DIGIT(bigp,(num_parts-1))); + if (n <= ERL_ONHEAP_BIN_LIMIT) { + res = erts_new_heap_binary(p,NULL,n,&b); + } else { + res = erts_new_mso_binary(p,NULL,n); + b = ((ProcBin *) binary_val(res))->bytes; + } + + if (endianess == am_big) { + Sint i,j; + j = 0; + d = BIG_DIGIT(bigp,0); + for (i=n-1;i>=0;--i) { + b[i] = d & 0xFF; + if (!((++j) % sizeof(ErtsDigit))) { + d = BIG_DIGIT(bigp,j / sizeof(ErtsDigit)); + } else { + d >>= 8; + } + } + } else { + Sint i,j; + j = 0; + d = BIG_DIGIT(bigp,0); + for (i=0;i<n;++i) { + b[i] = d & 0xFF; + if (!((++j) % sizeof(ErtsDigit))) { + d = BIG_DIGIT(bigp,j / sizeof(ErtsDigit)); + } else { + d >>= 8; + } + } + + } + BIF_RET(res); + } + badarg: + BIF_ERROR(p,BADARG); +} + +static BIF_RETTYPE do_decode_unsigned(Process *p, Eterm uns, Eterm endianess) +{ + byte *bytes; + Uint bitoffs, bitsize; + Uint size; + Eterm res; + + if (is_not_binary(uns) || is_not_atom(endianess) || + (endianess != am_big && endianess != am_little)) { + goto badarg; + } + ERTS_GET_BINARY_BYTES(uns, bytes, bitoffs, bitsize); + if (bitsize != 0) { + goto badarg; + } + /* align while rolling */ + size = binary_size(uns); + if (bitoffs) { + if (endianess == am_big) { + while (size && (((((Uint) bytes[0]) << bitoffs) | + (((Uint) bytes[1]) >> (8-bitoffs))) & 0xFF) == 0) { + ++bytes; + --size; + } + } else { + while(size && + (((((Uint) bytes[size-1]) << bitoffs) | + (((Uint) bytes[size]) >> (8-bitoffs))) & 0xFF) == 0) { + --size; + } + } + } else { + if (endianess == am_big) { + while (size && *bytes == 0) { + ++bytes; + --size; + } + } else { + while(size && bytes[size-1] == 0) { + --size; + } + } + } + if (!size) { + BIF_RET(make_small(0)); + } + + if (size <= sizeof(Uint)) { + Uint u = 0; + Sint i; + + if (endianess == am_big) { + if (bitoffs) { + for(i=0;i<size;++i) { + u <<=8; + u |= (((((Uint) bytes[i]) << bitoffs) | + (((Uint) bytes[i+1]) >> (8-bitoffs))) & 0xFF); + } + } else { + for(i=0;i<size;++i) { + u <<=8; + u |= bytes[i]; + } + } + } else { + + if (bitoffs) { + for(i=size-1;i>=0;--i) { + u <<=8; + u |= (((((Uint) bytes[i]) << bitoffs) | + (((Uint) bytes[i+1]) >> (8-bitoffs))) & 0xFF); + } + } else { + for(i=size-1;i>=0;--i) { + u <<=8; + u |= bytes[i]; + } + } + } + res = erts_make_integer(u,p); + BIF_RET(res); + } else { + /* Assume big, as we stripped away all zeroes from the MSB part of the binary */ + dsize_t num_parts = size / sizeof(ErtsDigit) + !!(size % sizeof(ErtsDigit)); + Eterm *bigp; + + bigp = HAlloc(p, BIG_NEED_SIZE(num_parts)); + *bigp = make_pos_bignum_header(num_parts); + res = make_big(bigp); + + if (endianess == am_big) { + Sint i,j; + ErtsDigit *d; + j = size; + d = &(BIG_DIGIT(bigp,num_parts - 1)); + *d = 0; + i = 0; + if(bitoffs) { + for (;;){ + (*d) <<= 8; + (*d) |= (((((Uint) bytes[i]) << bitoffs) | + (((Uint) bytes[i+1]) >> (8-bitoffs))) & 0xFF); + if (++i >= size) { + break; + } + if (!(--j % sizeof(ErtsDigit))) { + --d; + *d = 0; + } + } + } else { + for (;;){ + (*d) <<= 8; + (*d) |= bytes[i]; + if (++i >= size) { + break; + } + if (!(--j % sizeof(ErtsDigit))) { + --d; + *d = 0; + } + } + } + } else { + Sint i,j; + ErtsDigit *d; + j = size; + d = &(BIG_DIGIT(bigp,num_parts - 1)); + *d = 0; + i = size-1; + if (bitoffs) { + for (;;){ + (*d) <<= 8; + (*d) |= (((((Uint) bytes[i]) << bitoffs) | + (((Uint) bytes[i+1]) >> (8-bitoffs))) & 0xFF); + if (--i < 0) { + break; + } + if (!(--j % sizeof(ErtsDigit))) { + --d; + *d = 0; + } + } + } else { + for (;;){ + (*d) <<= 8; + (*d) |= bytes[i]; + if (--i < 0) { + break; + } + if (!(--j % sizeof(ErtsDigit))) { + --d; + *d = 0; + } + } + } + } + BIF_RET(res); + } + badarg: + BIF_ERROR(p,BADARG); +} + +BIF_RETTYPE binary_encode_unsigned_1(BIF_ALIST_1) +{ + return do_encode_unsigned(BIF_P,BIF_ARG_1,am_big); +} + +BIF_RETTYPE binary_encode_unsigned_2(BIF_ALIST_2) +{ + return do_encode_unsigned(BIF_P,BIF_ARG_1,BIF_ARG_2); +} + +BIF_RETTYPE binary_decode_unsigned_1(BIF_ALIST_1) +{ + return do_decode_unsigned(BIF_P,BIF_ARG_1,am_big); +} + +BIF_RETTYPE binary_decode_unsigned_2(BIF_ALIST_2) +{ + return do_decode_unsigned(BIF_P,BIF_ARG_1,BIF_ARG_2); +} + +/* + * Hard debug functions (dump) for the search structures + */ + +#ifdef HARDDEBUG +static void dump_bm_data(BMData *bm) +{ + int i,j; + erts_printf("Dumping Boyer-Moore structure.\n"); + erts_printf("=============================\n"); + erts_printf("Searchstring [%ld]:\n", bm->len); + erts_printf("<<"); + for (i = 0; i < bm->len; ++i) { + if (i > 0) { + erts_printf(", "); + } + erts_printf("%d", (int) bm->x[i]); + if (bm->x[i] >= 'A') { + erts_printf(" ($%c)",(char) bm->x[i]); + } + } + erts_printf(">>\n"); + erts_printf("GoodShift array:\n"); + for (i = 0; i < bm->len; ++i) { + erts_printf("GoodShift[%d]: %ld\n", i, bm->goodshift[i]); + } + erts_printf("BadShift array:\n"); + j = 0; + for (i = 0; i < ALPHABET_SIZE; i += j) { + for (j = 0; i + j < ALPHABET_SIZE && j < 6; ++j) { + erts_printf("BS[%03d]:%02ld, ", i+j, bm->badshift[i+j]); + } + erts_printf("\n"); + } +} + +static void dump_ac_node(ACNode *node, int indent, int ch) { + int i; + char *spaces = erts_alloc(ERTS_ALC_T_TMP, 10 * indent + 1); + memset(spaces,' ',10*indent); + spaces[10*indent] = '\0'; + erts_printf("%s-> %c\n",spaces,ch); + erts_printf("%sId: %u\n",spaces,(unsigned) node->id); + erts_printf("%sD: %u\n",spaces,(unsigned)node->d); + erts_printf("%sFinal: %d\n",spaces,(int)node->final); + erts_printf("%sFail: %u\n",spaces,(unsigned)node->h->id); + erts_free(ERTS_ALC_T_TMP,spaces); + for(i=0;i<ALPHABET_SIZE;++i) { + if (node->g[i] != NULL && node->g[i] != node) { + dump_ac_node(node->g[i],indent+1,i); + } + } +} + + +static void dump_ac_trie(ACTrie *act) +{ + erts_printf("Aho Corasick Trie dump.\n"); + erts_printf("=======================\n"); + erts_printf("Node counter: %u\n", (unsigned) act->idc); + erts_printf("Searchstring counter: %u\n", (unsigned) act->counter); + erts_printf("Trie:\n"); + dump_ac_node(act->root, 0, '0'); + return; +} +#endif diff --git a/erts/emulator/beam/erl_bif_chksum.c b/erts/emulator/beam/erl_bif_chksum.c index 445ba00ca7..06b7ffdf32 100644 --- a/erts/emulator/beam/erl_bif_chksum.c +++ b/erts/emulator/beam/erl_bif_chksum.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2008-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2008-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -49,9 +49,9 @@ void erts_init_bif_chksum(void) chksum_md5_2_exp.code[1] = am_atom_put("md5_trap",8); chksum_md5_2_exp.code[2] = 2; chksum_md5_2_exp.code[3] = - (Eterm) em_apply_bif; + (BeamInstr) em_apply_bif; chksum_md5_2_exp.code[4] = - (Eterm) &md5_2; + (BeamInstr) &md5_2; } diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index 9d5f0d9c02..2c2e283f65 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2009. All Rights Reserved. + * Copyright Ericsson AB 2006-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -1646,7 +1646,8 @@ static int do_unload_driver_entry(DE_Handle *dh, Eterm *save_name) if (save_name != NULL) { *save_name = mkatom(q->name); } - /* XXX:PaN Future locking problems? Don't dare to let go of the diver_list lock here!*/ + /* Future locking problems? Don't dare to let go of the + diver_list lock here!*/ if (q->finish) { int fpe_was_unmasked = erts_block_fpe(); (*(q->finish))(); diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c index 440b0b4f14..01e6977a2c 100644 --- a/erts/emulator/beam/erl_bif_guard.c +++ b/erts/emulator/beam/erl_bif_guard.c @@ -314,6 +314,30 @@ double_to_integer(Process* p, double x) return res; } +/******************************************************************************** + * binary_part guards. The actual implementation is in erl_bif_binary.c + ********************************************************************************/ +BIF_RETTYPE binary_part_3(BIF_ALIST_3) +{ + return erts_binary_part(BIF_P,BIF_ARG_1,BIF_ARG_2, BIF_ARG_3); +} + +BIF_RETTYPE binary_part_2(BIF_ALIST_2) +{ + Eterm *tp; + if (is_not_tuple(BIF_ARG_2)) { + goto badarg; + } + tp = tuple_val(BIF_ARG_2); + if (arityval(*tp) != 2) { + goto badarg; + } + return erts_binary_part(BIF_P,BIF_ARG_1,tp[1], tp[2]); + badarg: + BIF_ERROR(BIF_P,BADARG); +} + + /* * The following code is used when a guard that may build on the * heap is called directly. They must not use HAlloc(), but must @@ -630,3 +654,16 @@ gc_double_to_integer(Process* p, double x, Eterm* reg, Uint live) } return res; } + +/******************************************************************************** + * binary_part guards. The actual implementation is in erl_bif_binary.c + ********************************************************************************/ +Eterm erts_gc_binary_part_3(Process* p, Eterm* reg, Uint live) +{ + return erts_gc_binary_part(p,reg,live,0); +} + +Eterm erts_gc_binary_part_2(Process* p, Eterm* reg, Uint live) +{ + return erts_gc_binary_part(p,reg,live,1); +} diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index a34d400ed8..dace5b9297 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -38,9 +38,6 @@ #include "erl_instrument.h" #include "dist.h" #include "erl_gc.h" -#ifdef ELIB_ALLOC_IS_CLIB -#include "elib_stat.h" -#endif #ifdef HIPE #include "hipe_arch.h" #endif @@ -59,12 +56,19 @@ /* Keep erts_system_version as a global variable for easy access from a core */ static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE " (erts-" ERLANG_VERSION ")" +#if !HEAP_ON_C_STACK && !HALFWORD_HEAP + " [no-c-stack-objects]" +#endif #ifndef OTP_RELEASE " [source]" #endif #ifdef ARCH_64 +#if HALFWORD_HEAP + " [64-bit halfword]" +#else " [64-bit]" #endif +#endif #ifdef ERTS_SMP " [smp:%bpu:%bpu]" #endif @@ -115,22 +119,26 @@ static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE #endif static Eterm -bld_bin_list(Uint **hpp, Uint *szp, ProcBin* pb) +bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh) { + struct erl_off_heap_header* ohh; Eterm res = NIL; Eterm tuple; - for (; pb; pb = pb->next) { - Eterm val = erts_bld_uint(hpp, szp, (Uint) pb->val); - Eterm orig_size = erts_bld_uint(hpp, szp, pb->val->orig_size); - - if (szp) - *szp += 4+2; - if (hpp) { - Uint refc = (Uint) erts_smp_atomic_read(&pb->val->refc); - tuple = TUPLE3(*hpp, val, orig_size, make_small(refc)); - res = CONS(*hpp + 4, tuple, res); - *hpp += 4+2; + for (ohh = oh->first; ohh; ohh = ohh->next) { + if (ohh->thing_word == HEADER_PROC_BIN) { + ProcBin* pb = (ProcBin*) ohh; + Eterm val = erts_bld_uword(hpp, szp, (UWord) pb->val); + Eterm orig_size = erts_bld_uint(hpp, szp, pb->val->orig_size); + + if (szp) + *szp += 4+2; + if (hpp) { + Uint refc = (Uint) erts_smp_atomic_read(&pb->val->refc); + tuple = TUPLE3(*hpp, val, orig_size, make_small(refc)); + res = CONS(*hpp + 4, tuple, res); + *hpp += 4+2; + } } } return res; @@ -169,10 +177,10 @@ static void do_make_one_mon_element(ErtsMonitor *mon, void * vpmlc) Eterm tup; Eterm r = (IS_CONST(mon->ref) ? mon->ref - : STORE_NC(&(pmlc->hp), &MSO(pmlc->p).externals, mon->ref)); + : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->ref)); Eterm p = (IS_CONST(mon->pid) ? mon->pid - : STORE_NC(&(pmlc->hp), &MSO(pmlc->p).externals, mon->pid)); + : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->pid)); tup = TUPLE5(pmlc->hp, pmlc->tag, make_small(mon->type), r, p, mon->name); pmlc->hp += 6; pmlc->res = CONS(pmlc->hp, tup, pmlc->res); @@ -233,7 +241,7 @@ static void do_make_one_lnk_element(ErtsLink *lnk, void * vpllc) Eterm old_res, targets = NIL; Eterm p = (IS_CONST(lnk->pid) ? lnk->pid - : STORE_NC(&(pllc->hp), &MSO(pllc->p).externals, lnk->pid)); + : STORE_NC(&(pllc->hp), &MSO(pllc->p), lnk->pid)); if (lnk->type == LINK_NODE) { targets = make_small(ERTS_LINK_REFC(lnk)); } else if (ERTS_LINK_ROOT(lnk) != NULL) { @@ -624,12 +632,18 @@ static Eterm pi_1_keys[] = { #define ERTS_PI_1_NO_OF_KEYS (sizeof(pi_1_keys)/sizeof(Eterm)) static Eterm pi_1_keys_list; -static Uint pi_1_keys_list_heap[2*ERTS_PI_1_NO_OF_KEYS]; +#if HEAP_ON_C_STACK +static Eterm pi_1_keys_list_heap[2*ERTS_PI_1_NO_OF_KEYS]; +#endif static void process_info_init(void) { +#if HEAP_ON_C_STACK Eterm *hp = &pi_1_keys_list_heap[0]; +#else + Eterm *hp = erts_alloc(ERTS_ALC_T_LL_TEMP_TERM,sizeof(Eterm)*2*ERTS_PI_1_NO_OF_KEYS); +#endif int i; pi_1_keys_list = NIL; @@ -998,7 +1012,7 @@ process_info_aux(Process *BIF_P, hp = HAlloc(BIF_P, 3); res = am_undefined; } else { - Eterm* current; + BeamInstr* current; if (rp->current[0] == am_erlang && rp->current[1] == am_process_info && @@ -1127,9 +1141,9 @@ process_info_aux(Process *BIF_P, } else { /* Make our copy of the message */ - ASSERT(size_object(msg) == hfp->size); + ASSERT(size_object(msg) == hfp->used_size); msg = copy_struct(msg, - hfp->size, + hfp->used_size, &hp, &MSO(BIF_P)); } @@ -1212,7 +1226,7 @@ process_info_aux(Process *BIF_P, hp = HAlloc(BIF_P, 3 + mic.sz); res = NIL; for (i = 0; i < mic.mi_i; i++) { - item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); + item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); res = CONS(hp, item, res); hp += 2; } @@ -1245,9 +1259,7 @@ process_info_aux(Process *BIF_P, else { /* Monitor by pid. Build {process, Pid} and cons it. */ Eterm t; - Eterm pid = STORE_NC(&hp, - &MSO(BIF_P).externals, - mic.mi[i].entity); + Eterm pid = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); t = TUPLE2(hp, am_process, pid); hp += 3; res = CONS(hp, t, res); @@ -1269,7 +1281,7 @@ process_info_aux(Process *BIF_P, res = NIL; for (i = 0; i < mic.mi_i; ++i) { - item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); + item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); res = CONS(hp, item, res); hp += 2; } @@ -1478,7 +1490,7 @@ process_info_aux(Process *BIF_P, case am_group_leader: { int sz = NC_HEAP_SIZE(rp->group_leader); hp = HAlloc(BIF_P, 3 + sz); - res = STORE_NC(&hp, &MSO(BIF_P).externals, rp->group_leader); + res = STORE_NC(&hp, &MSO(BIF_P), rp->group_leader); break; } @@ -1503,9 +1515,9 @@ process_info_aux(Process *BIF_P, case am_binary: { Uint sz = 3; - (void) bld_bin_list(NULL, &sz, MSO(rp).mso); + (void) bld_bin_list(NULL, &sz, &MSO(rp)); hp = HAlloc(BIF_P, sz); - res = bld_bin_list(&hp, NULL, MSO(rp).mso); + res = bld_bin_list(&hp, NULL, &MSO(rp)); break; } @@ -1622,6 +1634,14 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ if (sel == am_allocator_sizes && arity == 2) { return erts_allocator_info_term(BIF_P, *tp, 1); + } else if (sel == am_wordsize && arity == 2) { + if (tp[0] == am_internal) { + return make_small(sizeof(Eterm)); + } + if (tp[0] == am_external) { + return make_small(sizeof(UWord)); + } + goto badarg; } else if (sel == am_allocated) { if (arity == 2) { Eterm res = THE_NON_VALUE; @@ -1871,6 +1891,37 @@ c_compiler_used(Eterm **hpp, Uint *szp) } +static int is_snif_term(Eterm module_atom) { + int i; + Atom *a = atom_tab(atom_val(module_atom)); + char *aname = (char *) a->name; + + /* if a->name has a '.' then the bif (snif) is bogus i.e a package */ + for (i = 0; i < a->len; i++) { + if (aname[i] == '.') + return 0; + } + + return 1; +} + +static Eterm build_snif_term(Eterm **hpp, Uint *szp, int ix, Eterm res) { + Eterm tup; + tup = erts_bld_tuple(hpp, szp, 3, bif_table[ix].module, bif_table[ix].name, make_small(bif_table[ix].arity)); + res = erts_bld_cons( hpp, szp, tup, res); + return res; +} + +static Eterm build_snifs_term(Eterm **hpp, Uint *szp, Eterm res) { + int i; + for (i = 0; i < BIF_SIZE; i++) { + if (is_snif_term(bif_table[i].module)) { + res = build_snif_term(hpp, szp, i, res); + } + } + return res; +} + BIF_RETTYPE system_info_1(BIF_ALIST_1) { Eterm res; @@ -1919,6 +1970,15 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(db_get_trace_control_word_0(BIF_P)); } else if (ERTS_IS_ATOM_STR("ets_realloc_moves", BIF_ARG_1)) { BIF_RET((erts_ets_realloc_always_moves) ? am_true : am_false); + } else if (ERTS_IS_ATOM_STR("snifs", BIF_ARG_1)) { + Uint size = 0; + Uint *szp; + + szp = &size; + build_snifs_term(NULL, szp, NIL); + hp = HAlloc(BIF_P, size); + res = build_snifs_term(&hp, NULL, NIL); + BIF_RET(res); } else if (BIF_ARG_1 == am_sequential_tracer) { val = erts_get_system_seq_tracer(); ASSERT(is_internal_pid(val) || is_internal_port(val) || val==am_false) @@ -2063,86 +2123,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(erts_alloc_util_allocators((void *) BIF_P)); } else if (BIF_ARG_1 == am_elib_malloc) { -#ifdef ELIB_ALLOC_IS_CLIB - struct elib_stat stat; - DECL_AM(heap_size); - DECL_AM(max_alloced_size); - DECL_AM(alloced_size); - DECL_AM(free_size); - DECL_AM(no_alloced_blocks); - DECL_AM(no_free_blocks); - DECL_AM(smallest_alloced_block); - DECL_AM(largest_free_block); - Eterm atoms[8]; - Eterm ints[8]; - Uint **hpp; - Uint sz; - Uint *szp; - int length; -#ifdef DEBUG - Uint *endp; -#endif - - elib_stat(&stat); - - /* First find out the heap size needed ... */ - hpp = NULL; - szp = &sz; - sz = 0; - - build_elib_malloc_term: - length = 0; - atoms[length] = AM_heap_size; - ints[length++] = erts_bld_uint(hpp, szp, - (Uint) stat.mem_total*sizeof(Uint)); - atoms[length] = AM_max_alloced_size; - ints[length++] = erts_bld_uint(hpp, szp, - (Uint) stat.mem_max_alloc*sizeof(Uint)); - atoms[length] = AM_alloced_size; - ints[length++] = erts_bld_uint(hpp, szp, - (Uint) stat.mem_alloc*sizeof(Uint)); - atoms[length] = AM_free_size; - ints[length++] = erts_bld_uint(hpp, szp, - (Uint) stat.mem_free*sizeof(Uint)); - atoms[length] = AM_no_alloced_blocks; - ints[length++] = erts_bld_uint(hpp, szp, (Uint) stat.mem_blocks); - atoms[length] = AM_no_free_blocks; - ints[length++] = erts_bld_uint(hpp, szp, (Uint) stat.free_blocks); - atoms[length] = AM_smallest_alloced_block; - ints[length++] = erts_bld_uint(hpp, szp, - (Uint) stat.min_used*sizeof(Uint)); - atoms[length] = AM_largest_free_block; - ints[length++] = erts_bld_uint(hpp, szp, - (Uint) stat.max_free*sizeof(Uint)); - - - - ASSERT(length <= sizeof(atoms)/sizeof(Eterm)); - ASSERT(length <= sizeof(ints)/sizeof(Eterm)); - - res = erts_bld_2tup_list(hpp, szp, length, atoms, ints); - - if (szp) { - /* ... and then build the term */ - hp = HAlloc(BIF_P, sz); -#ifdef DEBUG - endp = hp + sz; -#endif - - szp = NULL; - hpp = &hp; - goto build_elib_malloc_term; - } - -#ifdef DEBUG - ASSERT(endp == hp); -#endif - -#else /* #ifdef ELIB_ALLOC_IS_CLIB */ - res = am_false; -#endif /* #ifdef ELIB_ALLOC_IS_CLIB */ - - BIF_RET(res); + /* To be removed in R15 */ + BIF_RET(am_false); } else if (BIF_ARG_1 == am_os_version) { int major, minor, build; @@ -2253,6 +2235,15 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } else if (ERTS_IS_ATOM_STR("cpu_topology", BIF_ARG_1)) { res = erts_get_cpu_topology_term(BIF_P, am_used); BIF_TRAP1(erts_format_cpu_topology_trap, BIF_P, res); + } else if (ERTS_IS_ATOM_STR("update_cpu_info", BIF_ARG_1)) { + if (erts_update_cpu_info()) { + ERTS_DECL_AM(changed); + BIF_RET(AM_changed); + } + else { + ERTS_DECL_AM(unchanged); + BIF_RET(AM_unchanged); + } #if defined(__GNUC__) && defined(HAVE_SOLARIS_SPARC_PERFMON) } else if (ERTS_IS_ATOM_STR("ultrasparc_read_tick1", BIF_ARG_1)) { register unsigned high asm("%l0"); @@ -2324,7 +2315,10 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } /* Arguments that are unusual follow ... */ else if (ERTS_IS_ATOM_STR("logical_processors", BIF_ARG_1)) { - int no = erts_get_cpu_configured(erts_cpuinfo); + int no; + erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); + no = erts_get_cpu_configured(erts_cpuinfo); + erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); if (no > 0) BIF_RET(make_small((Uint) no)); else { @@ -2333,7 +2327,10 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } } else if (ERTS_IS_ATOM_STR("logical_processors_online", BIF_ARG_1)) { - int no = erts_get_cpu_online(erts_cpuinfo); + int no; + erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); + no = erts_get_cpu_online(erts_cpuinfo); + erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); if (no > 0) BIF_RET(make_small((Uint) no)); else { @@ -2342,7 +2339,10 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } } else if (ERTS_IS_ATOM_STR("logical_processors_available", BIF_ARG_1)) { - int no = erts_get_cpu_available(erts_cpuinfo); + int no; + erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); + no = erts_get_cpu_available(erts_cpuinfo); + erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); if (no > 0) BIF_RET(make_small((Uint) no)); else { @@ -2502,6 +2502,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(erts_sched_stat_term(BIF_P, 1)); } else if (ERTS_IS_ATOM_STR("taints", BIF_ARG_1)) { BIF_RET(erts_nif_taints(BIF_P)); + } else if (ERTS_IS_ATOM_STR("reader_groups_map", BIF_ARG_1)) { + BIF_RET(erts_get_reader_groups_map(BIF_P)); } BIF_ERROR(BIF_P, BADARG); @@ -2617,7 +2619,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) hp = HAlloc(BIF_P, 3 + mic.sz); res = NIL; for (i = 0; i < mic.mi_i; i++) { - item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); + item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); res = CONS(hp, item, res); hp += 2; } @@ -2637,7 +2639,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) res = NIL; for (i = 0; i < mic.mi_i; i++) { Eterm t; - item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); + item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); t = TUPLE2(hp, am_process, item); hp += 3; res = CONS(hp, t, res); @@ -2694,7 +2696,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) erts_doforall_links(prt->nlinks, &one_link_size, &size); for (bp = prt->bp; bp; bp = bp->next) - size += sizeof(ErlHeapFragment) + (bp->size - 1)*sizeof(Eterm); + size += sizeof(ErlHeapFragment) + (bp->alloc_size - 1)*sizeof(Eterm); if (prt->linebuf) size += sizeof(LineBuf) + prt->linebuf->ovsiz; @@ -2817,7 +2819,7 @@ fun_info_2(Process* p, Eterm fun, Eterm what) goto error; } } else if (is_export(fun)) { - Export* exp = (Export *) (export_val(fun))[1]; + Export* exp = (Export *) ((UWord) (export_val(fun))[1]); switch (what) { case am_type: hp = HAlloc(p, 3); @@ -3010,11 +3012,11 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) res = erts_run_queues_len(NULL); BIF_RET(make_small(res)); } else if (BIF_ARG_1 == am_wall_clock) { - Uint w1, w2; + UWord w1, w2; Eterm b1, b2; wall_clock_elapsed_time_both(&w1, &w2); - b1 = erts_make_integer(w1,BIF_P); - b2 = erts_make_integer(w2,BIF_P); + b1 = erts_make_integer((Uint) w1,BIF_P); + b2 = erts_make_integer((Uint) w2,BIF_P); hp = HAlloc(BIF_P,3); res = TUPLE2(hp, b1, b2); BIF_RET(res); @@ -3365,6 +3367,16 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("fake_scheduler_bindings", tp[1])) { return erts_fake_scheduler_bindings(BIF_P, tp[2]); } + else if (ERTS_IS_ATOM_STR("reader_groups_map", tp[1])) { + Sint groups; + if (is_not_small(tp[2])) + BIF_ERROR(BIF_P, BADARG); + groups = signed_val(tp[2]); + if (groups < (Sint) 1 || groups > (Sint) INT_MAX) + BIF_ERROR(BIF_P, BADARG); + + BIF_RET(erts_debug_reader_groups_map(BIF_P, (int) groups)); + } break; } default: @@ -3546,6 +3558,17 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } } } + else if (ERTS_IS_ATOM_STR("binary_loop_limit", BIF_ARG_1)) { + /* Used by binary_module_SUITE (stdlib) */ + Uint max_loops; + if (is_atom(BIF_ARG_2) && ERTS_IS_ATOM_STR("default", BIF_ARG_2)) { + max_loops = erts_binary_set_loop_limit(-1); + BIF_RET(make_small(max_loops)); + } else if (term_to_Uint(BIF_ARG_2, &max_loops) != 0) { + max_loops = erts_binary_set_loop_limit(max_loops); + BIF_RET(make_small(max_loops)); + } + } else if (ERTS_IS_ATOM_STR("re_loop_limit", BIF_ARG_1)) { /* Used by re_SUITE (stdlib) */ Uint max_loops; @@ -3658,8 +3681,8 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s * [{{file, line}, {tries, colls, {seconds, nanoseconds, n_blocks}}}] */ - ethr_atomic_read(&stats->tries, (long *)&tries); - ethr_atomic_read(&stats->colls, (long *)&colls); + tries = (unsigned long) ethr_atomic_read(&stats->tries); + colls = (unsigned long) ethr_atomic_read(&stats->colls); line = stats->line; timer_s = stats->timer.s; diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c index a9e8dd86f7..ce13469801 100644 --- a/erts/emulator/beam/erl_bif_lists.c +++ b/erts/emulator/beam/erl_bif_lists.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -89,13 +89,14 @@ BIF_RETTYPE append_2(BIF_ALIST_2) BIF_RET(copy); } +#define SMALL_VEC_SIZE 10 BIF_RETTYPE subtract_2(BIF_ALIST_2) { Eterm list; Eterm* hp; Uint need; Eterm res; - Eterm small_vec[10]; /* Preallocated memory for small lists */ + Eterm small_vec[SMALL_VEC_SIZE]; /* Preallocated memory for small lists */ Eterm* vec_p; Eterm* vp; int i; @@ -115,7 +116,7 @@ BIF_RETTYPE subtract_2(BIF_ALIST_2) BIF_RET(BIF_ARG_1); /* allocate element vector */ - if (n <= sizeof(small_vec)/sizeof(small_vec[0])) + if (n <= SMALL_VEC_SIZE) vec_p = small_vec; else vec_p = (Eterm*) erts_alloc(ERTS_ALC_T_TMP, n * sizeof(Eterm)); diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c index 6da72dcef9..deda7adc1f 100644 --- a/erts/emulator/beam/erl_bif_op.c +++ b/erts/emulator/beam/erl_bif_op.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -251,7 +251,7 @@ BIF_RETTYPE is_function_2(BIF_ALIST_2) BIF_RET(am_true); } } else if (is_export(BIF_ARG_1)) { - Export* exp = (Export *) (export_val(BIF_ARG_1))[1]; + Export* exp = (Export *) EXPAND_POINTER((export_val(BIF_ARG_1))[1]); if (exp->code[2] == (Uint) arity) { BIF_RET(am_true); diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index f454f2e12d..378c5e73fd 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2001-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -21,10 +21,6 @@ # include "config.h" #endif -#ifdef _OSE_ -# include "ose.h" -#endif - #include <ctype.h> #define ERTS_WANT_EXTERNAL_TAGS @@ -583,8 +579,8 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1) if (prt->bp == NULL) { /* MUST be CONST! */ res = prt->data; } else { - Eterm* hp = HAlloc(BIF_P, prt->bp->size); - res = copy_struct(prt->data, prt->bp->size, &hp, &MSO(BIF_P)); + Eterm* hp = HAlloc(BIF_P, prt->bp->used_size); + res = copy_struct(prt->data, prt->bp->used_size, &hp, &MSO(BIF_P)); } erts_smp_port_unlock(prt); BIF_RET(res); @@ -716,15 +712,17 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) } } else if (option == am_cd) { Eterm iolist; - Eterm heap[4]; + DeclareTmpHeap(heap,4,p); int r; + UseTmpHeap(4,p); heap[0] = *tp; heap[1] = make_list(heap+2); heap[2] = make_small(0); heap[3] = NIL; iolist = make_list(heap); r = io_list_to_buf(iolist, (char*) dir, MAXPATHLEN); + UnUseTmpHeap(4,p); if (r < 0) { goto badarg; } @@ -1077,27 +1075,33 @@ struct packet_callback_args Eterm res; /* Out */ int string_as_bin; /* return strings as binaries (http_bin): */ byte* aligned_ptr; + Uint bin_sz; Eterm orig; Uint bin_offs; byte bin_bitoffs; }; +#define in_area(ptr,start,nbytes) \ + ((unsigned long)((char*)(ptr) - (char*)(start)) < (nbytes)) + static Eterm http_bld_string(struct packet_callback_args* pca, Uint **hpp, Uint *szp, const char *str, Sint len) { Eterm res = THE_NON_VALUE; Uint size; + int make_subbin; if (pca->string_as_bin) { size = heap_bin_size(len); - + make_subbin = (size > ERL_SUB_BIN_SIZE + && in_area(str, pca->aligned_ptr, pca->bin_sz)); if (szp) { - *szp += (size > ERL_SUB_BIN_SIZE) ? ERL_SUB_BIN_SIZE : size; + *szp += make_subbin ? ERL_SUB_BIN_SIZE : size; } if (hpp) { res = make_binary(*hpp); - if (size > ERL_SUB_BIN_SIZE) { + if (make_subbin) { ErlSubBin* bin = (ErlSubBin*) *hpp; bin->thing_word = HEADER_SUB_BIN; bin->size = len; @@ -1328,7 +1332,7 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3) int packet_sz; /*-------Binaries involved: ------------------*/ byte* bin_ptr; /*| orig: original binary */ byte bin_bitsz; /*| bin: BIF_ARG_2, may be sub-binary of orig */ - Uint bin_sz; /*| packet: prefix of bin */ + /*| packet: prefix of bin */ char* body_ptr; /*| body: part of packet to return */ int body_sz; /*| rest: bin without packet */ struct packet_callback_args pca; @@ -1389,18 +1393,18 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3) } - bin_sz = binary_size(BIF_ARG_2); + pca.bin_sz = binary_size(BIF_ARG_2); ERTS_GET_BINARY_BYTES(BIF_ARG_2, bin_ptr, pca.bin_bitoffs, bin_bitsz); if (pca.bin_bitoffs != 0) { - pca.aligned_ptr = erts_alloc(ERTS_ALC_T_TMP, bin_sz); - erts_copy_bits(bin_ptr, pca.bin_bitoffs, 1, pca.aligned_ptr, 0, 1, bin_sz*8); + pca.aligned_ptr = erts_alloc(ERTS_ALC_T_TMP, pca.bin_sz); + erts_copy_bits(bin_ptr, pca.bin_bitoffs, 1, pca.aligned_ptr, 0, 1, pca.bin_sz*8); } else { pca.aligned_ptr = bin_ptr; } - packet_sz = packet_get_length(type, (char*)pca.aligned_ptr, bin_sz, + packet_sz = packet_get_length(type, (char*)pca.aligned_ptr, pca.bin_sz, max_plen, trunc_len, &http_state); - if (!(packet_sz > 0 && packet_sz <= bin_sz)) { + if (!(packet_sz > 0 && packet_sz <= pca.bin_sz)) { if (packet_sz < 0) { goto error; } @@ -1456,7 +1460,7 @@ error: rest = (ErlSubBin *) hp; rest->thing_word = HEADER_SUB_BIN; - rest->size = bin_sz - packet_sz; + rest->size = pca.bin_sz - packet_sz; rest->offs = pca.bin_offs + packet_sz; rest->orig = pca.orig; rest->bitoffs = pca.bin_bitoffs; diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index c027cd5984..d4a8a3aaa7 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -76,8 +76,8 @@ void erts_init_bif_re(void) re_exec_trap_export.code[0] = am_erlang; re_exec_trap_export.code[1] = am_re_run_trap; re_exec_trap_export.code[2] = 3; - re_exec_trap_export.code[3] = (Eterm) em_apply_bif; - re_exec_trap_export.code[4] = (Eterm) &re_exec_trap; + re_exec_trap_export.code[3] = (BeamInstr) em_apply_bif; + re_exec_trap_export.code[4] = (BeamInstr) &re_exec_trap; grun_trap_exportp = erts_export_put(am_re,am_grun,3); urun_trap_exportp = erts_export_put(am_re,am_urun,3); @@ -103,7 +103,7 @@ Sint erts_re_set_loop_limit(Sint limit) static int term_to_int(Eterm term, int *sp) { -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP if (is_small(term)) { Uint x = signed_val(term); @@ -154,7 +154,7 @@ static int term_to_int(Eterm term, int *sp) static Eterm make_signed_integer(int x, Process *p) { -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP return make_small(x); #else Eterm* hp; diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c index 172bb37952..4ae2f6ebf4 100644 --- a/erts/emulator/beam/erl_bif_timer.c +++ b/erts/emulator/beam/erl_bif_timer.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -155,7 +155,7 @@ create_ref(Uint *hp, Uint32 *ref_numbers, Uint32 len) erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); } -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP hp[0] = make_ref_thing_header(len/2 + 1); datap = (Uint32 *) &hp[1]; *(datap++) = len; @@ -173,13 +173,13 @@ create_ref(Uint *hp, Uint32 *ref_numbers, Uint32 len) static int eq_non_standard_ref_numbers(Uint32 *rn1, Uint32 len1, Uint32 *rn2, Uint32 len2) { -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP #define MAX_REF_HEAP_SZ (1+(ERTS_MAX_REF_NUMBERS/2+1)) #else #define MAX_REF_HEAP_SZ (1+ERTS_MAX_REF_NUMBERS) #endif - Uint r1_hp[MAX_REF_HEAP_SZ]; - Uint r2_hp[MAX_REF_HEAP_SZ]; + DeclareTmpHeapNoproc(r1_hp,(MAX_REF_HEAP_SZ * 2)); + Eterm *r2_hp = r1_hp +MAX_REF_HEAP_SZ; return eq(create_ref(r1_hp, rn1, len1), create_ref(r2_hp, rn2, len2)); #undef MAX_REF_HEAP_SZ @@ -357,7 +357,7 @@ bif_timer_timeout(ErtsBifTimer* btm) rp, &rp_locks); } else { - Eterm old_size = bp->size; + Eterm old_size = bp->used_size; bp = erts_resize_message_buffer(bp, old_size + wrap_size, &message, 1); hp = &bp->mem[0] + old_size; @@ -398,7 +398,7 @@ setup_bif_timer(Uint32 xflags, if (!term_to_Uint(time, &timeout)) return THE_NON_VALUE; -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP if ((timeout >> 32) != 0) return THE_NON_VALUE; #endif diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 7dff5e0eeb..0509e51a6f 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -40,8 +40,7 @@ #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) -static erts_smp_mtx_t trace_pattern_mutex; -const struct trace_pattern_flags erts_trace_pattern_flags_off = {0, 0, 0, 0}; +const struct trace_pattern_flags erts_trace_pattern_flags_off = {0, 0, 0, 0, 0}; static int erts_default_trace_pattern_is_on; static Binary *erts_default_match_spec; static Binary *erts_default_meta_match_spec; @@ -65,7 +64,6 @@ static void clear_trace_bif(int bif_index); void erts_bif_trace_init(void) { - erts_smp_mtx_init(&trace_pattern_mutex, "trace_pattern"); erts_default_trace_pattern_is_on = 0; erts_default_match_spec = NULL; erts_default_meta_match_spec = NULL; @@ -86,7 +84,7 @@ trace_pattern_2(Process* p, Eterm MFA, Eterm Pattern) Eterm trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) { - Eterm mfa[3]; + DeclareTmpHeap(mfa,3,p); /* Not really heap here, but might be when setting pattern */ int i; int matches = 0; int specified = 0; @@ -101,6 +99,7 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); erts_smp_block_system(0); + UseTmpHeap(3,p); /* * Check and compile the match specification. */ @@ -185,6 +184,14 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) flags.breakpoint = 1; flags.call_count = 1; break; + case am_call_time: + if (is_global) { + goto error; + } + flags.breakpoint = 1; + flags.call_time = 1; + break; + default: goto error; } @@ -194,8 +201,8 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) goto error; } - if (match_prog_set && !flags.local && !flags.meta && flags.call_count) { - /* A match prog is not allowed with just call_count */ + if (match_prog_set && !flags.local && !flags.meta && (flags.call_count || flags.call_time)) { + /* A match prog is not allowed with just call_count or call_time*/ goto error; } @@ -234,6 +241,8 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) |= flags.meta; erts_default_trace_pattern_flags.call_count |= (on == 1) ? flags.call_count : 0; + erts_default_trace_pattern_flags.call_time + |= (on == 1) ? flags.call_time : 0; } else { erts_default_trace_pattern_flags.local &= ~flags.local; @@ -241,10 +250,13 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) &= ~flags.meta; erts_default_trace_pattern_flags.call_count &= ~flags.call_count; + erts_default_trace_pattern_flags.call_time + &= ~flags.call_time; if (! (erts_default_trace_pattern_flags.breakpoint = erts_default_trace_pattern_flags.local | erts_default_trace_pattern_flags.meta | - erts_default_trace_pattern_flags.call_count)) { + erts_default_trace_pattern_flags.call_count | + erts_default_trace_pattern_flags.call_time)) { erts_default_trace_pattern_is_on = !!on; /* i.e off */ } } @@ -266,8 +278,9 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) if (on) { if (on != 1) { flags.call_count = 0; + flags.call_time = 0; } - flags.breakpoint = flags.local | flags.meta | flags.call_count; + flags.breakpoint = flags.local | flags.meta | flags.call_count | flags.call_time; erts_default_trace_pattern_flags = flags; /* Struct copy */ erts_default_trace_pattern_is_on = !!flags.breakpoint; } @@ -312,7 +325,7 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) MatchSetUnref(match_prog_set); done: - + UnUseTmpHeap(3,p); erts_smp_release_system(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); @@ -322,6 +335,7 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) MatchSetUnref(match_prog_set); + UnUseTmpHeap(3,p); erts_smp_release_system(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); BIF_ERROR(p, BADARG); @@ -334,7 +348,6 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on, struct trace_pattern_flags *trace_pattern_flags, Eterm *meta_tracer_pid) { - erts_smp_mtx_lock(&trace_pattern_mutex); if (trace_pattern_is_on) *trace_pattern_is_on = erts_default_trace_pattern_is_on; if (match_spec) @@ -345,12 +358,10 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on, *trace_pattern_flags = erts_default_trace_pattern_flags; if (meta_tracer_pid) *meta_tracer_pid = erts_default_meta_tracer_pid; - erts_smp_mtx_unlock(&trace_pattern_mutex); } - Uint erts_trace_flag2bit(Eterm flag) { @@ -378,7 +389,7 @@ erts_trace_flag2bit(Eterm flag) default: return 0; } } - + /* Scan the argument list and sort out the trace flags. ** ** Returns !0 on success, 0 on failure. @@ -929,6 +940,7 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) #define FUNC_TRACE_LOCAL_TRACE (1<<2) #define FUNC_TRACE_META_TRACE (1<<3) #define FUNC_TRACE_COUNT_TRACE (1<<4) +#define FUNC_TRACE_TIME_TRACE (1<<5) /* * Returns either FUNC_TRACE_NOEXIST, FUNC_TRACE_UNTRACED, * FUNC_TRACE_GLOBAL_TRACE, or, @@ -943,16 +955,18 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) * * If the return value contains FUNC_TRACE_COUNT_TRACE, *count is set. */ -static int function_is_traced(Eterm mfa[3], - Binary **ms, /* out */ - Binary **ms_meta, /* out */ +static int function_is_traced(Process *p, + Eterm mfa[3], + Binary **ms, /* out */ + Binary **ms_meta, /* out */ Eterm *tracer_pid_meta, /* out */ - Sint *count) /* out */ + Sint *count, /* out */ + Eterm *call_time) /* out */ { Export e; Export* ep; int i; - Uint *code; + BeamInstr *code; /* First look for an export entry */ e.code[0] = mfa[0]; @@ -960,12 +974,12 @@ static int function_is_traced(Eterm mfa[3], e.code[2] = mfa[2]; if ((ep = export_get(&e)) != NULL) { if (ep->address == ep->code+3 && - ep->code[3] != (Uint) em_call_error_handler) { - if (ep->code[3] == (Uint) em_call_traced_function) { + ep->code[3] != (BeamInstr) em_call_error_handler) { + if (ep->code[3] == (BeamInstr) em_call_traced_function) { *ms = ep->match_prog_set; return FUNC_TRACE_GLOBAL_TRACE; } - if (ep->code[3] == (Uint) em_apply_bif) { + if (ep->code[3] == (BeamInstr) em_apply_bif) { for (i = 0; i < BIF_SIZE; ++i) { if (bif_export[i] == ep) { int r = 0; @@ -978,10 +992,13 @@ static int function_is_traced(Eterm mfa[3], r |= FUNC_TRACE_LOCAL_TRACE; *ms = ep->match_prog_set; } - if (erts_is_mtrace_bif(ep->code+3, ms_meta, + if (erts_is_mtrace_break(ep->code+3, ms_meta, tracer_pid_meta)) { r |= FUNC_TRACE_META_TRACE; } + if (erts_is_time_break(p, ep->code+3, call_time)) { + r |= FUNC_TRACE_TIME_TRACE; + } } return r ? r : FUNC_TRACE_UNTRACED; } @@ -999,7 +1016,9 @@ static int function_is_traced(Eterm mfa[3], | (erts_is_mtrace_break(code, ms_meta, tracer_pid_meta) ? FUNC_TRACE_META_TRACE : 0) | (erts_is_count_break(code, count) - ? FUNC_TRACE_COUNT_TRACE : 0); + ? FUNC_TRACE_COUNT_TRACE : 0) + | (erts_is_time_break(p, code, call_time) + ? FUNC_TRACE_TIME_TRACE : 0); return r ? r : FUNC_TRACE_UNTRACED; } @@ -1011,15 +1030,19 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) { Eterm* tp; Eterm* hp; - Eterm mfa[3]; + DeclareTmpHeap(mfa,3,p); /* Not really heap here, but might be when setting pattern */ Binary *ms = NULL, *ms_meta = NULL; Sint count = 0; Eterm traced = am_false; Eterm match_spec = am_false; Eterm retval = am_false; Eterm meta = am_false; + Eterm call_time = NIL; int r; + + UseTmpHeap(3,p); + if (!is_tuple(func_spec)) { goto error; } @@ -1034,12 +1057,29 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) mfa[1] = tp[2]; mfa[2] = signed_val(tp[3]); - r = function_is_traced(mfa, &ms, &ms_meta, &meta, &count); +#ifdef ERTS_SMP + if ( (key == am_call_time) || (key == am_all)) { + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); + erts_smp_block_system(0); + } +#endif + + r = function_is_traced(p, mfa, &ms, &ms_meta, &meta, &count, &call_time); + +#ifdef ERTS_SMP + if ( (key == am_call_time) || (key == am_all)) { + erts_smp_release_system(); + erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + } +#endif + switch (r) { case FUNC_TRACE_NOEXIST: + UnUseTmpHeap(3,p); hp = HAlloc(p, 3); return TUPLE2(hp, key, am_undefined); case FUNC_TRACE_UNTRACED: + UnUseTmpHeap(3,p); hp = HAlloc(p, 3); return TUPLE2(hp, key, am_false); case FUNC_TRACE_GLOBAL_TRACE: @@ -1085,8 +1125,13 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) erts_make_integer(count, p); } break; + case am_call_time: + if (r & FUNC_TRACE_TIME_TRACE) { + retval = call_time; + } + break; case am_all: { - Eterm match_spec_meta = am_false, c = am_false, t; + Eterm match_spec_meta = am_false, c = am_false, t, ct = am_false; if (ms) { match_spec = MatchSetGetSource(ms); @@ -1104,10 +1149,15 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) erts_make_integer(-count-1, p) : erts_make_integer(count, p); } - hp = HAlloc(p, (3+2)*5); + if (r & FUNC_TRACE_TIME_TRACE) { + ct = call_time; + } + hp = HAlloc(p, (3+2)*6); retval = NIL; t = TUPLE2(hp, am_call_count, c); hp += 3; retval = CONS(hp, t, retval); hp += 2; + t = TUPLE2(hp, am_call_time, ct); hp += 3; + retval = CONS(hp, t, retval); hp += 2; t = TUPLE2(hp, am_meta_match_spec, match_spec_meta); hp += 3; retval = CONS(hp, t, retval); hp += 2; t = TUPLE2(hp, am_meta, meta); hp += 3; @@ -1120,10 +1170,12 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) default: goto error; } + UnUseTmpHeap(3,p); hp = HAlloc(p, 3); return TUPLE2(hp, key, retval); error: + UnUseTmpHeap(3,p); BIF_ERROR(p, BADARG); } @@ -1201,6 +1253,13 @@ trace_info_on_load(Process* p, Eterm key) } else { return TUPLE2(hp, key, am_false); } + case am_call_time: + hp = HAlloc(p, 3); + if (erts_default_trace_pattern_flags.call_time) { + return TUPLE2(hp, key, am_true); + } else { + return TUPLE2(hp, key, am_false); + } case am_all: { Eterm match_spec = am_false, meta_match_spec = am_false, r = NIL, t; @@ -1275,6 +1334,7 @@ erts_set_trace_pattern(Eterm* mfa, int specified, for (j = 0; j < specified && mfa[j] == ep->code[j]; j++) { /* Empty loop body */ } + if (j == specified) { if (on) { if (! flags.breakpoint) @@ -1312,7 +1372,7 @@ erts_set_trace_pattern(Eterm* mfa, int specified, if (erts_bif_trace_flags[i] & BIF_TRACE_AS_META) { ASSERT(ExportIsBuiltIn(bif_export[i])); erts_clear_mtrace_bif - ((Uint *)bif_export[i]->code + 3); + ((BeamInstr *)bif_export[i]->code + 3); erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_META; } set_trace_bif(i, match_prog_set); @@ -1341,12 +1401,18 @@ erts_set_trace_pattern(Eterm* mfa, int specified, } if (flags.meta) { erts_set_mtrace_bif - ((Uint *)bif_export[i]->code + 3, + ((BeamInstr *)bif_export[i]->code + 3, meta_match_prog_set, meta_tracer_pid); erts_bif_trace_flags[i] |= BIF_TRACE_AS_META; erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_GLOBAL; m = 1; } + if (flags.call_time) { + erts_set_time_trace_bif(bif_export[i]->code + 3, on); + /* I don't want to remove any other tracers */ + erts_bif_trace_flags[i] |= BIF_TRACE_AS_CALL_TIME; + m = 1; + } if (erts_bif_trace_flags[i]) { setup_bif_trace(i); } @@ -1361,11 +1427,16 @@ erts_set_trace_pattern(Eterm* mfa, int specified, if (flags.meta) { if (erts_bif_trace_flags[i] & BIF_TRACE_AS_META) { erts_clear_mtrace_bif - ((Uint *)bif_export[i]->code + 3); + ((BeamInstr *)bif_export[i]->code + 3); erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_META; } m = 1; } + if (flags.call_time) { + erts_clear_time_trace_bif(bif_export[i]->code + 3); + erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_CALL_TIME; + m = 1; + } if (! erts_bif_trace_flags[i]) { reset_bif_trace(i); } @@ -1383,6 +1454,7 @@ erts_set_trace_pattern(Eterm* mfa, int specified, erts_clear_trace_break(mfa, specified); erts_clear_mtrace_break(mfa, specified); erts_clear_count_break(mfa, specified); + erts_clear_time_break(mfa, specified); } else { int m = 0; if (flags.local) { @@ -1396,6 +1468,9 @@ erts_set_trace_pattern(Eterm* mfa, int specified, if (flags.call_count) { m = erts_set_count_break(mfa, specified, on); } + if (flags.call_time) { + m = erts_set_time_break(mfa, specified, on); + } /* All assignments to 'm' above should give the same value, * so just use the last */ matches += m; @@ -1411,6 +1486,9 @@ erts_set_trace_pattern(Eterm* mfa, int specified, if (flags.call_count) { m = erts_clear_count_break(mfa, specified); } + if (flags.call_time) { + m = erts_clear_time_break(mfa, specified); + } /* All assignments to 'm' above should give the same value, * so just use the last */ matches += m; @@ -1430,9 +1508,9 @@ static int setup_func_trace(Export* ep, void* match_prog) { if (ep->address == ep->code+3) { - if (ep->code[3] == (Uint) em_call_error_handler) { + if (ep->code[3] == (BeamInstr) em_call_error_handler) { return 0; - } else if (ep->code[3] == (Uint) em_call_traced_function) { + } else if (ep->code[3] == (BeamInstr) em_call_traced_function) { MatchSetUnref(ep->match_prog_set); ep->match_prog_set = match_prog; MatchSetRef(ep->match_prog_set); @@ -1452,8 +1530,8 @@ setup_func_trace(Export* ep, void* match_prog) return 0; } - ep->code[3] = (Uint) em_call_traced_function; - ep->code[4] = (Uint) ep->address; + ep->code[3] = (BeamInstr) em_call_traced_function; + ep->code[4] = (BeamInstr) ep->address; ep->address = ep->code+3; ep->match_prog_set = match_prog; MatchSetRef(ep->match_prog_set); @@ -1465,7 +1543,7 @@ static void setup_bif_trace(int bif_index) { ASSERT(ExportIsBuiltIn(ep)); ASSERT(ep->code[4]); - ep->code[4] = (Uint) bif_table[bif_index].traced; + ep->code[4] = (BeamInstr) bif_table[bif_index].traced; } static void set_trace_bif(int bif_index, void* match_prog) { @@ -1492,9 +1570,9 @@ static int reset_func_trace(Export* ep) { if (ep->address == ep->code+3) { - if (ep->code[3] == (Uint) em_call_error_handler) { + if (ep->code[3] == (BeamInstr) em_call_error_handler) { return 0; - } else if (ep->code[3] == (Uint) em_call_traced_function) { + } else if (ep->code[3] == (BeamInstr) em_call_traced_function) { ep->address = (Uint *) ep->code[4]; MatchSetUnref(ep->match_prog_set); ep->match_prog_set = NULL; @@ -1527,8 +1605,8 @@ static void reset_bif_trace(int bif_index) { ASSERT(ExportIsBuiltIn(ep)); ASSERT(ep->code[4]); ASSERT(! ep->match_prog_set); - ASSERT(! erts_is_mtrace_bif((Uint *)ep->code+3, NULL, NULL)); - ep->code[4] = (Uint) bif_table[bif_index].f; + ASSERT(! erts_is_mtrace_break((BeamInstr *)ep->code+3, NULL, NULL)); + ep->code[4] = (BeamInstr) bif_table[bif_index].f; } static void clear_trace_bif(int bif_index) { @@ -2083,7 +2161,7 @@ trace_delivered_1(BIF_ALIST_1) #ifdef ERTS_SMP bp = new_message_buffer(REF_THING_SIZE + 4); hp = &bp->mem[0]; - msg_ref = STORE_NC(&hp, &bp->off_heap.externals, ref); + msg_ref = STORE_NC(&hp, &bp->off_heap, ref); #else hp = HAlloc(BIF_P, 4); msg_ref = ref; diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index 1f948a9684..a569fe2e85 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -21,6 +21,7 @@ #define __ERL_BINARY_H #include "erl_threads.h" +#include "bif.h" /* * Maximum number of bytes to place in a heap binary. @@ -150,7 +151,16 @@ do { \ void erts_init_binary(void); -byte* erts_get_aligned_binary_bytes_extra(Eterm, byte**, unsigned extra); +byte* erts_get_aligned_binary_bytes_extra(Eterm, byte**, ErtsAlcType_t, unsigned extra); + +/* + * Common implementation for erlang:list_to_binary/1 and binary:list_to_bin/1 + */ + +BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg); +BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is_tuple); +BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen); + #if defined(__i386__) || !defined(__GNUC__) /* @@ -164,10 +174,11 @@ byte* erts_get_aligned_binary_bytes_extra(Eterm, byte**, unsigned extra); #endif #define ERTS_CHK_BIN_ALIGNMENT(B) \ - do { ASSERT(!(B) || (((Uint) &((Binary *)(B))->orig_bytes[0]) & ERTS_BIN_ALIGNMENT_MASK) == ((Uint) 0)) } while(0) + do { ASSERT(!(B) || (((UWord) &((Binary *)(B))->orig_bytes[0]) & ERTS_BIN_ALIGNMENT_MASK) == ((UWord) 0)) } while(0) ERTS_GLB_INLINE byte* erts_get_aligned_binary_bytes(Eterm bin, byte** base_ptr); ERTS_GLB_INLINE void erts_free_aligned_binary_bytes(byte* buf); +ERTS_GLB_INLINE void erts_free_aligned_binary_bytes_extra(byte* buf, ErtsAlcType_t); ERTS_GLB_INLINE Binary *erts_bin_drv_alloc_fnf(Uint size); ERTS_GLB_INLINE Binary *erts_bin_drv_alloc(Uint size); ERTS_GLB_INLINE Binary *erts_bin_nrml_alloc(Uint size); @@ -184,17 +195,23 @@ ERTS_GLB_INLINE Binary *erts_create_magic_binary(Uint size, ERTS_GLB_INLINE byte* erts_get_aligned_binary_bytes(Eterm bin, byte** base_ptr) { - return erts_get_aligned_binary_bytes_extra(bin, base_ptr, 0); + return erts_get_aligned_binary_bytes_extra(bin, base_ptr, ERTS_ALC_T_TMP, 0); } ERTS_GLB_INLINE void -erts_free_aligned_binary_bytes(byte* buf) +erts_free_aligned_binary_bytes_extra(byte* buf, ErtsAlcType_t allocator) { if (buf) { - erts_free(ERTS_ALC_T_TMP, (void *) buf); + erts_free(allocator, (void *) buf); } } +ERTS_GLB_INLINE void +erts_free_aligned_binary_bytes(byte* buf) +{ + erts_free_aligned_binary_bytes_extra(buf,ERTS_ALC_T_TMP); +} + /* Explicit extra bytes allocated to counter buggy drivers. ** These extra bytes where earlier (< R13B04) added by an alignment-bug ** in this code. Do we dare remove this in some major release (R14?) maybe? diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index e4f5d50ddf..88d2c06246 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -255,7 +255,7 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff * Simply shift whole bytes into the result. */ switch (BYTE_OFFSET(n)) { -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP case 7: w = (w << 8) | *bp++; case 6: w = (w << 8) | *bp++; case 5: w = (w << 8) | *bp++; @@ -360,7 +360,7 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff case 3: v32 = LSB[0] + (LSB[1]<<8) + (LSB[2]<<16); goto big_small; -#if !defined(ARCH_64) +#if !defined(ARCH_64) || HALFWORD_HEAP case 4: v32 = (LSB[0] + (LSB[1]<<8) + (LSB[2]<<16) + (LSB[3]<<24)); if (!IS_USMALL(sgn, v32)) { @@ -1335,12 +1335,12 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term, hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = used_size_in_bytes; - pb->next = MSO(c_p).mso; - MSO(c_p).mso = pb; + pb->next = MSO(c_p).first; + MSO(c_p).first = (struct erl_off_heap_header*)pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = PB_IS_WRITABLE | PB_ACTIVE_WRITER; - MSO(c_p).overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm)); /* * Now allocate the sub binary and set its size to include the @@ -1506,12 +1506,12 @@ erts_bs_init_writable(Process* p, Eterm sz) hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = 0; - pb->next = MSO(p).mso; - MSO(p).mso = pb; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*) pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = PB_IS_WRITABLE | PB_ACTIVE_WRITER; - MSO(p).overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); /* * Now allocate the sub binary. diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h index e3f8e0b679..0f67733fa4 100644 --- a/erts/emulator/beam/erl_bits.h +++ b/erts/emulator/beam/erl_bits.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -63,7 +63,7 @@ typedef struct erl_bin_match_struct{ #define HEADER_NUM_SLOTS(hdr) (header_arity(hdr)-sizeof(ErlBinMatchState)/sizeof(Eterm)+1) #define make_matchstate(_Ms) make_boxed((Eterm*)(_Ms)) -#define ms_matchbuffer(_Ms) &(((ErlBinMatchState*)(_Ms - TAG_PRIMARY_BOXED))->mb) +#define ms_matchbuffer(_Ms) &(((ErlBinMatchState*) boxed_val(_Ms))->mb) #if defined(ERTS_SMP) diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 15b1c6bb56..b0369a402b 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -78,16 +78,24 @@ enum DbIterSafety { ** The main meta table, containing all ets tables. */ #ifdef ERTS_SMP -# define META_MAIN_TAB_LOCK_CNT 16 -static union { - erts_smp_spinlock_t lck; - byte _cache_line_alignment[64]; -}meta_main_tab_locks[META_MAIN_TAB_LOCK_CNT]; + +#define ERTS_META_MAIN_TAB_LOCK_TAB_BITS 8 +#define ERTS_META_MAIN_TAB_LOCK_TAB_SIZE (1 << ERTS_META_MAIN_TAB_LOCK_TAB_BITS) +#define ERTS_META_MAIN_TAB_LOCK_TAB_MASK (ERTS_META_MAIN_TAB_LOCK_TAB_SIZE - 1) + +typedef union { + erts_smp_rwmtx_t rwmtx; + byte cache_line_align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE( + sizeof(erts_smp_rwmtx_t))]; +} erts_meta_main_tab_lock_t; + +static erts_meta_main_tab_lock_t *meta_main_tab_locks; + #endif static struct { union { DbTable *tb; /* Only directly readable if slot is ALIVE */ - Uint next_free; /* (index<<2)|1 if slot is FREE */ + UWord next_free; /* (index<<2)|1 if slot is FREE */ }u; } *meta_main_tab; @@ -104,17 +112,13 @@ static struct { #define MARK_SLOT_DEAD(i) (meta_main_tab[(i)].u.next_free |= 2) #define GET_ANY_SLOT_TAB(i) ((DbTable*)(meta_main_tab[(i)].u.next_free & ~(1|2))) /* dead or alive */ -static ERTS_INLINE void meta_main_tab_lock(unsigned slot) -{ -#ifdef ERTS_SMP - erts_smp_spin_lock(&meta_main_tab_locks[slot % META_MAIN_TAB_LOCK_CNT].lck); -#endif -} - -static ERTS_INLINE void meta_main_tab_unlock(unsigned slot) +static ERTS_INLINE erts_smp_rwmtx_t * +get_meta_main_tab_lock(unsigned slot) { #ifdef ERTS_SMP - erts_smp_spin_unlock(&meta_main_tab_locks[slot % META_MAIN_TAB_LOCK_CNT].lck); + return &meta_main_tab_locks[slot & ERTS_META_MAIN_TAB_LOCK_TAB_MASK].rwmtx; +#else + return NULL; #endif } @@ -166,7 +170,8 @@ struct meta_name_tab_entry* meta_name_tab_bucket(Eterm name, typedef enum { LCK_READ=1, /* read only access */ LCK_WRITE=2, /* exclusive table write access */ - LCK_WRITE_REC=3 /* record write access */ + LCK_WRITE_REC=3, /* record write access */ + LCK_NONE=4 } db_lock_kind_t; extern DbTableMethod db_hash; @@ -187,7 +192,7 @@ static Eterm ms_delete_all_buff[8]; /* To compare with for deletion static void fix_table_locked(Process* p, DbTable* tb); static void unfix_table_locked(Process* p, DbTable* tb, db_lock_kind_t* kind); -static void set_heir(Process* me, DbTable* tb, Eterm heir, Eterm heir_data); +static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data); static void free_heir_data(DbTable*); static void free_fixations_locked(DbTable *tb); @@ -214,17 +219,17 @@ Export ets_select_continue_exp; */ static Export ets_delete_continue_exp; -static ERTS_INLINE DbTable* db_ref(DbTable* tb) +static ERTS_INLINE DbTable* db_ref(DbTable* tb, db_lock_kind_t kind) { - if (tb != NULL) { + if (tb != NULL && kind != LCK_READ) { erts_refc_inc(&tb->common.ref, 2); } return tb; } -static ERTS_INLINE DbTable* db_unref(DbTable* tb) +static ERTS_INLINE DbTable* db_unref(DbTable* tb, db_lock_kind_t kind) { - if (!erts_refc_dectest(&tb->common.ref, 0)) { + if (kind != LCK_READ && !erts_refc_dectest(&tb->common.ref, 0)) { #ifdef HARDDEBUG if (erts_smp_atomic_read(&tb->common.memory_size) != sizeof(DbTable)) { erts_fprintf(stderr, "ets: db_unref memory remain=%ld fix=%x\n", @@ -256,12 +261,19 @@ static ERTS_INLINE DbTable* db_unref(DbTable* tb) return tb; } -static ERTS_INLINE void db_init_lock(DbTable* tb, char *rwname, char* fixname) +static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock, + char *rwname, char* fixname) { +#ifdef ERTS_SMP + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + if (use_frequent_read_lock) + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; +#endif erts_refc_init(&tb->common.ref, 1); erts_refc_init(&tb->common.fixref, 0); #ifdef ERTS_SMP - erts_smp_rwmtx_init_x(&tb->common.rwlock, rwname, tb->common.the_name); + erts_smp_rwmtx_init_opt_x(&tb->common.rwlock, &rwmtx_opt, + rwname, tb->common.the_name); erts_smp_mtx_init_x(&tb->common.fixlock, fixname, tb->common.the_name); tb->common.is_thread_safe = !(tb->common.status & DB_FINE_LOCKED); #endif @@ -297,7 +309,7 @@ static ERTS_INLINE void db_lock_take_over_ref(DbTable* tb, db_lock_kind_t kind) static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind) { - (void) db_ref(tb); + (void) db_ref(tb, kind); #ifdef ERTS_SMP db_lock_take_over_ref(tb, kind); #endif @@ -331,7 +343,7 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind) } } #endif - (void) db_unref(tb); /* May delete table... */ + (void) db_unref(tb, kind); /* May delete table... */ } @@ -349,32 +361,49 @@ static ERTS_INLINE void db_meta_unlock(DbTable* tb, db_lock_kind_t kind) } static ERTS_INLINE -DbTable* db_get_table(Process *p, - Eterm id, - int what, - db_lock_kind_t kind) +DbTable* db_get_table_aux(Process *p, + Eterm id, + int what, + db_lock_kind_t kind, + int meta_already_locked) { DbTable *tb = NULL; + erts_smp_rwmtx_t *mtl = NULL; if (is_small(id)) { Uint slot = unsigned_val(id) & meta_main_tab_slot_mask; - meta_main_tab_lock(slot); + if (!meta_already_locked) { + mtl = get_meta_main_tab_lock(slot); + erts_smp_rwmtx_rlock(mtl); + } +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) + else { + erts_smp_rwmtx_t *test_mtl = get_meta_main_tab_lock(slot); + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(test_mtl) + || erts_lc_rwmtx_is_rwlocked(test_mtl)); + } +#endif if (slot < db_max_tabs && IS_SLOT_ALIVE(slot)) { /* SMP: inc to prevent race, between unlock of meta_main_tab_lock * and the table locking outside the meta_main_tab_lock */ - tb = db_ref(meta_main_tab[slot].u.tb); + tb = db_ref(meta_main_tab[slot].u.tb, kind); } - meta_main_tab_unlock(slot); } else if (is_atom(id)) { - erts_smp_rwmtx_t* rwlock; - struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&rwlock); - erts_smp_rwmtx_rlock(rwlock); + struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&mtl); + if (!meta_already_locked) + erts_smp_rwmtx_rlock(mtl); + else{ + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(mtl) + || erts_lc_rwmtx_is_rwlocked(mtl)); + mtl = NULL; + } + if (bucket->pu.tb != NULL) { if (is_atom(bucket->u.name_atom)) { /* single */ if (bucket->u.name_atom == id) { - tb = db_ref(bucket->pu.tb); + tb = db_ref(bucket->pu.tb, kind); } } else { /* multi */ @@ -382,23 +411,33 @@ DbTable* db_get_table(Process *p, Uint i; for (i=0; i<cnt; i++) { if (bucket->pu.mvec[i].u.name_atom == id) { - tb = db_ref(bucket->pu.mvec[i].pu.tb); + tb = db_ref(bucket->pu.mvec[i].pu.tb, kind); break; } } } } - erts_smp_rwmtx_runlock(rwlock); } if (tb) { db_lock_take_over_ref(tb, kind); - if (tb->common.id == id && ((tb->common.status & what) != 0 || - p->id == tb->common.owner)) { - return tb; + if (tb->common.id != id + || ((tb->common.status & what) == 0 && p->id != tb->common.owner)) { + db_unlock(tb, kind); + tb = NULL; } - db_unlock(tb, kind); } - return NULL; + if (mtl) + erts_smp_rwmtx_runlock(mtl); + return tb; +} + +static ERTS_INLINE +DbTable* db_get_table(Process *p, + Eterm id, + int what, + db_lock_kind_t kind) +{ + return db_get_table_aux(p, id, what, kind, 0); } /* Requires meta_main_tab_locks[slot] locked. @@ -413,15 +452,15 @@ static ERTS_INLINE void free_slot(int slot) erts_smp_spin_unlock(&meta_main_tab_main_lock); } -static int insert_named_tab(Eterm name_atom, DbTable* tb) +static int insert_named_tab(Eterm name_atom, DbTable* tb, int have_lock) { int ret = 0; erts_smp_rwmtx_t* rwlock; struct meta_name_tab_entry* new_entry; struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom, &rwlock); - - erts_smp_rwmtx_rwlock(rwlock); + if (!have_lock) + erts_smp_rwmtx_rwlock(rwlock); if (bucket->pu.tb == NULL) { /* empty */ new_entry = bucket; @@ -468,17 +507,32 @@ static int insert_named_tab(Eterm name_atom, DbTable* tb) ret = 1; /* Ok */ done: - erts_smp_rwmtx_rwunlock(rwlock); + if (!have_lock) + erts_smp_rwmtx_rwunlock(rwlock); return ret; } -static int remove_named_tab(Eterm name_atom) +static int remove_named_tab(DbTable *tb, int have_lock) { int ret = 0; erts_smp_rwmtx_t* rwlock; + Eterm name_atom = tb->common.id; struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom, &rwlock); - erts_smp_rwmtx_rwlock(rwlock); +#ifdef ERTS_SMP + if (!have_lock && erts_smp_rwmtx_tryrwlock(rwlock) == EBUSY) { + /* + * We keep our increased refc over this op in order to + * prevent the table from disapearing. + */ + erts_smp_rwmtx_rwunlock(&tb->common.rwlock); + erts_smp_rwmtx_rwlock(rwlock); + erts_smp_rwmtx_rwlock(&tb->common.rwlock); + } +#endif + + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(rwlock)); + if (bucket->pu.tb == NULL) { goto done; } @@ -529,7 +583,8 @@ static int remove_named_tab(Eterm name_atom) ret = 1; /* Ok */ done: - erts_smp_rwmtx_rwunlock(rwlock); + if (!have_lock) + erts_smp_rwmtx_rwunlock(rwlock); return ret; } @@ -704,12 +759,13 @@ BIF_RETTYPE ets_update_element_3(BIF_ALIST_3) int cret = DB_ERROR_BADITEM; Eterm list; Eterm iter; - Eterm cell[2]; + DeclareTmpHeap(cell,2,BIF_P); DbUpdateHandle handle; if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) { BIF_ERROR(BIF_P, BADARG); } + UseTmpHeap(2,BIF_P); if (!(tb->common.status & (DB_SET | DB_ORDERED_SET))) { goto bail_out; } @@ -762,6 +818,7 @@ finalize: tb->common.meth->db_finalize_dbterm(&handle); bail_out: + UnUseTmpHeap(2,BIF_P); db_unlock(tb, LCK_WRITE_REC); switch (cret) { @@ -794,8 +851,8 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) Eterm* ret_list_currp = NULL; Eterm* ret_list_prevp = NULL; Eterm iter; - Eterm cell[2]; - Eterm tuple[3]; + DeclareTmpHeap(cell,5,BIF_P); + Eterm *tuple = cell+2; DbUpdateHandle handle; Uint halloc_size = 0; /* overestimated heap usage */ Eterm* htop; /* actual heap usage */ @@ -805,6 +862,9 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) { BIF_ERROR(BIF_P, BADARG); } + + UseTmpHeap(5,BIF_P); + if (!(tb->common.status & (DB_SET | DB_ORDERED_SET))) { goto bail_out; } @@ -951,6 +1011,7 @@ finalize: tb->common.meth->db_finalize_dbterm(&handle); bail_out: + UnUseTmpHeap(5,BIF_P); db_unlock(tb, LCK_WRITE_REC); switch (cret) { @@ -1127,6 +1188,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) { DbTable* tb; Eterm ret; + erts_smp_rwmtx_t *lck1, *lck2; #ifdef HARDDEBUG erts_fprintf(stderr, @@ -1135,34 +1197,65 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]); #endif - if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE)) == NULL) { + + if (is_not_atom(BIF_ARG_2)) { BIF_ERROR(BIF_P, BADARG); } - if (is_not_atom(BIF_ARG_2)) { - goto badarg; + (void) meta_name_tab_bucket(BIF_ARG_2, &lck1); + + if (is_small(BIF_ARG_1)) { + Uint slot = unsigned_val(BIF_ARG_1) & meta_main_tab_slot_mask; + lck2 = get_meta_main_tab_lock(slot); } + else if (is_atom(BIF_ARG_1)) { + (void) meta_name_tab_bucket(BIF_ARG_1, &lck2); + if (lck1 == lck2) + lck2 = NULL; + else if (lck1 > lck2) { + erts_smp_rwmtx_t *tmp = lck1; + lck1 = lck2; + lck2 = tmp; + } + } + else { + BIF_ERROR(BIF_P, BADARG); + } + + erts_smp_rwmtx_rwlock(lck1); + if (lck2) + erts_smp_rwmtx_rwlock(lck2); + + tb = db_get_table_aux(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE, 1); + if (!tb) + goto badarg; if (is_not_atom(tb->common.id)) { /* Not a named table */ tb->common.the_name = BIF_ARG_2; goto done; } - if (!insert_named_tab(BIF_ARG_2,tb)) { + if (!insert_named_tab(BIF_ARG_2, tb, 1)) goto badarg; - } - if (!remove_named_tab(tb->common.id)) { + + if (!remove_named_tab(tb, 1)) erl_exit(1,"Could not find named tab %s", tb->common.id); - } tb->common.id = tb->common.the_name = BIF_ARG_2; done: ret = tb->common.id; db_unlock(tb, LCK_WRITE); + erts_smp_rwmtx_rwunlock(lck1); + if (lck2) + erts_smp_rwmtx_rwunlock(lck2); BIF_RET(ret); badarg: - db_unlock(tb, LCK_WRITE); + if (tb) + db_unlock(tb, LCK_WRITE); + erts_smp_rwmtx_rwunlock(lck1); + if (lck2) + erts_smp_rwmtx_rwunlock(lck2); BIF_ERROR(BIF_P, BADARG); } @@ -1180,13 +1273,14 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) Eterm val; Eterm ret; Eterm heir; - Eterm heir_data; + UWord heir_data; Uint32 status; Sint keypos; - int is_named, is_fine_locked; + int is_named, is_fine_locked, frequent_read; int cret; - Eterm meta_tuple[3]; + DeclareTmpHeap(meta_tuple,3,BIF_P); DbTableMethod* meth; + erts_smp_rwmtx_t *mmtl; if (is_not_atom(BIF_ARG_1)) { BIF_ERROR(BIF_P, BADARG); @@ -1199,8 +1293,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) keypos = 1; is_named = 0; is_fine_locked = 0; + frequent_read = 0; heir = am_none; - heir_data = am_undefined; + heir_data = (UWord) am_undefined; list = BIF_ARG_2; while(is_list(list)) { @@ -1232,6 +1327,13 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) is_fine_locked = 0; } else break; } + else if (tp[1] == am_read_concurrency) { + if (tp[2] == am_true) { + frequent_read = 1; + } else if (tp[2] == am_false) { + frequent_read = 0; + } else break; + } else if (tp[1] == am_heir && tp[2] == am_none) { heir = am_none; heir_data = am_undefined; @@ -1280,6 +1382,11 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); } +#ifdef ERTS_SMP + if (frequent_read && !(status & DB_PRIVATE)) + status |= DB_FREQ_READ; +#endif + /* we create table outside any table lock * and take the unusal cost of destroy table if it * fails to find a slot @@ -1302,7 +1409,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) tb->common.type = status & ERTS_ETS_TABLE_TYPES; /* Note, 'type' is *read only* from now on... */ #endif - db_init_lock(tb, "db_tab", "db_tab_fix"); + db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ), + "db_tab", "db_tab_fix"); tb->common.keypos = keypos; tb->common.owner = BIF_P->id; set_heir(BIF_P, tb, heir, heir_data); @@ -1345,15 +1453,17 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) tb->common.id = ret; tb->common.slot = slot; /* store slot for erase */ - meta_main_tab_lock(slot); + mmtl = get_meta_main_tab_lock(slot); + erts_smp_rwmtx_rwlock(mmtl); meta_main_tab[slot].u.tb = tb; ASSERT(IS_SLOT_ALIVE(slot)); - meta_main_tab_unlock(slot); + erts_smp_rwmtx_rwunlock(mmtl); - if (is_named && !insert_named_tab(BIF_ARG_1, tb)) { - meta_main_tab_lock(slot); + if (is_named && !insert_named_tab(BIF_ARG_1, tb, 0)) { + mmtl = get_meta_main_tab_lock(slot); + erts_smp_rwmtx_rwlock(mmtl); free_slot(slot); - meta_main_tab_unlock(slot); + erts_smp_rwmtx_rwunlock(mmtl); db_lock_take_over_ref(tb,LCK_WRITE); free_heir_data(tb); @@ -1375,6 +1485,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) erts_smp_atomic_read(&meta_pid_to_fixed_tab->common.memory_size)); #endif + UseTmpHeap(3,BIF_P); + db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC); if (db_put_hash(meta_pid_to_tab, TUPLE2(meta_tuple, BIF_P->id, make_small(slot)), @@ -1383,6 +1495,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) } db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); + UnUseTmpHeap(3,BIF_P); + BIF_RET(ret); } @@ -1489,6 +1603,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) { int trap; DbTable* tb; + erts_smp_rwmtx_t *mmtl; #ifdef HARDDEBUG erts_fprintf(stderr, @@ -1510,16 +1625,26 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) tb->common.status &= ~(DB_PROTECTED|DB_PUBLIC|DB_PRIVATE); tb->common.status |= DB_DELETE; - meta_main_tab_lock(tb->common.slot); + mmtl = get_meta_main_tab_lock(tb->common.slot); +#ifdef ERTS_SMP + if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) { + /* + * We keep our increased refc over this op in order to + * prevent the table from disapearing. + */ + erts_smp_rwmtx_rwunlock(&tb->common.rwlock); + erts_smp_rwmtx_rwlock(mmtl); + erts_smp_rwmtx_rwlock(&tb->common.rwlock); + } +#endif /* We must keep the slot, to be found by db_proc_dead() if process dies */ MARK_SLOT_DEAD(tb->common.slot); - meta_main_tab_unlock(tb->common.slot); - if (is_atom(tb->common.id)) { - remove_named_tab(tb->common.id); - } + erts_smp_rwmtx_rwunlock(mmtl); + if (is_atom(tb->common.id)) + remove_named_tab(tb, 0); if (tb->common.owner != BIF_P->id) { - Eterm meta_tuple[3]; + DeclareTmpHeap(meta_tuple,3,BIF_P); /* * The table is being deleted by a process other than its owner. @@ -1527,6 +1652,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) * current process will be killed (e.g. by an EXIT signal), we will * now transfer the ownership to the current process. */ + UseTmpHeap(3,BIF_P); db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC); db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner, make_small(tb->common.slot)); @@ -1538,6 +1664,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) TUPLE2(meta_tuple,BIF_P->id,make_small(tb->common.slot)), 0); db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); + UnUseTmpHeap(3,BIF_P); } /* disable inheritance */ free_heir_data(tb); @@ -1554,9 +1681,15 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) * (it looks like an continuation pointer), but that is will crash the * emulator if this BIF is call traced. */ +#if HALFWORD_HEAP + Eterm *hp = HAlloc(BIF_P, 3); + hp[0] = make_pos_bignum_header(2); + *((UWord *) (UWord) (hp+1)) = (UWord) tb; +#else Eterm *hp = HAlloc(BIF_P, 2); hp[0] = make_pos_bignum_header(1); hp[1] = (Eterm) tb; +#endif BIF_TRAP1(&ets_delete_continue_exp, BIF_P, make_big(hp)); } else { @@ -1571,7 +1704,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3) { Process* to_proc = NULL; ErtsProcLocks to_locks = ERTS_PROC_LOCK_MAIN; - Eterm buf[5]; + DeclareTmpHeap(buf,5,BIF_P); Eterm to_pid = BIF_ARG_2; Eterm from_pid; DbTable* tb = NULL; @@ -1593,6 +1726,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3) goto badarg; /* or should we be idempotent? return false maybe */ } + UseTmpHeap(5,BIF_P); db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC); db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner, make_small(tb->common.slot)); @@ -1610,6 +1744,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3) TUPLE4(buf, am_ETS_TRANSFER, tb->common.id, from_pid, BIF_ARG_3), 0); erts_smp_proc_unlock(to_proc, to_locks); + UnUseTmpHeap(5,BIF_P); BIF_RET(am_true); badarg: @@ -1624,11 +1759,12 @@ BIF_RETTYPE ets_setopts_2(BIF_ALIST_2) Eterm* tp; Eterm opt; Eterm heir = THE_NON_VALUE; - Eterm heir_data = THE_NON_VALUE; + UWord heir_data = (UWord) THE_NON_VALUE; Uint32 protection = 0; - Eterm fakelist[2]; + DeclareTmpHeap(fakelist,2,BIF_P); Eterm tail; + UseTmpHeap(2,BIF_P); for (tail = is_tuple(BIF_ARG_2) ? CONS(fakelist, BIF_ARG_2, NIL) : BIF_ARG_2; is_list(tail); tail = CDR(list_val(tail))) { @@ -1681,9 +1817,11 @@ BIF_RETTYPE ets_setopts_2(BIF_ALIST_2) } db_unlock (tb,LCK_WRITE); + UnUseTmpHeap(2,BIF_P); BIF_RET(am_true); badarg: + UnUseTmpHeap(2,BIF_P); if (tb != NULL) { db_unlock(tb,LCK_WRITE); } @@ -1896,14 +2034,15 @@ BIF_RETTYPE ets_all_0(BIF_ALIST_0) previous = NIL; j = 0; for(i = 0; (i < t_max_tabs && j < t_tabs_cnt); i++) { - meta_main_tab_lock(i); + erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(i); + erts_smp_rwmtx_rlock(mmtl); if (IS_SLOT_ALIVE(i)) { j++; tb = meta_main_tab[i].u.tb; previous = CONS(hp, tb->common.id, previous); hp += 2; } - meta_main_tab_unlock(i); + erts_smp_rwmtx_runlock(mmtl); } HRelease(BIF_P, hendp, hp); BIF_RET(previous); @@ -1949,29 +2088,37 @@ BIF_RETTYPE ets_match_1(BIF_ALIST_1) BIF_RETTYPE ets_match_2(BIF_ALIST_2) { Eterm ms; - Eterm buff[8]; + DeclareTmpHeap(buff,8,BIF_P); Eterm *hp = buff; - /*hp = HAlloc(BIF_P, 8);*/ + Eterm res; + + UseTmpHeap(8,BIF_P); ms = CONS(hp, am_DollarDollar, NIL); hp += 2; ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); hp += 4; ms = CONS(hp, ms, NIL); - return ets_select_2(BIF_P, BIF_ARG_1, ms); + res = ets_select_2(BIF_P, BIF_ARG_1, ms); + UnUseTmpHeap(8,BIF_P); + return res; } BIF_RETTYPE ets_match_3(BIF_ALIST_3) { Eterm ms; - Eterm buff[8]; + DeclareTmpHeap(buff,8,BIF_P); Eterm *hp = buff; - /*hp = HAlloc(BIF_P, 8);*/ + Eterm res; + + UseTmpHeap(8,BIF_P); ms = CONS(hp, am_DollarDollar, NIL); hp += 2; ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); hp += 4; ms = CONS(hp, ms, NIL); - return ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3); + res = ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3); + UnUseTmpHeap(8,BIF_P); + return res; } @@ -2385,29 +2532,37 @@ BIF_RETTYPE ets_match_object_1(BIF_ALIST_1) BIF_RETTYPE ets_match_object_2(BIF_ALIST_2) { Eterm ms; - Eterm buff[8]; + DeclareTmpHeap(buff,8,BIF_P); Eterm *hp = buff; - /*hp = HAlloc(BIF_P, 8);*/ + Eterm res; + + UseTmpHeap(8,BIF_P); ms = CONS(hp, am_DollarUnderscore, NIL); hp += 2; ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); hp += 4; ms = CONS(hp, ms, NIL); - return ets_select_2(BIF_P, BIF_ARG_1, ms); + res = ets_select_2(BIF_P, BIF_ARG_1, ms); + UnUseTmpHeap(8,BIF_P); + return res; } BIF_RETTYPE ets_match_object_3(BIF_ALIST_3) { Eterm ms; - Eterm buff[8]; + DeclareTmpHeap(buff,8,BIF_P); Eterm *hp = buff; - /*hp = HAlloc(BIF_P, 8);*/ + Eterm res; + + UseTmpHeap(8,BIF_P); ms = CONS(hp, am_DollarUnderscore, NIL); hp += 2; ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); hp += 4; ms = CONS(hp, ms, NIL); - return ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3); + res = ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3); + UnUseTmpHeap(8,BIF_P); + return res; } /* @@ -2558,7 +2713,7 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3) BIF_TRAP3(bif_export[BIF_ets_match_spec_run_r_3], BIF_P,lst,BIF_ARG_2,ret); } - res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), 0, &dummy); + res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), NULL, 0, &dummy); if (is_value(res)) { sz = size_object(res); hp = HAlloc(BIF_P, sz + 2); @@ -2585,18 +2740,36 @@ void init_db(void) { DbTable init_tb; int i; - extern Eterm* em_apply_bif; + extern BeamInstr* em_apply_bif; Eterm *hp; unsigned bits; size_t size; #ifdef ERTS_SMP - for (i=0; i<META_MAIN_TAB_LOCK_CNT; i++) { - erts_smp_spinlock_init_x(&meta_main_tab_locks[i].lck, "meta_main_tab_slot", make_small(i)); + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + meta_main_tab_locks = erts_alloc(ERTS_ALC_T_DB_TABLES, + (sizeof(erts_meta_main_tab_lock_t) + * (ERTS_META_MAIN_TAB_LOCK_TAB_SIZE+1))); + + if ((((Uint) meta_main_tab_locks) & ERTS_CACHE_LINE_MASK) != 0) + meta_main_tab_locks = ((erts_meta_main_tab_lock_t *) + ((((Uint) meta_main_tab_locks) + & ~ERTS_CACHE_LINE_MASK) + + ERTS_CACHE_LINE_SIZE)); + + ASSERT((((Uint) meta_main_tab_locks) & ERTS_CACHE_LINE_MASK) == 0); + + for (i = 0; i < ERTS_META_MAIN_TAB_LOCK_TAB_SIZE; i++) { + erts_smp_rwmtx_init_opt_x(&meta_main_tab_locks[i].rwmtx, &rwmtx_opt, + "meta_main_tab_slot", make_small(i)); } erts_smp_spinlock_init(&meta_main_tab_main_lock, "meta_main_tab_main"); for (i=0; i<META_NAME_TAB_LOCK_CNT; i++) { - erts_smp_rwmtx_init_x(&meta_name_tab_rwlocks[i].lck, "meta_name_tab", make_small(i)); + erts_smp_rwmtx_init_opt_x(&meta_name_tab_rwlocks[i].lck, &rwmtx_opt, + "meta_name_tab", make_small(i)); } #endif @@ -2714,9 +2887,9 @@ void init_db(void) ets_select_delete_continue_exp.code[1] = am_atom_put("delete_trap",11); ets_select_delete_continue_exp.code[2] = 1; ets_select_delete_continue_exp.code[3] = - (Eterm) em_apply_bif; + (BeamInstr) em_apply_bif; ets_select_delete_continue_exp.code[4] = - (Eterm) &ets_select_delete_1; + (BeamInstr) &ets_select_delete_1; /* Non visual BIF to trap to. */ memset(&ets_select_count_continue_exp, 0, sizeof(Export)); @@ -2726,9 +2899,9 @@ void init_db(void) ets_select_count_continue_exp.code[1] = am_atom_put("count_trap",11); ets_select_count_continue_exp.code[2] = 1; ets_select_count_continue_exp.code[3] = - (Eterm) em_apply_bif; + (BeamInstr) em_apply_bif; ets_select_count_continue_exp.code[4] = - (Eterm) &ets_select_count_1; + (BeamInstr) &ets_select_count_1; /* Non visual BIF to trap to. */ memset(&ets_select_continue_exp, 0, sizeof(Export)); @@ -2738,9 +2911,9 @@ void init_db(void) ets_select_continue_exp.code[1] = am_atom_put("select_trap",11); ets_select_continue_exp.code[2] = 1; ets_select_continue_exp.code[3] = - (Eterm) em_apply_bif; + (BeamInstr) em_apply_bif; ets_select_continue_exp.code[4] = - (Eterm) &ets_select_trap_1; + (BeamInstr) &ets_select_trap_1; /* Non visual BIF to trap to. */ memset(&ets_delete_continue_exp, 0, sizeof(Export)); @@ -2748,8 +2921,8 @@ void init_db(void) ets_delete_continue_exp.code[0] = am_ets; ets_delete_continue_exp.code[1] = am_atom_put("delete_trap",11); ets_delete_continue_exp.code[2] = 1; - ets_delete_continue_exp.code[3] = (Eterm) em_apply_bif; - ets_delete_continue_exp.code[4] = (Eterm) &ets_delete_trap; + ets_delete_continue_exp.code[3] = (BeamInstr) em_apply_bif; + ets_delete_continue_exp.code[4] = (BeamInstr) &ets_delete_trap; hp = ms_delete_all_buff; ms_delete_all = CONS(hp, am_true, NIL); @@ -2843,9 +3016,9 @@ static int give_away_to_heir(Process* p, DbTable* tb) { Process* to_proc; ErtsProcLocks to_locks = ERTS_PROC_LOCK_MAIN; - Eterm buf[5]; + DeclareTmpHeap(buf,5,p); Eterm to_pid; - Eterm heir_data; + UWord heir_data; ASSERT(tb->common.owner == p->id); ASSERT(is_internal_pid(tb->common.heir)); @@ -2856,12 +3029,12 @@ retry: to_pid, to_locks, ERTS_P2P_FLG_TRY_LOCK); if (to_proc == ERTS_PROC_LOCK_BUSY) { - db_ref(tb); /* while unlocked */ + db_ref(tb, LCK_NONE); /* while unlocked */ db_unlock(tb,LCK_WRITE); to_proc = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, to_pid, to_locks); db_lock(tb,LCK_WRITE); - tb = db_unref(tb); + tb = db_unref(tb, LCK_NONE); ASSERT(tb != NULL); if (tb->common.owner != p->id) { @@ -2888,6 +3061,7 @@ retry: erts_smp_proc_unlock(to_proc, to_locks); return 0; /* heir dead and pid reused, table still mine */ } + UseTmpHeap(5,p); db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC); db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner, make_small(tb->common.slot)); @@ -2899,7 +3073,7 @@ retry: TUPLE2(buf,to_pid,make_small(tb->common.slot)), 0); db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); - + UnUseTmpHeap(5,p); db_unlock(tb,LCK_WRITE); heir_data = tb->common.heir_data; if (!is_immed(heir_data)) { @@ -2968,12 +3142,13 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) while (state->slots.ix < state->slots.size) { DbTable *tb = NULL; Sint ix = unsigned_val(state->slots.arr[state->slots.ix]); - meta_main_tab_lock(ix); + erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix); + erts_smp_rwmtx_rlock(mmtl); if (!IS_SLOT_FREE(ix)) { - tb = db_ref(GET_ANY_SLOT_TAB(ix)); + tb = db_ref(GET_ANY_SLOT_TAB(ix), LCK_WRITE); ASSERT(tb); } - meta_main_tab_unlock(ix); + erts_smp_rwmtx_runlock(mmtl); if (tb) { int do_yield; db_lock_take_over_ref(tb, LCK_WRITE); @@ -3005,7 +3180,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) tb->common.status |= DB_DELETE; if (is_atom(tb->common.id)) - remove_named_tab(tb->common.id); + remove_named_tab(tb, 0); free_heir_data(tb); free_fixations_locked(tb); @@ -3055,12 +3230,13 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) while (state->slots.ix < state->slots.size) { DbTable *tb = NULL; Sint ix = unsigned_val(state->slots.arr[state->slots.ix]); - meta_main_tab_lock(ix); + erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix); + erts_smp_rwmtx_rlock(mmtl); if (IS_SLOT_ALIVE(ix)) { - tb = db_ref(meta_main_tab[ix].u.tb); + tb = db_ref(meta_main_tab[ix].u.tb, LCK_WRITE_REC); ASSERT(tb); } - meta_main_tab_unlock(ix); + erts_smp_rwmtx_runlock(mmtl); if (tb) { int reds; DbFixation** pp; @@ -3145,7 +3321,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) static void fix_table_locked(Process* p, DbTable* tb) { DbFixation *fix; - Eterm meta_tuple[3]; + DeclareTmpHeap(meta_tuple,3,p); #ifdef ERTS_SMP erts_smp_mtx_lock(&tb->common.fixlock); @@ -3179,12 +3355,15 @@ static void fix_table_locked(Process* p, DbTable* tb) erts_smp_mtx_unlock(&tb->common.fixlock); #endif p->flags |= F_USING_DB; + UseTmpHeap(3,p); db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC); if (db_put_hash(meta_pid_to_fixed_tab, TUPLE2(meta_tuple, p->id, make_small(tb->common.slot)), 0) != DB_ERROR_NONE) { + UnUseTmpHeap(3,p); erl_exit(1,"Could not insert ets metadata in safe_fixtable."); } + UnUseTmpHeap(3,p); db_meta_unlock(meta_pid_to_fixed_tab, LCK_WRITE_REC); } @@ -3231,6 +3410,7 @@ unlocked: #ifdef ERTS_SMP if (*kind_p == LCK_READ && tb->common.is_thread_safe) { /* Must have write lock while purging pseudo-deleted (OTP-8166) */ + db_ref(tb, LCK_WRITE); /* LCK_WRITE need it, but not LCK_READ */ erts_smp_rwmtx_runlock(&tb->common.rwlock); erts_smp_rwmtx_rwlock(&tb->common.rwlock); *kind_p = LCK_WRITE; @@ -3264,7 +3444,7 @@ static void free_fixations_locked(DbTable *tb) tb->common.fixations = NULL; } -static void set_heir(Process* me, DbTable* tb, Eterm heir, Eterm heir_data) +static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data) { tb->common.heir = heir; if (heir == am_none) { @@ -3285,10 +3465,13 @@ static void set_heir(Process* me, DbTable* tb, Eterm heir, Eterm heir_data) } if (!is_immed(heir_data)) { - Eterm tmp[2]; + DeclareTmpHeap(tmp,2,me); + + UseTmpHeap(2,me); /* Make a dummy 1-tuple around data to use db_get_term() */ - heir_data = (Eterm) db_get_term(&tb->common, NULL, 0, + heir_data = (UWord) db_get_term(&tb->common, NULL, 0, TUPLE1(tmp,heir_data)); + UnUseTmpHeap(2,me); ASSERT(!is_immed(heir_data)); } tb->common.heir_data = heir_data; @@ -3311,10 +3494,13 @@ static BIF_RETTYPE ets_delete_trap(Process *p, Eterm cont) { int trap; Eterm* ptr = big_val(cont); - DbTable *tb = (DbTable *) ptr[1]; + DbTable *tb = *((DbTable **) (UWord) (ptr + 1)); +#if HALFWORD_HEAP + ASSERT(*ptr == make_pos_bignum_header(2)); +#else ASSERT(*ptr == make_pos_bignum_header(1)); - +#endif db_lock(tb, LCK_WRITE); trap = free_table_cont(p, tb, 0, 1); db_unlock(tb, LCK_WRITE); @@ -3337,6 +3523,7 @@ static int free_table_cont(Process *p, int clean_meta_tab) { Eterm result; + erts_smp_rwmtx_t *mmtl; #ifdef HARDDEBUG if (!first) { @@ -3362,9 +3549,20 @@ static int free_table_cont(Process *p, tb->common.id); #endif /* Completely done - we will not get called again. */ - meta_main_tab_lock(tb->common.slot); + mmtl = get_meta_main_tab_lock(tb->common.slot); +#ifdef ERTS_SMP + if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) { + /* + * We keep our increased refc over this op in order to + * prevent the table from disapearing. + */ + erts_smp_rwmtx_rwunlock(&tb->common.rwlock); + erts_smp_rwmtx_rwlock(mmtl); + erts_smp_rwmtx_rwlock(&tb->common.rwlock); + } +#endif free_slot(tb->common.slot); - meta_main_tab_unlock(tb->common.slot); + erts_smp_rwmtx_rwunlock(mmtl); if (clean_meta_tab) { db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC); @@ -3372,7 +3570,7 @@ static int free_table_cont(Process *p, make_small(tb->common.slot)); db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); } - db_unref(tb); + db_unref(tb, LCK_NONE); BUMP_REDS(p, 100); return 0; } diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 4141f9766b..5abd2e50fa 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -623,12 +623,16 @@ int db_create_hash(Process *p, DbTable *tbl) erts_smp_atomic_init(&tb->is_resizing, 0); #ifdef ERTS_SMP if (tb->common.type & DB_FINE_LOCKED) { + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; int i; + if (tb->common.type & DB_FREQ_READ) + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; tb->locks = (DbTableHashFineLocks*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, /* Other type maybe? */ (DbTable *) tb, sizeof(DbTableHashFineLocks)); for (i=0; i<DB_HASH_LOCK_CNT; ++i) { - erts_rwmtx_init_x(&tb->locks->lck_vec[i].lck, "db_hash_slot", make_small(i)); + erts_smp_rwmtx_init_opt_x(&tb->locks->lck_vec[i].lck, &rwmtx_opt, + "db_hash_slot", make_small(i)); } /* This important property is needed to guarantee that the buckets * involved in a grow/shrink operation it protected by the same lock: @@ -1284,7 +1288,7 @@ static int db_select_continue_hash(Process *p, (match_res = db_prog_match(p,mp, make_tuple(current->dbterm.tpl), - 0,&dummy), + NULL,0,&dummy), is_value(match_res))) { if (all_objects) { hp = HAlloc(p, current->dbterm.size + 2); @@ -1462,7 +1466,7 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl, if (current->hvalue != INVALID_HASH) { match_res = db_prog_match(p,mpi.mp, make_tuple(current->dbterm.tpl), - 0,&dummy); + NULL,0,&dummy); if (is_value(match_res)) { if (mpi.all_objects) { hp = HAlloc(p, current->dbterm.size + 2); @@ -1641,7 +1645,7 @@ static int db_select_count_hash(Process *p, if (current != NULL) { if (current->hvalue != INVALID_HASH) { if (db_prog_match(p, mpi.mp, make_tuple(current->dbterm.tpl), - 0, &dummy) == am_true) { + NULL,0, &dummy) == am_true) { ++got; } --num_left; @@ -1792,7 +1796,7 @@ static int db_select_delete_hash(Process *p, int did_erase = 0; if ((db_prog_match(p,mpi.mp, make_tuple((*current)->dbterm.tpl), - 0,&dummy)) == am_true) { + NULL,0,&dummy)) == am_true) { if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */ if (slot_ix != last_pseudo_delete) { add_fixed_deletion(tb, slot_ix); @@ -1904,7 +1908,7 @@ static int db_select_delete_continue_hash(Process *p, else { int did_erase = 0; if ((db_prog_match(p,mp,make_tuple((*current)->dbterm.tpl), - 0,&dummy)) == am_true) { + NULL,0,&dummy)) == am_true) { if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */ if (slot_ix != last_pseudo_delete) { add_fixed_deletion(tb, slot_ix); @@ -2005,7 +2009,7 @@ static int db_select_count_continue_hash(Process *p, continue; } if (db_prog_match(p, mp, make_tuple(current->dbterm.tpl), - 0,&dummy) == am_true) { + NULL,0,&dummy) == am_true) { ++got; } --num_left; @@ -2741,6 +2745,7 @@ static void db_finalize_dbterm_hash(DbUpdateHandle* handle) ASSERT(&oldp->dbterm == handle->dbterm); if (handle->mustResize) { + ErlOffHeap tmp_offheap; Eterm* top; Eterm copy; DbTerm* newDbTerm; @@ -2751,18 +2756,15 @@ static void db_finalize_dbterm_hash(DbUpdateHandle* handle) newDbTerm = &newp->dbterm; newDbTerm->size = handle->new_size; - newDbTerm->off_heap.mso = NULL; - newDbTerm->off_heap.externals = NULL; - #ifndef HYBRID /* FIND ME! */ - newDbTerm->off_heap.funs = NULL; - #endif - newDbTerm->off_heap.overhead = 0; + tmp_offheap.first = NULL; + tmp_offheap.overhead = 0; /* make a flat copy */ top = DBTERM_BUF(newDbTerm); copy = copy_struct(make_tuple(handle->dbterm->tpl), handle->new_size, - &top, &newDbTerm->off_heap); + &top, &tmp_offheap); + newDbTerm->first_oh = tmp_offheap.first; DBTERM_SET_TPL(newDbTerm,tuple_val(copy)); WUNLOCK_HASH(lck); @@ -2805,7 +2807,11 @@ void db_foreach_offheap_hash(DbTable *tbl, for (i = 0; i < nactive; i++) { list = BUCKET(tb,i); while(list != 0) { - (*func)(&(list->dbterm.off_heap), arg); + ErlOffHeap tmp_offheap; + tmp_offheap.first = list->dbterm.first_oh; + tmp_offheap.overhead = 0; + (*func)(&tmp_offheap, arg); + list->dbterm.first_oh = tmp_offheap.first; list = list->next; } } diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index b421da591b..5644e85f97 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -443,9 +443,9 @@ void db_initialize_tree(void) ets_select_reverse_exp.code[1] = am_reverse; ets_select_reverse_exp.code[2] = 3; ets_select_reverse_exp.code[3] = - (Eterm) em_apply_bif; + (BeamInstr) em_apply_bif; ets_select_reverse_exp.code[4] = - (Eterm) &ets_select_reverse; + (BeamInstr) &ets_select_reverse; return; }; @@ -1081,11 +1081,12 @@ static int db_select_continue_tree(Process *p, static int db_select_tree(Process *p, DbTable *tbl, Eterm pattern, int reverse, Eterm *ret) { + /* Strategy: Traverse backwards to build resulting list from tail to head */ DbTableTree *tb = &tbl->tree; DbTreeStack* stack; struct select_context sc; struct mp_info mpi; - Eterm lastkey = NIL; + Eterm lastkey = THE_NON_VALUE; Eterm key; Eterm continuation; unsigned sz; @@ -1293,7 +1294,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, DbTreeStack* stack; struct select_count_context sc; struct mp_info mpi; - Eterm lastkey = NIL; + Eterm lastkey = THE_NON_VALUE; Eterm key; Eterm continuation; unsigned sz; @@ -1395,7 +1396,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, DbTreeStack* stack; struct select_context sc; struct mp_info mpi; - Eterm lastkey = NIL; + Eterm lastkey = THE_NON_VALUE; Eterm key; Eterm continuation; unsigned sz; @@ -1636,7 +1637,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, DbTableTree *tb = &tbl->tree; struct select_delete_context sc; struct mp_info mpi; - Eterm lastkey = NIL; + Eterm lastkey = THE_NON_VALUE; Eterm key; Eterm continuation; unsigned sz; @@ -1817,10 +1818,14 @@ do_db_tree_foreach_offheap(TreeDbTerm *tdbt, void (*func)(ErlOffHeap *, void *), void * arg) { + ErlOffHeap tmp_offheap; if(!tdbt) return; do_db_tree_foreach_offheap(tdbt->left, func, arg); - (*func)(&(tdbt->dbterm.off_heap), arg); + tmp_offheap.first = tdbt->dbterm.first_oh; + tmp_offheap.overhead = 0; + (*func)(&tmp_offheap, arg); + tdbt->dbterm.first_oh = tmp_offheap.first; do_db_tree_foreach_offheap(tdbt->right, func, arg); } @@ -2574,6 +2579,7 @@ static int db_lookup_dbterm_tree(DbTable *tbl, Eterm key, DbUpdateHandle* handle static void db_finalize_dbterm_tree(DbUpdateHandle* handle) { if (handle->mustResize) { + ErlOffHeap tmp_offheap; Eterm* top; Eterm copy; DbTerm* newDbTerm; @@ -2588,18 +2594,15 @@ static void db_finalize_dbterm_tree(DbUpdateHandle* handle) newDbTerm = &newp->dbterm; newDbTerm->size = handle->new_size; - newDbTerm->off_heap.mso = NULL; - newDbTerm->off_heap.externals = NULL; - #ifndef HYBRID /* FIND ME! */ - newDbTerm->off_heap.funs = NULL; - #endif - newDbTerm->off_heap.overhead = 0; + tmp_offheap.first = NULL; + tmp_offheap.overhead = 0; /* make a flat copy */ top = DBTERM_BUF(newDbTerm); copy = copy_struct(make_tuple(handle->dbterm->tpl), handle->new_size, - &top, &newDbTerm->off_heap); + &top, &tmp_offheap); + newDbTerm->first_oh = tmp_offheap.first; DBTERM_SET_TPL(newDbTerm,tuple_val(copy)); db_free_term_data(handle->dbterm); @@ -2628,7 +2631,7 @@ static void traverse_backwards(DbTableTree *tb, { TreeDbTerm *this, *next; - if (lastkey == NIL) { + if (lastkey == THE_NON_VALUE) { stack->pos = stack->slot = 0; if (( this = tb->root ) == NULL) { return; @@ -2666,7 +2669,7 @@ static void traverse_forward(DbTableTree *tb, { TreeDbTerm *this, *next; - if (lastkey == NIL) { + if (lastkey == THE_NON_VALUE) { stack->pos = stack->slot = 0; if (( this = tb->root ) == NULL) { return; @@ -3023,7 +3026,7 @@ static int doit_select(DbTableTree *tb, TreeDbTerm *this, void *ptr, } ret = db_prog_match(sc->p, sc->mp, make_tuple(this->dbterm.tpl), - 0, &dummy); + NULL,0, &dummy); if (is_value(ret)) { Uint sz; Eterm *hp; @@ -3072,7 +3075,7 @@ static int doit_select_count(DbTableTree *tb, TreeDbTerm *this, void *ptr, } ret = db_prog_match(sc->p, sc->mp, make_tuple(this->dbterm.tpl), - 0, &dummy); + NULL,0, &dummy); if (ret == am_true) { ++(sc->got); } @@ -3105,7 +3108,7 @@ static int doit_select_chunk(DbTableTree *tb, TreeDbTerm *this, void *ptr, ret = db_prog_match(sc->p, sc->mp, make_tuple(this->dbterm.tpl), - 0, &dummy); + NULL,0, &dummy); if (is_value(ret)) { Uint sz; Eterm *hp; @@ -3158,7 +3161,7 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr, return 0; ret = db_prog_match(sc->p, sc->mp, make_tuple(this->dbterm.tpl), - 0, &dummy); + NULL,0, &dummy); if (ret == am_true) { key = GETKEY(sc->tb, this->dbterm.tpl); linkout_tree(sc->tb, key); diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 8c373451fd..2f34561234 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1998-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1998-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -117,6 +117,10 @@ do { \ erts_free(ERTS_ALC_T_DB_MC_STK, (Name).data); \ } while (0) + +#define TermWords(t) (((t) / (sizeof(UWord)/sizeof(Eterm))) + !!((t) % (sizeof(UWord)/sizeof(Eterm)))) + + static ERTS_INLINE Process * get_proc(Process *cp, Uint32 cp_locks, Eterm id, Uint32 id_locks) { @@ -281,7 +285,7 @@ typedef struct dmc_guard_bif { */ DMC_DECLARE_STACK_TYPE(Eterm); -DMC_DECLARE_STACK_TYPE(Uint); +DMC_DECLARE_STACK_TYPE(UWord); DMC_DECLARE_STACK_TYPE(unsigned); @@ -359,12 +363,7 @@ static ErtsMatchPseudoProcess *match_pseudo_process; static ERTS_INLINE void cleanup_match_pseudo_process(ErtsMatchPseudoProcess *mpsp, int keep_heap) { - if (mpsp->process.mbuf - || mpsp->process.off_heap.mso -#ifndef HYBRID /* FIND ME! */ - || mpsp->process.off_heap.funs -#endif - || mpsp->process.off_heap.externals) { + if (mpsp->process.mbuf || mpsp->process.off_heap.first) { erts_cleanup_empty_process(&mpsp->process); } #ifdef DEBUG @@ -382,7 +381,7 @@ cleanup_match_pseudo_process(ErtsMatchPseudoProcess *mpsp, int keep_heap) else { int i; for (i = 0; i < ERTS_DEFAULT_MS_HEAP_SIZE; i++) { -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP mpsp->default_heap[i] = (Eterm) 0xdeadbeefdeadbeef; #else mpsp->default_heap[i] = (Eterm) 0xdeadbeef; @@ -830,42 +829,42 @@ static Uint my_size_object(Eterm t); static Eterm my_copy_struct(Eterm t, Eterm **hp, ErlOffHeap* off_heap); /* Guard compilation */ -static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(Uint) *text, +static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(UWord) *text, Eterm t); static DMCRet dmc_list(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant); static DMCRet dmc_tuple(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant); static DMCRet dmc_variable(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant); static DMCRet dmc_fun(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant); static DMCRet dmc_expr(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant); static DMCRet compile_guard_expr(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t); /* match expression subroutine */ static DMCRet dmc_one_term(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(Eterm) *stack, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm c); @@ -1185,7 +1184,7 @@ Eterm erts_match_set_run(Process *p, Binary *mpsp, Eterm ret; ret = db_prog_match(p, mpsp, - (Eterm) args, + NIL, args, num_args, return_flags); #if defined(HARDDEBUG) if (is_non_value(ret)) { @@ -1204,6 +1203,32 @@ Eterm erts_match_set_run(Process *p, Binary *mpsp, */ } +static Eterm erts_match_set_run_ets(Process *p, Binary *mpsp, + Eterm args, int num_args, + Uint32 *return_flags) +{ + Eterm ret; + + ret = db_prog_match(p, mpsp, + args, NULL, + num_args, return_flags); +#if defined(HARDDEBUG) + if (is_non_value(ret)) { + erts_fprintf(stderr, "Failed\n"); + } else { + erts_fprintf(stderr, "Returning : %T\n", ret); + } +#endif + return ret; + /* Returns + * THE_NON_VALUE if no match + * am_false if {message,false} has been called, + * am_true if {message,_} has not been called or + * if {message,true} has been called, + * Msg if {message,Msg} has been called. + */ +} + /* ** API Used by other erl_db modules. */ @@ -1245,7 +1270,7 @@ Binary *db_match_compile(Eterm *matchexpr, { DMCHeap heap; DMC_STACK_TYPE(Eterm) stack; - DMC_STACK_TYPE(Uint) text; + DMC_STACK_TYPE(UWord) text; DMCContext context; MatchProg *ret = NULL; Eterm t; @@ -1380,7 +1405,7 @@ restart: /* ** There is one single top variable in the match expression - ** iff the text is tho Uint's and the single instruction + ** iff the text is two Uint's and the single instruction ** is 'matchBind' or it is only a skip. */ context.special = @@ -1491,8 +1516,8 @@ restart: ** A special case is when the match expression is a single binding ** (i.e '$1'), then the field single_variable is set to 1. */ - bp = erts_create_magic_binary(((sizeof(MatchProg) - sizeof(Uint)) + - (DMC_STACK_NUM(text) * sizeof(Uint))), + bp = erts_create_magic_binary(((sizeof(MatchProg) - sizeof(UWord)) + + (DMC_STACK_NUM(text) * sizeof(UWord))), erts_db_match_prog_destructor); ret = Binary2MatchProg(bp); ret->saved_program_buf = NULL; @@ -1501,7 +1526,7 @@ restart: ret->num_bindings = heap.used; ret->single_variable = context.special; sys_memcpy(ret->text, DMC_STACK_DATA(text), - DMC_STACK_NUM(text) * sizeof(Uint)); + DMC_STACK_NUM(text) * sizeof(UWord)); ret->heap_size = ((heap.used * sizeof(Eterm)) + (max_eheap_need * sizeof(Eterm)) + (context.stack_need * sizeof(Eterm *)) + @@ -1517,10 +1542,9 @@ restart: */ context.save = NULL; error: /* Here is were we land when compilation failed. */ - while (context.save != NULL) { - ErlHeapFragment *ll = context.save->next; + if (context.save != NULL) { free_message_buffer(context.save); - context.save = ll; + context.save = NULL; } DMC_FREE(stack); DMC_FREE(text); @@ -1537,15 +1561,11 @@ error: /* Here is were we land when compilation failed. */ void erts_db_match_prog_destructor(Binary *bprog) { MatchProg *prog; - ErlHeapFragment *tmp, *ll; if (bprog == NULL) return; prog = Binary2MatchProg(bprog); - tmp = prog->term_save; - while (tmp != NULL) { - ll = tmp->next; - free_message_buffer(tmp); - tmp = ll; + if (prog->term_save != NULL) { + free_message_buffer(prog->term_save); } if (prog->saved_program_buf != NULL) free_message_buffer(prog->saved_program_buf); @@ -1591,6 +1611,7 @@ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity) ** i.e. 'DCOMP_TRACE' was specified */ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, + Eterm *termp, int arity, Uint32 *return_flags) { @@ -1601,7 +1622,8 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, Eterm **sp; Eterm *esp; Eterm *hp; - Uint *pc = prog->text; + BeamInstr *cp; + UWord *pc = prog->text; Eterm *ehp; Eterm ret; Uint n = 0; /* To avoid warning. */ @@ -1616,9 +1638,9 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, int fail_label; int atomic_trace; #ifdef DMC_DEBUG - unsigned long *heap_fence; - unsigned long *eheap_fence; - unsigned long *stack_fence; + Uint *heap_fence; + Uint *eheap_fence; + Uint *stack_fence; Uint save_op; #endif /* DMC_DEBUG */ @@ -1654,9 +1676,9 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, #ifdef DMC_DEBUG save_op = 0; - heap_fence = (unsigned long *) mpsp->heap + prog->eheap_offset - 1; - eheap_fence = (unsigned long *) mpsp->heap + prog->stack_offset - 1; - stack_fence = (unsigned long *) mpsp->heap + prog->heap_size - 1; + heap_fence = (Uint *) mpsp->heap + prog->eheap_offset - 1; + eheap_fence = (Uint *) mpsp->heap + prog->stack_offset - 1; + stack_fence = (Uint *) mpsp->heap + prog->heap_size - 1; *heap_fence = FENCE_PATTERN; *eheap_fence = FENCE_PATTERN; *stack_fence = FENCE_PATTERN; @@ -1709,11 +1731,11 @@ restart: n = *pc++; if ((int) n != arity) FAIL(); - ep = (Eterm *) *ep; + ep = termp; break; case matchArrayBind: /* When the array size is unknown. */ n = *pc++; - hp[n] = dpm_array_to_list(psp, (Eterm *) term, arity); + hp[n] = dpm_array_to_list(psp, termp, arity); break; case matchTuple: /* *ep is a tuple of arity n */ if (!is_tuple(*ep)) @@ -1770,29 +1792,34 @@ restart: FAIL(); if (memcmp(float_val(*ep) + 1, pc, sizeof(double))) FAIL(); - pc += 2; + pc += TermWords(2); ++ep; break; case matchEqRef: if (!is_ref(*ep)) FAIL(); - if (!eq(*ep, make_internal_ref(pc))) + if (!eq(*ep, make_internal_ref((Uint *) pc))) FAIL(); - i = thing_arityval(*pc); - pc += i+1; + i = thing_arityval(*((Uint *) pc)); + pc += TermWords(i+1); ++ep; break; case matchEqBig: if (!is_big(*ep)) FAIL(); tp = big_val(*ep); - if (*tp != *pc) - FAIL(); - i = BIG_ARITY(pc); - while(i--) - if (*++tp != *++pc) + { + Eterm *epc = (Eterm *) pc; + if (*tp != *epc) FAIL(); - ++pc; + i = BIG_ARITY(epc); + pc += TermWords(i+1); + while(i--) { + if (*++tp != *++epc) { + FAIL(); + } + } + } ++ep; break; case matchEq: @@ -1884,7 +1911,7 @@ restart: break; case matchPushArrayAsList: n = arity; /* Only happens when 'term' is an array */ - tp = (Eterm *) term; + tp = termp; *esp++ = make_list(ehp); while (n--) { *ehp++ = *tp++; @@ -1897,7 +1924,7 @@ restart: break; case matchPushArrayAsListU: /* This instruction is NOT efficient. */ - *esp++ = dpm_array_to_list(psp, (Eterm *) term, arity); + *esp++ = dpm_array_to_list(psp, termp, arity); break; case matchTrue: if (*--esp != am_true) @@ -2095,17 +2122,17 @@ restart: } break; case matchCaller: - if (!(c_p->cp) || !(hp = find_function_from_pc(c_p->cp))) { + if (!(c_p->cp) || !(cp = find_function_from_pc(c_p->cp))) { *esp++ = am_undefined; } else { *esp++ = make_tuple(ehp); - ehp[0] = make_arityval(3); - ehp[1] = hp[0]; - ehp[2] = hp[1]; - ehp[3] = make_small(hp[2]); - ehp += 4; - } - break; + ehp[0] = make_arityval(3); + ehp[1] = cp[0]; + ehp[2] = cp[1]; + ehp[3] = make_small((Uint) cp[2]); + ehp += 4; + } + break; case matchSilent: --esp; if (*esp == am_true) { @@ -2300,7 +2327,7 @@ void db_free_dmc_err_info(DMCErrInfo *ei){ */ Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr) { - Eterm big_tmp[2]; + DeclareTmpHeapNoproc(big_tmp,2); Eterm res; Sint ires; Eterm arg1; @@ -2318,6 +2345,7 @@ Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr) } } else { + UseTmpHeapNoproc(2); switch(NUMBER_CODE(counter, incr)) { case SMALL_BIG: arg1 = small_to_big(signed_val(counter), big_tmp); @@ -2332,12 +2360,14 @@ Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr) arg2 = counter; break; default: + UnUseTmpHeapNoproc(2); return THE_NON_VALUE; } res = big_plus(arg1, arg2, *hpp); if (is_big(res)) { *hpp += BIG_NEED_SIZE(big_size(res)); } + UnUseTmpHeapNoproc(2); return res; } } @@ -2421,9 +2451,13 @@ void* db_get_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) DbTerm* p; Eterm copy; Eterm *top; + ErlOffHeap tmp_offheap; if (old != 0) { - erts_cleanup_offheap(&old->off_heap); + tmp_offheap.first = old->first_oh; + tmp_offheap.overhead = 0; + erts_cleanup_offheap(&tmp_offheap); + old->first_oh = tmp_offheap.first; if (size == old->size) { p = old; } else { @@ -2459,15 +2493,12 @@ void* db_get_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) p = (DbTerm*) ((void *)(((char *) structp) + offset)); } p->size = size; - p->off_heap.mso = NULL; - p->off_heap.externals = NULL; -#ifndef HYBRID /* FIND ME! */ - p->off_heap.funs = NULL; -#endif - p->off_heap.overhead = 0; + tmp_offheap.first = NULL; + tmp_offheap.overhead = 0; top = DBTERM_BUF(p); - copy = copy_struct(obj, size, &top, &p->off_heap); + copy = copy_struct(obj, size, &top, &tmp_offheap); + p->first_oh = tmp_offheap.first; DBTERM_SET_TPL(p,tuple_val(copy)); return structp; @@ -2476,7 +2507,10 @@ void* db_get_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) void db_free_term_data(DbTerm* p) { - erts_cleanup_offheap(&p->off_heap); + ErlOffHeap tmp_offheap; + tmp_offheap.first = p->first_oh; + tmp_offheap.overhead = 0; + erts_cleanup_offheap(&tmp_offheap); } @@ -2606,7 +2640,7 @@ static void add_dmc_err(DMCErrInfo *err_info, static DMCRet dmc_one_term(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(Eterm) *stack, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm c) { Sint n; @@ -2704,27 +2738,80 @@ static DMCRet dmc_one_term(DMCContext *context, DMC_PUSH(*stack, c); break; case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE): - n = thing_arityval(*internal_ref_val(c)); DMC_PUSH(*text, matchEqRef); +#if HALFWORD_HEAP + { + union { + UWord u; + Uint t[2]; + } fiddle; + ASSERT(thing_arityval(*internal_ref_val(c)) == 3); + fiddle.t[0] = *internal_ref_val(c); + fiddle.t[1] = (Uint) internal_ref_val(c)[1]; + DMC_PUSH(*text, fiddle.u); + fiddle.t[0] = (Uint) internal_ref_val(c)[2]; + fiddle.t[1] = (Uint) internal_ref_val(c)[3]; + DMC_PUSH(*text, fiddle.u); + } +#else + n = thing_arityval(*internal_ref_val(c)); DMC_PUSH(*text, *internal_ref_val(c)); for (i = 1; i <= n; ++i) { DMC_PUSH(*text, (Uint) internal_ref_val(c)[i]); } +#endif break; case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE): case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE): n = thing_arityval(*big_val(c)); DMC_PUSH(*text, matchEqBig); +#if HALFWORD_HEAP + { + union { + UWord u; + Uint t[2]; + } fiddle; + ASSERT(n >= 1); + fiddle.t[0] = *big_val(c); + fiddle.t[1] = big_val(c)[1]; + DMC_PUSH(*text, fiddle.u); + for (i = 2; i <= n; ++i) { + fiddle.t[0] = big_val(c)[i]; + if (++i <= n) { + fiddle.t[1] = big_val(c)[i]; + } else { + fiddle.t[1] = (Uint) 0; + } + DMC_PUSH(*text, fiddle.u); + } + } +#else DMC_PUSH(*text, *big_val(c)); for (i = 1; i <= n; ++i) { DMC_PUSH(*text, (Uint) big_val(c)[i]); } +#endif break; case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): DMC_PUSH(*text,matchEqFloat); +#if HALFWORD_HEAP + { + union { + UWord u; + Uint t[2]; + } fiddle; + fiddle.t[0] = float_val(c)[1]; + fiddle.t[1] = float_val(c)[2]; + DMC_PUSH(*text, fiddle.u); + } +#else DMC_PUSH(*text, (Uint) float_val(c)[1]); - /* XXX: this reads and pushes random junk on ARCH_64 */ +#ifdef ARCH_64 + DMC_PUSH(*text, (Uint) 0); +#else DMC_PUSH(*text, (Uint) float_val(c)[2]); +#endif +#endif break; default: /* BINARY, FUN, VECTOR, or EXTERNAL */ /* @@ -2753,7 +2840,7 @@ static DMCRet dmc_one_term(DMCContext *context, ** Match guard compilation */ -static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(Uint) *text, +static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(UWord) *text, Eterm t) { int sz; @@ -2807,7 +2894,7 @@ add_dmc_err((ContextP)->err_info, String, -1, T, dmcWarning) static DMCRet dmc_list(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -2843,11 +2930,11 @@ static DMCRet dmc_list(DMCContext *context, static DMCRet dmc_tuple(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { - DMC_STACK_TYPE(Uint) instr_save; + DMC_STACK_TYPE(UWord) instr_save; int all_constant = 1; int textpos = DMC_STACK_NUM(*text); Eterm *p = tuple_val(t); @@ -2903,7 +2990,7 @@ static DMCRet dmc_tuple(DMCContext *context, static DMCRet dmc_whole_expression(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -2931,7 +3018,7 @@ static DMCRet dmc_whole_expression(DMCContext *context, static DMCRet dmc_variable(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -2952,7 +3039,7 @@ static DMCRet dmc_variable(DMCContext *context, static DMCRet dmc_all_bindings(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -2979,7 +3066,7 @@ static DMCRet dmc_all_bindings(DMCContext *context, static DMCRet dmc_const(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -2996,7 +3083,7 @@ static DMCRet dmc_const(DMCContext *context, static DMCRet dmc_and(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3025,7 +3112,7 @@ static DMCRet dmc_and(DMCContext *context, static DMCRet dmc_or(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3055,7 +3142,7 @@ static DMCRet dmc_or(DMCContext *context, static DMCRet dmc_andalso(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3104,7 +3191,7 @@ static DMCRet dmc_andalso(DMCContext *context, static DMCRet dmc_orelse(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3152,7 +3239,7 @@ static DMCRet dmc_orelse(DMCContext *context, static DMCRet dmc_message(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3194,7 +3281,7 @@ static DMCRet dmc_message(DMCContext *context, static DMCRet dmc_self(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3214,7 +3301,7 @@ static DMCRet dmc_self(DMCContext *context, static DMCRet dmc_return_trace(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3244,7 +3331,7 @@ static DMCRet dmc_return_trace(DMCContext *context, static DMCRet dmc_exception_trace(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3276,7 +3363,7 @@ static DMCRet dmc_exception_trace(DMCContext *context, static DMCRet dmc_is_seq_trace(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3302,7 +3389,7 @@ static DMCRet dmc_is_seq_trace(DMCContext *context, static DMCRet dmc_set_seq_token(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3351,7 +3438,7 @@ static DMCRet dmc_set_seq_token(DMCContext *context, static DMCRet dmc_get_seq_token(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3388,7 +3475,7 @@ static DMCRet dmc_get_seq_token(DMCContext *context, static DMCRet dmc_display(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3428,7 +3515,7 @@ static DMCRet dmc_display(DMCContext *context, static DMCRet dmc_process_dump(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3458,7 +3545,7 @@ static DMCRet dmc_process_dump(DMCContext *context, static DMCRet dmc_enable_trace(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3518,7 +3605,7 @@ static DMCRet dmc_enable_trace(DMCContext *context, static DMCRet dmc_disable_trace(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3578,7 +3665,7 @@ static DMCRet dmc_disable_trace(DMCContext *context, static DMCRet dmc_trace(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3652,7 +3739,7 @@ static DMCRet dmc_trace(DMCContext *context, static DMCRet dmc_caller(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3685,7 +3772,7 @@ static DMCRet dmc_caller(DMCContext *context, static DMCRet dmc_silent(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3727,7 +3814,7 @@ static DMCRet dmc_silent(DMCContext *context, static DMCRet dmc_fun(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3844,7 +3931,7 @@ static DMCRet dmc_fun(DMCContext *context, erl_exit(1,"ets:match() internal error, " "guard with more than 3 arguments."); } - DMC_PUSH(*text, (Uint) b->biff); + DMC_PUSH(*text, (UWord) b->biff); context->stack_used -= (((int) a) - 2); if (context->stack_used > context->stack_need) context->stack_need = context->stack_used; @@ -3853,7 +3940,7 @@ static DMCRet dmc_fun(DMCContext *context, static DMCRet dmc_expr(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3916,7 +4003,7 @@ static DMCRet dmc_expr(DMCContext *context, static DMCRet compile_guard_expr(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm l) { DMCRet ret; @@ -4031,7 +4118,7 @@ static int match_compact(ErlHeapFragment *expr, DMCErrInfo *err_info) DMC_INIT_STACK(heap); p = expr->mem; - i = expr->size; + i = expr->used_size; while (i--) { if (is_thing(*p)) { a = thing_arityval(*p); @@ -4060,7 +4147,7 @@ static int match_compact(ErlHeapFragment *expr, DMCErrInfo *err_info) } p = expr->mem; - i = expr->size; + i = expr->used_size; while (i--) { if (is_thing(*p)) { a = thing_arityval(*p); @@ -4230,7 +4317,7 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) Eterm l; Uint32 ret_flags; Uint sz; - Eterm *save_cp; + BeamInstr *save_cp; if (trace && !(is_list(against) || against == NIL)) { return THE_NON_VALUE; @@ -4271,17 +4358,18 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) ++n; l = CDR(list_val(l)); } + save_cp = p->cp; + p->cp = NULL; + res = erts_match_set_run(p, mps, arr, n, &ret_flags); + p->cp = save_cp; } else { n = 0; - arr = (Eterm *) against; + arr = NULL; + res = erts_match_set_run_ets(p, mps, against, n, &ret_flags); } /* We are in the context of a BIF, {caller} should return 'undefined' */ - save_cp = p->cp; - p->cp = NULL; - res = erts_match_set_run(p, mps, arr, n, &ret_flags); - p->cp = save_cp; if (is_non_value(res)) { res = am_false; } @@ -4324,7 +4412,7 @@ static Eterm seq_trace_fake(Process *p, Eterm arg1) static void db_match_dis(Binary *bp) { MatchProg *prog = Binary2MatchProg(bp); - Uint *t = prog->text; + UWord *t = prog->text; Uint n; Eterm p; int first; @@ -4390,41 +4478,48 @@ static void db_match_dis(Binary *bp) break; case matchEqRef: ++t; - n = thing_arityval(*t); - ++t; - erts_printf("EqRef\t(%d) {", (int) n); - first = 1; - while (n--) { - if (first) - first = 0; - else - erts_printf(", "); -#ifdef ARCH_64 - erts_printf("0x%016bpx", *t); + { + RefThing *rt = (RefThing *) t; + int ri; + n = thing_arityval(rt->header); + erts_printf("EqRef\t(%d) {", (int) n); + first = 1; + for (ri = 0; ri < n; ++ri) { + if (first) + first = 0; + else + erts_printf(", "); +#if defined(ARCH_64) && !HALFWORD_HEAP + erts_printf("0x%016bpx", rt->data.ui[ri]); #else - erts_printf("0x%08bpx", *t); + erts_printf("0x%08bpx", rt->data.ui[ri]); #endif - ++t; + } } + t += TermWords(REF_THING_SIZE); erts_printf("}\n"); break; case matchEqBig: ++t; n = thing_arityval(*t); - ++t; - erts_printf("EqBig\t(%d) {", (int) n); - first = 1; - while (n--) { - if (first) - first = 0; - else - erts_printf(", "); -#ifdef ARCH_64 - erts_printf("0x%016bpx", *t); + { + Eterm *et = (Eterm *) t; + t += TermWords(n+1); + erts_printf("EqBig\t(%d) {", (int) n); + first = 1; + ++n; + while (n--) { + if (first) + first = 0; + else + erts_printf(", "); +#if defined(ARCH_64) && !HALFWORD_HEAP + erts_printf("0x%016bpx", *et); #else - erts_printf("0x%08bpx", *t); + erts_printf("0x%08bpx", *et); #endif - ++t; + ++et; + } } erts_printf("}\n"); break; @@ -4432,8 +4527,8 @@ static void db_match_dis(Binary *bp) ++t; { double num; - memcpy(&num,t, 2 * sizeof(*t)); - t += 2; + memcpy(&num,t,sizeof(double)); + t += TermWords(2); erts_printf("EqFloat\t%f\n", num); } break; diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 4fc7b4f52e..0f333e8b34 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1998-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1998-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -57,7 +57,7 @@ * A datatype for a database entry stored out of a process heap */ typedef struct db_term { - ErlOffHeap off_heap; /* Off heap data for term. */ + struct erl_off_heap_header* first_oh; /* Off heap data for term. */ Uint size; /* Size of term in "words" */ Eterm tpl[1]; /* Untagged "constant pointer" to top tuple */ /* (assumed to be first in buffer) */ @@ -212,7 +212,7 @@ typedef struct db_table_common { #endif Eterm owner; /* Pid of the creator */ Eterm heir; /* Pid of the heir */ - Eterm heir_data; /* To send in ETS-TRANSFER (is_immed or (DbTerm*) */ + UWord heir_data; /* To send in ETS-TRANSFER (is_immed or (DbTerm*) */ SysTimeval heir_started; /* To further identify the heir */ Eterm the_name; /* an atom */ Eterm id; /* atom | integer */ @@ -240,8 +240,9 @@ typedef struct db_table_common { #define DB_DUPLICATE_BAG (1 << 8) #define DB_ORDERED_SET (1 << 9) #define DB_DELETE (1 << 10) /* table is being deleted */ +#define DB_FREQ_READ (1 << 11) -#define ERTS_ETS_TABLE_TYPES (DB_BAG|DB_SET|DB_DUPLICATE_BAG|DB_ORDERED_SET|DB_FINE_LOCKED) +#define ERTS_ETS_TABLE_TYPES (DB_BAG|DB_SET|DB_DUPLICATE_BAG|DB_ORDERED_SET|DB_FINE_LOCKED|DB_FREQ_READ) #define IS_HASH_TABLE(Status) (!!((Status) & \ (DB_BAG | DB_SET | DB_DUPLICATE_BAG))) @@ -304,9 +305,9 @@ typedef struct match_prog { Uint eheap_offset; Uint stack_offset; #ifdef DMC_DEBUG - Uint* prog_end; /* End of program */ + UWord* prog_end; /* End of program */ #endif - Uint text[1]; /* Beginning of program */ + UWord text[1]; /* Beginning of program */ } MatchProg; /* @@ -366,7 +367,7 @@ Binary *db_match_compile(Eterm *matchexpr, Eterm *guards, Uint flags, DMCErrInfo *err_info); /* Returns newly allocated MatchProg binary with refc == 0*/ -Eterm db_prog_match(Process *p, Binary *prog, Eterm term, int arity, +Eterm db_prog_match(Process *p, Binary *prog, Eterm term, Eterm *termp, int arity, Uint32 *return_flags /* Zeroed on enter */); /* returns DB_ERROR_NONE if matches, 1 if not matches and some db error on error. */ diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c index e5c3c76fdd..d7d6fcf0a2 100644 --- a/erts/emulator/beam/erl_debug.c +++ b/erts/emulator/beam/erl_debug.c @@ -207,11 +207,7 @@ pdisplay1(int to, void *to_arg, Process* p, Eterm obj) case FLOAT_DEF: { FloatDef ff; GET_DOUBLE(obj, ff); -#ifdef _OSE_ - erts_print(to, to_arg, "%e", ff.fd); -#else erts_print(to, to_arg, "%.20e", ff.fd); -#endif } break; case BINARY_DEF: @@ -235,9 +231,9 @@ pps(Process* p, Eterm* stop) } while(sp >= stop) { - erts_print(to, to_arg, "%0*lx: ", PTR_SIZE, (Eterm) sp); + erts_print(to, to_arg, "%0*lx: ", PTR_SIZE, (UWord) sp); if (is_catch(*sp)) { - erts_print(to, to_arg, "catch %d", (Uint)catch_pc(*sp)); + erts_print(to, to_arg, "catch %ld", (UWord)catch_pc(*sp)); } else { paranoid_display(to, to_arg, p, *sp); } @@ -265,7 +261,7 @@ static int verify_eterm(Process *p,Eterm element) return 1; for (mbuf = p->mbuf; mbuf; mbuf = mbuf->next) { - if (WITHIN(ptr, &mbuf->mem[0], &mbuf->mem[0] + mbuf->size)) { + if (WITHIN(ptr, &mbuf->mem[0], &mbuf->mem[0] + mbuf->used_size)) { return 1; } } @@ -312,7 +308,7 @@ void erts_check_stack(Process *p) if (IN_HEAP(p, ptr)) continue; for (mbuf = p->mbuf; mbuf; mbuf = mbuf->next) - if (WITHIN(ptr, &mbuf->mem[0], &mbuf->mem[0] + mbuf->size)) { + if (WITHIN(ptr, &mbuf->mem[0], &mbuf->mem[0] + mbuf->used_size)) { in_mbuf = 1; break; } @@ -750,7 +746,7 @@ static void print_process_memory(Process *p) PTR_SIZE, "heap fragments", dashes, dashes, dashes, dashes); while (bp) { - print_untagged_memory(bp->mem,bp->mem + bp->size); + print_untagged_memory(bp->mem,bp->mem + bp->used_size); bp = bp->next; } } @@ -895,5 +891,29 @@ void print_memory_info(Process *p) #endif erts_printf("+-----------------%s-%s-%s-%s-+\n",dashes,dashes,dashes,dashes); } +#if !HEAP_ON_C_STACK && defined(DEBUG) +Eterm *erts_debug_allocate_tmp_heap(int size, Process *p) +{ + ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p)); + int offset = sd->num_tmp_heap_used; + + ASSERT(offset+size <= TMP_HEAP_SIZE); + return (sd->tmp_heap)+offset; +} +void erts_debug_use_tmp_heap(int size, Process *p) +{ + ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p)); + + sd->num_tmp_heap_used += size; + ASSERT(sd->num_tmp_heap_used <= TMP_HEAP_SIZE); +} +void erts_debug_unuse_tmp_heap(int size, Process *p) +{ + ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p)); + + sd->num_tmp_heap_used -= size; + ASSERT(sd->num_tmp_heap_used >= 0); +} +#endif #endif diff --git a/erts/emulator/beam/erl_debug.h b/erts/emulator/beam/erl_debug.h index 74f4a00b63..bdfbaddbbf 100644 --- a/erts/emulator/beam/erl_debug.h +++ b/erts/emulator/beam/erl_debug.h @@ -1,26 +1,27 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2004-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ #ifndef _ERL_DEBUG_H_ #define _ERL_DEBUG_H_ - #ifdef DEBUG +#include "erl_term.h" + #ifdef HIPE #include "hipe_debug.h" #endif @@ -92,6 +93,11 @@ extern void print_tagged_memory(Eterm *start, Eterm *end); extern void print_untagged_memory(Eterm *start, Eterm *end); extern void print_memory(Process *p); extern void print_memory_info(Process *p); +#if defined(DEBUG) && !HEAP_ON_C_STACK +extern Eterm *erts_debug_allocate_tmp_heap(int, Process *); +extern void erts_debug_use_tmp_heap(int, Process *); +extern void erts_debug_unuse_tmp_heap(int, Process *); +#endif #ifdef HYBRID extern void print_ma_info(void); diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 489e74d960..9733c0e5b5 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -48,6 +48,10 @@ # define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG # undef SIZEOF_LONG_LONG #endif +#ifdef HALFWORD_HEAP_EMULATOR +# define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR +# undef HALFWORD_HEAP_EMULATOR +#endif #include "erl_int_sizes_config.h" #if defined(SIZEOF_CHAR_SAVED__) && SIZEOF_CHAR_SAVED__ != SIZEOF_CHAR # error SIZEOF_CHAR mismatch @@ -65,6 +69,11 @@ # error SIZEOF_LONG_LONG mismatch #endif +/* This is OK to override by the NIF/driver implementor */ +#if defined(HALFWORD_HEAP_EMULATOR_SAVED__) && !defined(HALFWORD_HEAP_EMULATOR) +#define HALFWORD_HEAP_EMULATOR HALFWORD_HEAP_EMULATOR_SAVED__ +#endif + #include "erl_drv_nif.h" #include <stdlib.h> diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index 50d8c25c46..d42820ddf3 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2007-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2007-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -186,10 +186,9 @@ int erl_drv_mutex_trylock(ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res = dmtx ? ethr_mutex_trylock(&dmtx->mtx) : EINVAL; - if (res != 0 && res != EBUSY) - fatal_error(res, "erl_drv_mutex_trylock()"); - return res; + if (!dmtx) + fatal_error(EINVAL, "erl_drv_mutex_trylock()"); + return ethr_mutex_trylock(&dmtx->mtx); #else return 0; #endif @@ -199,9 +198,9 @@ void erl_drv_mutex_lock(ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res = dmtx ? ethr_mutex_lock(&dmtx->mtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_mutex_lock()"); + if (!dmtx) + fatal_error(EINVAL, "erl_drv_mutex_lock()"); + ethr_mutex_lock(&dmtx->mtx); #endif } @@ -209,9 +208,9 @@ void erl_drv_mutex_unlock(ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res = dmtx ? ethr_mutex_unlock(&dmtx->mtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_mutex_unlock()"); + if (!dmtx) + fatal_error(EINVAL, "erl_drv_mutex_unlock()"); + ethr_mutex_unlock(&dmtx->mtx); #endif } @@ -256,9 +255,9 @@ void erl_drv_cond_signal(ErlDrvCond *dcnd) { #ifdef USE_THREADS - int res = dcnd ? ethr_cond_signal(&dcnd->cnd) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_cond_signal()"); + if (!dcnd) + fatal_error(EINVAL, "erl_drv_cond_signal()"); + ethr_cond_signal(&dcnd->cnd); #endif } @@ -266,9 +265,9 @@ void erl_drv_cond_broadcast(ErlDrvCond *dcnd) { #ifdef USE_THREADS - int res = dcnd ? ethr_cond_broadcast(&dcnd->cnd) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_cond_broadcast()"); + if (!dcnd) + fatal_error(EINVAL, "erl_drv_cond_broadcast()"); + ethr_cond_broadcast(&dcnd->cnd); #endif } @@ -277,18 +276,13 @@ void erl_drv_cond_wait(ErlDrvCond *dcnd, ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res; if (!dcnd || !dmtx) { - res = EINVAL; - error: - fatal_error(res, "erl_drv_cond_wait()"); + fatal_error(EINVAL, "erl_drv_cond_wait()"); } while (1) { - res = ethr_cond_wait(&dcnd->cnd, &dmtx->mtx); + int res = ethr_cond_wait(&dcnd->cnd, &dmtx->mtx); if (res == 0) break; - if (res != EINTR) - goto error; } #endif } @@ -333,10 +327,9 @@ int erl_drv_rwlock_tryrlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_tryrlock(&drwlck->rwmtx) : EINVAL; - if (res != 0 && res != EBUSY) - fatal_error(res, "erl_drv_rwlock_tryrlock()"); - return res; + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_tryrlock()"); + return ethr_rwmutex_tryrlock(&drwlck->rwmtx); #else return 0; #endif @@ -346,9 +339,9 @@ void erl_drv_rwlock_rlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_rlock(&drwlck->rwmtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_rwlock_rlock()"); + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_rlock()"); + ethr_rwmutex_rlock(&drwlck->rwmtx); #endif } @@ -356,9 +349,9 @@ void erl_drv_rwlock_runlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_runlock(&drwlck->rwmtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_rwlock_runlock()"); + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_runlock()"); + ethr_rwmutex_runlock(&drwlck->rwmtx); #endif } @@ -366,10 +359,9 @@ int erl_drv_rwlock_tryrwlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_tryrwlock(&drwlck->rwmtx) : EINVAL; - if (res != 0 && res != EBUSY) - fatal_error(res, "erl_drv_rwlock_tryrwlock()"); - return res; + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_tryrwlock()"); + return ethr_rwmutex_tryrwlock(&drwlck->rwmtx); #else return 0; #endif @@ -379,9 +371,9 @@ void erl_drv_rwlock_rwlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_rwlock(&drwlck->rwmtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_rwlock_rwlock()"); + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_rwlock()"); + ethr_rwmutex_rwlock(&drwlck->rwmtx); #endif } @@ -389,9 +381,9 @@ void erl_drv_rwlock_rwunlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_rwunlock(&drwlck->rwmtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_rwlock_rwunlock()"); + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_rwunlock()"); + ethr_rwmutex_rwunlock(&drwlck->rwmtx); #endif } @@ -603,11 +595,7 @@ erl_drv_thread_create(char *name, dtid->name = ((char *) dtid) + sizeof(struct ErlDrvTid_); sys_strcpy(dtid->name, name); } -#ifdef ERTS_ENABLE_LOCK_COUNT - res = erts_lcnt_thr_create(&dtid->tid, erl_drv_thread_wrapper, dtid, use_opts); -#else res = ethr_thr_create(&dtid->tid, erl_drv_thread_wrapper, dtid, use_opts); -#endif if (res != 0) { erts_free(ERTS_ALC_T_DRV_TID, dtid); diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index 79e844b315..5dce5ad262 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2000-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -37,8 +37,6 @@ static erts_smp_rwmtx_t erts_fun_table_lock; #define erts_fun_read_unlock() erts_smp_rwmtx_runlock(&erts_fun_table_lock) #define erts_fun_write_lock() erts_smp_rwmtx_rwlock(&erts_fun_table_lock) #define erts_fun_write_unlock() erts_smp_rwmtx_rwunlock(&erts_fun_table_lock) -#define erts_fun_init_lock() erts_smp_rwmtx_init(&erts_fun_table_lock, \ - "fun_tab") static HashValue fun_hash(ErlFunEntry* obj); static int fun_cmp(ErlFunEntry* obj1, ErlFunEntry* obj2); @@ -50,15 +48,19 @@ static void fun_free(ErlFunEntry* obj); * to unloaded_fun[]. The -1 in unloaded_fun[0] will be interpreted * as an illegal arity when attempting to call a fun. */ -static Eterm unloaded_fun_code[3] = {NIL, -1, 0}; -static Eterm* unloaded_fun = unloaded_fun_code + 2; +static BeamInstr unloaded_fun_code[3] = {NIL, -1, 0}; +static BeamInstr* unloaded_fun = unloaded_fun_code + 2; void erts_init_fun_table(void) { HashFunctions f; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + erts_smp_rwmtx_init_opt(&erts_fun_table_lock, &rwmtx_opt, "fun_tab"); - erts_fun_init_lock(); f.hash = (H_FUN) fun_hash; f.cmp = (HCMP_FUN) fun_cmp; f.alloc = (HALLOC_FUN) fun_alloc; @@ -192,22 +194,8 @@ erts_erase_fun_entry(ErlFunEntry* fe) erts_fun_write_unlock(); } -#ifndef HYBRID /* FIND ME! */ -void -erts_cleanup_funs(ErlFunThing* funp) -{ - while (funp) { - ErlFunEntry* fe = funp->fe; - if (erts_refc_dectest(&fe->refc, 0) == 0) { - erts_erase_fun_entry(fe); - } - funp = funp->next; - } -} -#endif - void -erts_cleanup_funs_on_purge(Eterm* start, Eterm* end) +erts_cleanup_funs_on_purge(BeamInstr* start, BeamInstr* end) { int limit; HashBucket** bucket; @@ -222,7 +210,7 @@ erts_cleanup_funs_on_purge(Eterm* start, Eterm* end) while (b) { ErlFunEntry* fe = (ErlFunEntry *) b; - Eterm* addr = fe->address; + BeamInstr* addr = fe->address; if (start <= addr && addr < end) { fe->address = unloaded_fun; diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index fb5e75649b..2f165afa06 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2000-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -33,10 +33,10 @@ typedef struct erl_fun_entry { int index; /* New style index. */ int old_uniq; /* Unique number (old_style) */ int old_index; /* Old style index */ - Eterm* address; /* Pointer to code for fun */ + BeamInstr* address; /* Pointer to code for fun */ #ifdef HIPE - Eterm* native_address; /* Native entry code for fun. */ + UWord* native_address; /* Native entry code for fun. */ #endif Uint arity; /* The arity of the fun. */ @@ -53,12 +53,12 @@ typedef struct erl_fun_entry { typedef struct erl_fun_thing { Eterm thing_word; /* Subtag FUN_SUBTAG. */ + ErlFunEntry* fe; /* Pointer to fun entry. */ #ifndef HYBRID /* FIND ME! */ - struct erl_fun_thing* next; /* Next fun in mso list. */ + struct erl_off_heap_header* next; #endif - ErlFunEntry* fe; /* Pointer to fun entry. */ #ifdef HIPE - Eterm* native_address; /* Native code for the fun. */ + UWord* native_address; /* Native code for the fun. */ #endif Uint arity; /* The arity of the fun. */ Uint num_free; /* Number of free variables (in env). */ @@ -86,7 +86,7 @@ void erts_erase_fun_entry(ErlFunEntry* fe); #ifndef HYBRID /* FIND ME! */ void erts_cleanup_funs(ErlFunThing* funp); #endif -void erts_cleanup_funs_on_purge(Eterm* start, Eterm* end); +void erts_cleanup_funs_on_purge(BeamInstr* start, BeamInstr* end); void erts_dump_fun_entries(int, void *); #endif diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index e9bf37a173..0f4d2a2ef9 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -110,9 +110,7 @@ static Uint adjust_after_fullsweep(Process *p, int size_before, int need, Eterm *objv, int nobj); static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj); static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj); -static void sweep_proc_bins(Process *p, int fullsweep); -static void sweep_proc_funs(Process *p, int fullsweep); -static void sweep_proc_externals(Process *p, int fullsweep); +static void sweep_off_heap(Process *p, int fullsweep); static void offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size); static void offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size); static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size, @@ -126,7 +124,7 @@ static void disallow_heap_frag_ref_in_old_heap(Process* p); static void disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj); #endif -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP # define MAX_HEAP_SIZES 154 #else # define MAX_HEAP_SIZES 55 @@ -145,6 +143,16 @@ erts_init_gc(void) { int i = 0; + ASSERT(offsetof(ProcBin,thing_word) == offsetof(struct erl_off_heap_header,thing_word)); + ASSERT(offsetof(ProcBin,thing_word) == offsetof(ErlFunThing,thing_word)); + ASSERT(offsetof(ProcBin,thing_word) == offsetof(ExternalThing,header)); + ASSERT(offsetof(ProcBin,size) == offsetof(struct erl_off_heap_header,size)); + ASSERT(offsetof(ProcBin,size) == offsetof(ErlSubBin,size)); + ASSERT(offsetof(ProcBin,size) == offsetof(ErlHeapBin,size)); + ASSERT(offsetof(ProcBin,next) == offsetof(struct erl_off_heap_header,next)); + ASSERT(offsetof(ProcBin,next) == offsetof(ErlFunThing,next)); + ASSERT(offsetof(ProcBin,next) == offsetof(ExternalThing,next)); + erts_smp_spinlock_init(&info_lck, "gc_info"); garbage_cols = 0; reclaimed = 0; @@ -286,25 +294,14 @@ erts_offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, offset_heap_ptr(hp, sz, offs, (char *) low, ((char *)high)-((char *)low)); } + #define ptr_within(ptr, low, high) ((ptr) < (high) && (ptr) >= (low)) void erts_offset_off_heap(ErlOffHeap *ohp, Sint offs, Eterm* low, Eterm* high) { - if (ohp->mso && ptr_within((Eterm *)ohp->mso, low, high)) { - Eterm** uptr = (Eterm**) (void *) &ohp->mso; - *uptr += offs; - } - -#ifndef HYBRID /* FIND ME! */ - if (ohp->funs && ptr_within((Eterm *)ohp->funs, low, high)) { - Eterm** uptr = (Eterm**) (void *) &ohp->funs; - *uptr += offs; - } -#endif - - if (ohp->externals && ptr_within((Eterm *)ohp->externals, low, high)) { - Eterm** uptr = (Eterm**) (void *) &ohp->externals; + if (ohp->first && ptr_within((Eterm *)ohp->first, low, high)) { + Eterm** uptr = (Eterm**) (void *) &ohp->first; *uptr += offs; } } @@ -504,14 +501,8 @@ erts_garbage_collect_hibernate(Process* p) cleanup_rootset(&rootset); - if (MSO(p).mso) { - sweep_proc_bins(p, 1); - } - if (MSO(p).funs) { - sweep_proc_funs(p, 1); - } - if (MSO(p).externals) { - sweep_proc_externals(p, 1); + if (MSO(p).first) { + sweep_off_heap(p, 1); } /* @@ -667,7 +658,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size) case TAG_PRIMARY_BOXED: ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *g_ptr++ = val; } else if (in_area(ptr, area, area_size)) { @@ -679,7 +670,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size) case TAG_PRIMARY_LIST: ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { /* Moved */ + if (IS_MOVED_CONS(val)) { /* Moved */ *g_ptr++ = ptr[1]; } else if (in_area(ptr, area, area_size)) { MOVE_CONS(ptr,val,old_htop,g_ptr++); @@ -752,7 +743,10 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) * is large enough. */ - if (OLD_HEAP(p) && mature <= OLD_HEND(p) - OLD_HTOP(p)) { + if (OLD_HEAP(p) && + ((mature <= OLD_HEND(p) - OLD_HTOP(p)) && + ((BIN_VHEAP_MATURE(p) < ( BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)))) && + ((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) { ErlMessage *msgp; Uint size_after; Uint need_after; @@ -913,7 +907,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj) case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *g_ptr++ = val; } else if (in_area(ptr, heap, mature_size)) { @@ -929,7 +923,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj) case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { /* Moved */ + if (IS_MOVED_CONS(val)) { /* Moved */ *g_ptr++ = ptr[1]; } else if (in_area(ptr, heap, mature_size)) { MOVE_CONS(ptr,val,old_htop,g_ptr++); @@ -972,7 +966,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj) case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *n_hp++ = val; } else if (in_area(ptr, heap, mature_size)) { @@ -987,7 +981,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj) case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; } else if (in_area(ptr, heap, mature_size)) { MOVE_CONS(ptr,val,old_htop,n_hp++); @@ -1008,7 +1002,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj) Eterm* origptr = &(mb->orig); ptr = boxed_val(*origptr); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { *origptr = val; mb->base = binary_bytes(val); } else if (in_area(ptr, heap, mature_size)) { @@ -1041,15 +1035,8 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj) OLD_HTOP(p) = old_htop; HIGH_WATER(p) = (HEAP_START(p) != HIGH_WATER(p)) ? n_heap : n_htop; - if (MSO(p).mso) { - sweep_proc_bins(p, 0); - } - - if (MSO(p).funs) { - sweep_proc_funs(p, 0); - } - if (MSO(p).externals) { - sweep_proc_externals(p, 0); + if (MSO(p).first) { + sweep_off_heap(p, 0); } #ifdef HARDDEBUG @@ -1161,7 +1148,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *g_ptr++ = val; } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { @@ -1175,7 +1162,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { *g_ptr++ = ptr[1]; } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { MOVE_CONS(ptr,val,n_htop,g_ptr++); @@ -1216,7 +1203,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *n_hp++ = val; } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { @@ -1229,7 +1216,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { MOVE_CONS(ptr,val,n_htop,n_hp++); @@ -1249,7 +1236,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) origptr = &(mb->orig); ptr = boxed_val(*origptr); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { *origptr = val; mb->base = binary_bytes(*origptr); } else if (in_area(ptr, src, src_size) || @@ -1271,17 +1258,11 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) } } - if (MSO(p).mso) { - sweep_proc_bins(p, 1); - } - if (MSO(p).funs) { - sweep_proc_funs(p, 1); - } - if (MSO(p).externals) { - sweep_proc_externals(p, 1); + if (MSO(p).first) { + sweep_off_heap(p, 1); } - if (OLD_HEAP(p) != NULL) { + if (OLD_HEAP(p) != NULL) { ERTS_HEAP_FREE(ERTS_ALC_T_OLD_HEAP, OLD_HEAP(p), (OLD_HEND(p) - OLD_HEAP(p)) * sizeof(Eterm)); @@ -1305,6 +1286,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) HIGH_WATER(p) = HEAP_TOP(p); ErtsGcQuickSanityCheck(p); + /* * Copy newly received message onto the end of the new heap. */ @@ -1392,17 +1374,12 @@ combined_message_size(Process* p) static void remove_message_buffers(Process* p) { - ErlHeapFragment* bp = MBUF(p); - - MBUF(p) = NULL; - MBUF_SIZE(p) = 0; - while (bp != NULL) { - ErlHeapFragment* next_bp = bp->next; - free_message_buffer(bp); - bp = next_bp; - } + if (MBUF(p) != NULL) { + free_message_buffer(MBUF(p)); + MBUF(p) = NULL; + } + MBUF_SIZE(p) = 0; } - #ifdef HARDDEBUG /* @@ -1433,12 +1410,12 @@ disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj) case TAG_PRIMARY_BOXED: { ptr = _unchecked_boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); objv++; } else { for (qb = mbuf; qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) { + if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1450,11 +1427,11 @@ disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj) case TAG_PRIMARY_LIST: { ptr = _unchecked_list_val(gval); val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { objv++; } else { for (qb = mbuf; qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) { + if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1499,7 +1476,7 @@ disallow_heap_frag_ref_in_heap(Process* p) ptr = _unchecked_boxed_val(val); if (!in_area(ptr, heap, heap_size)) { for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) { + if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1509,7 +1486,7 @@ disallow_heap_frag_ref_in_heap(Process* p) ptr = _unchecked_list_val(val); if (!in_area(ptr, heap, heap_size)) { for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) { + if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1551,26 +1528,26 @@ disallow_heap_frag_ref_in_old_heap(Process* p) val = *hp++; switch (primary_tag(val)) { case TAG_PRIMARY_BOXED: - ptr = (Eterm *) val; + ptr = (Eterm *) EXPAND_POINTER(val); if (!in_area(ptr, old_heap, old_heap_size)) { if (in_area(ptr, new_heap, new_heap_size)) { abort(); } for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) { + if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } } break; case TAG_PRIMARY_LIST: - ptr = (Eterm *) val; + ptr = (Eterm *) EXPAND_POINTER(val); if (!in_area(ptr, old_heap, old_heap_size)) { if (in_area(ptr, new_heap, new_heap_size)) { abort(); } for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) { + if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1610,7 +1587,7 @@ sweep_rootset(Rootset* rootset, Eterm* htop, char* src, Uint src_size) case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *g_ptr++ = val; } else if (in_area(ptr, src, src_size)) { @@ -1623,7 +1600,7 @@ sweep_rootset(Rootset* rootset, Eterm* htop, char* src, Uint src_size) case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { /* Moved */ + if (IS_MOVED_CONS(val)) { *g_ptr++ = ptr[1]; } else if (in_area(ptr, src, src_size)) { MOVE_CONS(ptr,val,htop,g_ptr++); @@ -1657,7 +1634,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size) case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *n_hp++ = val; } else if (in_area(ptr, src, src_size)) { @@ -1670,7 +1647,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size) case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; } else if (in_area(ptr, src, src_size)) { MOVE_CONS(ptr,val,n_htop,n_hp++); @@ -1690,7 +1667,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size) origptr = &(mb->orig); ptr = boxed_val(*origptr); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { *origptr = val; mb->base = binary_bytes(*origptr); } else if (in_area(ptr, src, src_size)) { @@ -1722,7 +1699,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *heap_ptr++ = val; } else if (in_area(ptr, src, src_size)) { @@ -1735,7 +1712,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { *heap_ptr++ = ptr[1]; } else if (in_area(ptr, src, src_size)) { MOVE_CONS(ptr,val,htop,heap_ptr++); @@ -1830,28 +1807,6 @@ collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop, return n_htop; } -#ifdef DEBUG -static Eterm follow_moved(Eterm term) -{ - Eterm* ptr; - switch (primary_tag(term)) { - case TAG_PRIMARY_IMMED1: - break; - case TAG_PRIMARY_BOXED: - ptr = boxed_val(term); - if (IS_MOVED(*ptr)) term = *ptr; - break; - case TAG_PRIMARY_LIST: - ptr = list_val(term); - if (is_non_value(ptr[0])) term = ptr[1]; - break; - default: - abort(); - } - return term; -} -#endif - static Uint setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) { @@ -2030,8 +1985,8 @@ shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj) HEAP_SIZE(p) = new_sz; } -static Uint -do_next_vheap_size(Uint vheap, Uint vheap_sz) { +static Uint64 +do_next_vheap_size(Uint64 vheap, Uint64 vheap_sz) { /* grow * @@ -2048,131 +2003,53 @@ do_next_vheap_size(Uint vheap, Uint vheap_sz) { * ---------------------- */ - if (vheap > (Uint) (vheap_sz*3/4)) { + if ((Uint64) vheap/3 > (Uint64) (vheap_sz/4)) { + Uint64 new_vheap_sz = vheap_sz; - while(vheap > (Uint) (vheap_sz*3/4)) { - vheap_sz = vheap_sz*2; + while((Uint64) vheap/3 > (Uint64) (vheap_sz/4)) { + /* the golden ratio = 1.618 */ + new_vheap_sz = (Uint64) vheap_sz * 1.618; + if (new_vheap_sz < vheap_sz ) { + return vheap_sz; + } + vheap_sz = new_vheap_sz; } - return erts_next_heap_size(vheap_sz, 0); + return vheap_sz; } - if (vheap < (Uint) (vheap_sz/4)) { - return erts_next_heap_size((Uint) (vheap_sz / 2), 0); + if (vheap < (Uint64) (vheap_sz/4)) { + return (vheap_sz >> 1); } return vheap_sz; } -static Uint -next_vheap_size(Process* p, Uint vheap, Uint vheap_sz) { - vheap_sz = do_next_vheap_size(vheap, vheap_sz); - return vheap_sz < p->min_vheap_size ? p->min_vheap_size : vheap_sz; -} - -static void -sweep_proc_externals(Process *p, int fullsweep) -{ - ExternalThing** prev; - ExternalThing* ptr; - char* oh = 0; - Uint oh_size = 0; - - if (fullsweep == 0) { - oh = (char *) OLD_HEAP(p); - oh_size = (char *) OLD_HEND(p) - oh; - } - - prev = &MSO(p).externals; - ptr = MSO(p).externals; - - while (ptr) { - Eterm* ppt = (Eterm *) ptr; - - if (IS_MOVED(*ppt)) { /* Object is alive */ - ExternalThing* ro = external_thing_ptr(*ppt); - - *prev = ro; /* Patch to moved pos */ - prev = &ro->next; - ptr = ro->next; - } else if (in_area(ppt, oh, oh_size)) { - /* - * Object resides on old heap, and we just did a - * generational collection - keep object in list. - */ - prev = &ptr->next; - ptr = ptr->next; - } else { /* Object has not been moved - deref it */ - erts_deref_node_entry(ptr->node); - *prev = ptr = ptr->next; - } - } - ASSERT(*prev == NULL); -} - -static void -sweep_proc_funs(Process *p, int fullsweep) -{ - ErlFunThing** prev; - ErlFunThing* ptr; - char* oh = 0; - Uint oh_size = 0; - - if (fullsweep == 0) { - oh = (char *) OLD_HEAP(p); - oh_size = (char *) OLD_HEND(p) - oh; - } - - prev = &MSO(p).funs; - ptr = MSO(p).funs; - - while (ptr) { - Eterm* ppt = (Eterm *) ptr; - - if (IS_MOVED(*ppt)) { /* Object is alive */ - ErlFunThing* ro = (ErlFunThing *) fun_val(*ppt); - - *prev = ro; /* Patch to moved pos */ - prev = &ro->next; - ptr = ro->next; - } else if (in_area(ppt, oh, oh_size)) { - /* - * Object resides on old heap, and we just did a - * generational collection - keep object in list. - */ - prev = &ptr->next; - ptr = ptr->next; - } else { /* Object has not been moved - deref it */ - ErlFunEntry* fe = ptr->fe; - - *prev = ptr = ptr->next; - if (erts_refc_dectest(&fe->refc, 0) == 0) { - erts_erase_fun_entry(fe); - } - } - } - ASSERT(*prev == NULL); +static Uint64 +next_vheap_size(Process* p, Uint64 vheap, Uint64 vheap_sz) { + Uint64 new_vheap_sz = do_next_vheap_size(vheap, vheap_sz); + return new_vheap_sz < p->min_vheap_size ? p->min_vheap_size : new_vheap_sz; } struct shrink_cand_data { - ProcBin* new_candidates; - ProcBin* new_candidates_end; - ProcBin* old_candidates; + struct erl_off_heap_header* new_candidates; + struct erl_off_heap_header* new_candidates_end; + struct erl_off_heap_header* old_candidates; Uint no_of_candidates; Uint no_of_active; }; static ERTS_INLINE void link_live_proc_bin(struct shrink_cand_data *shrink, - ProcBin ***prevppp, - ProcBin **pbpp, + struct erl_off_heap_header*** prevppp, + struct erl_off_heap_header** currpp, int new_heap) { - ProcBin *pbp = *pbpp; - - *pbpp = pbp->next; + ProcBin *pbp = (ProcBin*) *currpp; + ASSERT(**prevppp == *currpp); + *currpp = pbp->next; if (pbp->flags & (PB_ACTIVE_WRITER|PB_IS_WRITABLE)) { ASSERT(((pbp->flags & (PB_ACTIVE_WRITER|PB_IS_WRITABLE)) == (PB_ACTIVE_WRITER|PB_IS_WRITABLE)) @@ -2189,15 +2066,16 @@ link_live_proc_bin(struct shrink_cand_data *shrink, /* Our allocators are 8 byte aligned, i.e., shrinking with less than 8 bytes will have no real effect */ if (unused >= 8) { /* A shrink candidate; save in candidate list */ + **prevppp = pbp->next; if (new_heap) { if (!shrink->new_candidates) - shrink->new_candidates_end = pbp; + shrink->new_candidates_end = (struct erl_off_heap_header*)pbp; pbp->next = shrink->new_candidates; - shrink->new_candidates = pbp; + shrink->new_candidates = (struct erl_off_heap_header*)pbp; } else { pbp->next = shrink->old_candidates; - shrink->old_candidates = pbp; + shrink->old_candidates = (struct erl_off_heap_header*)pbp; } shrink->no_of_candidates++; return; @@ -2205,83 +2083,117 @@ link_live_proc_bin(struct shrink_cand_data *shrink, } } - /* Not a shrink candidate; keep in original mso list */ - **prevppp = pbp; + /* Not a shrink candidate; keep in original mso list */ *prevppp = &pbp->next; - } -static void -sweep_proc_bins(Process *p, int fullsweep) +static void +sweep_off_heap(Process *p, int fullsweep) { struct shrink_cand_data shrink = {0}; - ProcBin** prev; - ProcBin* ptr; - Binary* bptr; - char* oh = NULL; - Uint oh_size = 0; - Uint bin_vheap = 0; + struct erl_off_heap_header* ptr; + struct erl_off_heap_header** prev; + char* oheap = NULL; + Uint oheap_sz = 0; + Uint64 bin_vheap = 0; +#ifdef DEBUG + int seen_mature = 0; +#endif if (fullsweep == 0) { - oh = (char *) OLD_HEAP(p); - oh_size = (char *) OLD_HEND(p) - oh; + oheap = (char *) OLD_HEAP(p); + oheap_sz = (char *) OLD_HEND(p) - oheap; } BIN_OLD_VHEAP(p) = 0; - prev = &MSO(p).mso; - ptr = MSO(p).mso; + prev = &MSO(p).first; + ptr = MSO(p).first; - /* - * Note: In R7 we no longer force a fullsweep when we find binaries - * on the old heap. The reason is that with the introduction of the - * bit syntax we can expect binaries to be used a lot more. Note that - * in earlier releases a brand new binary (or any other term) could - * be put on the old heap during a gen-gc fullsweep, but this is - * no longer the case in R7. + /* Firts part of the list will reside on the (old) new-heap. + * Keep if moved, otherwise deref. */ while (ptr) { - Eterm* ppt = (Eterm *) ptr; - - if (IS_MOVED(*ppt)) { /* Object is alive */ - bin_vheap += ptr->size / sizeof(Eterm); - ptr = (ProcBin*) binary_val(*ppt); - link_live_proc_bin(&shrink, - &prev, - &ptr, - !in_area(ptr, oh, oh_size)); - } else if (in_area(ppt, oh, oh_size)) { - /* - * Object resides on old heap, and we just did a - * generational collection - keep object in list. - */ - BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/ - link_live_proc_bin(&shrink, &prev, &ptr, 0); - } else { /* Object has not been moved - deref it */ - - *prev = ptr->next; - bptr = ptr->val; - if (erts_refc_dectest(&bptr->refc, 0) == 0) - erts_bin_free(bptr); - ptr = *prev; - } + if (IS_MOVED_BOXED(ptr->thing_word)) { + ASSERT(!in_area(ptr, oheap, oheap_sz)); + *prev = ptr = (struct erl_off_heap_header*) boxed_val(ptr->thing_word); + ASSERT(!IS_MOVED_BOXED(ptr->thing_word)); + if (ptr->thing_word == HEADER_PROC_BIN) { + int to_new_heap = !in_area(ptr, oheap, oheap_sz); + ASSERT(to_new_heap == !seen_mature || (!to_new_heap && (seen_mature=1))); + if (to_new_heap) { + bin_vheap += ptr->size / sizeof(Eterm); + } else { + BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/ + } + link_live_proc_bin(&shrink, &prev, &ptr, to_new_heap); + } + else { + prev = &ptr->next; + ptr = ptr->next; + } + } + else if (!in_area(ptr, oheap, oheap_sz)) { + /* garbage */ + switch (thing_subtag(ptr->thing_word)) { + case REFC_BINARY_SUBTAG: + { + Binary* bptr = ((ProcBin*)ptr)->val; + if (erts_refc_dectest(&bptr->refc, 0) == 0) { + erts_bin_free(bptr); + } + break; + } + case FUN_SUBTAG: + { + ErlFunEntry* fe = ((ErlFunThing*)ptr)->fe; + if (erts_refc_dectest(&fe->refc, 0) == 0) { + erts_erase_fun_entry(fe); + } + break; + } + default: + ASSERT(is_external_header(ptr->thing_word)); + erts_deref_node_entry(((ExternalThing*)ptr)->node); + } + *prev = ptr = ptr->next; + } + else break; /* and let old-heap loop continue */ } - if (BIN_OLD_VHEAP(p) >= BIN_OLD_VHEAP_SZ(p)) { - FLAGS(p) |= F_NEED_FULLSWEEP; + /* The rest of the list resides on old-heap, and we just did a + * generational collection - keep objects in list. + */ + while (ptr) { + ASSERT(in_area(ptr, oheap, oheap_sz)); + ASSERT(!IS_MOVED_BOXED(ptr->thing_word)); + if (ptr->thing_word == HEADER_PROC_BIN) { + BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/ + link_live_proc_bin(&shrink, &prev, &ptr, 0); + } + else { + ASSERT(is_fun_header(ptr->thing_word) || + is_external_header(ptr->thing_word)); + prev = &ptr->next; + ptr = ptr->next; + } } - BIN_VHEAP_SZ(p) = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p)); - BIN_OLD_VHEAP_SZ(p) = next_vheap_size(p, BIN_OLD_VHEAP(p), BIN_OLD_VHEAP_SZ(p)); - MSO(p).overhead = bin_vheap; + if (fullsweep) { + BIN_OLD_VHEAP_SZ(p) = next_vheap_size(p, BIN_OLD_VHEAP(p) + MSO(p).overhead, BIN_OLD_VHEAP_SZ(p)); + } + BIN_VHEAP_SZ(p) = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p)); + MSO(p).overhead = bin_vheap; + BIN_VHEAP_MATURE(p) = bin_vheap; /* * If we got any shrink candidates, check them out. */ if (shrink.no_of_candidates) { - ProcBin *candlist[] = {shrink.new_candidates, shrink.old_candidates}; + ProcBin *candlist[] = { (ProcBin*)shrink.new_candidates, + (ProcBin*)shrink.old_candidates }; Uint leave_unused = 0; int i; @@ -2293,21 +2205,21 @@ sweep_proc_bins(Process *p, int fullsweep) } for (i = 0; i < sizeof(candlist)/sizeof(candlist[0]); i++) { - - for (ptr = candlist[i]; ptr; ptr = ptr->next) { - Uint new_size = ptr->size; + ProcBin* pb; + for (pb = candlist[i]; pb; pb = (ProcBin*)pb->next) { + Uint new_size = pb->size; if (leave_unused) { new_size += (new_size * 100) / leave_unused; /* Our allocators are 8 byte aligned, i.e., shrinking with less than 8 bytes will have no real effect */ - if (new_size + 8 >= ptr->val->orig_size) + if (new_size + 8 >= pb->val->orig_size) continue; } - ptr->val = erts_bin_realloc(ptr->val, new_size); - ptr->val->orig_size = new_size; - ptr->bytes = (byte *) ptr->val->orig_bytes; + pb->val = erts_bin_realloc(pb->val, new_size); + pb->val->orig_size = new_size; + pb->bytes = (byte *) pb->val->orig_bytes; } } @@ -2316,21 +2228,20 @@ sweep_proc_bins(Process *p, int fullsweep) * We now potentially have the mso list divided into three lists: * - shrink candidates on new heap (inactive writable with unused data) * - shrink candidates on old heap (inactive writable with unused data) - * - other binaries (read only + active writable ...) + * - other binaries (read only + active writable ...) + funs and externals * * Put them back together: new candidates -> other -> old candidates * This order will ensure that the list only refers from new * generation to old and never from old to new *which is important*. */ if (shrink.new_candidates) { - if (prev == &MSO(p).mso) /* empty other binaries list */ + if (prev == &MSO(p).first) /* empty other binaries list */ prev = &shrink.new_candidates_end->next; else - shrink.new_candidates_end->next = MSO(p).mso; - MSO(p).mso = shrink.new_candidates; + shrink.new_candidates_end->next = MSO(p).first; + MSO(p).first = shrink.new_candidates; } } - *prev = shrink.old_candidates; } @@ -2361,15 +2272,17 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) tari = thing_arityval(val); switch (thing_subtag(val)) { case REFC_BINARY_SUBTAG: + case FUN_SUBTAG: + case EXTERNAL_PID_SUBTAG: + case EXTERNAL_PORT_SUBTAG: + case EXTERNAL_REF_SUBTAG: { - ProcBin* pb = (ProcBin*) hp; - Eterm** uptr = (Eterm **) (void *) &pb->next; + struct erl_off_heap_header* oh = (struct erl_off_heap_header*) hp; - if (*uptr && in_area((Eterm *)pb->next, area, area_size)) { + if (in_area(oh->next, area, area_size)) { + Eterm** uptr = (Eterm **) (void *) &oh->next; *uptr += offs; /* Patch the mso chain */ } - sz -= tari; - hp += tari + 1; } break; case BIN_MATCHSTATE_SUBTAG: @@ -2380,40 +2293,11 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) mb->orig = offset_ptr(mb->orig, offs); mb->base = binary_bytes(mb->orig); } - sz -= tari; - hp += tari + 1; } break; - case FUN_SUBTAG: - { - ErlFunThing* funp = (ErlFunThing *) hp; - Eterm** uptr = (Eterm **) (void *) &funp->next; - - if (*uptr && in_area((Eterm *)funp->next, area, area_size)) { - *uptr += offs; - } - sz -= tari; - hp += tari + 1; - } - break; - case EXTERNAL_PID_SUBTAG: - case EXTERNAL_PORT_SUBTAG: - case EXTERNAL_REF_SUBTAG: - { - ExternalThing* etp = (ExternalThing *) hp; - Eterm** uptr = (Eterm **) (void *) &etp->next; - - if (*uptr && in_area((Eterm *)etp->next, area, area_size)) { - *uptr += offs; - } - sz -= tari; - hp += tari + 1; - } - break; - default: - sz -= tari; - hp += tari + 1; } + sz -= tari; + hp += tari + 1; break; } default: @@ -2450,18 +2334,8 @@ offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size) { - if (MSO(p).mso && in_area((Eterm *)MSO(p).mso, area, area_size)) { - Eterm** uptr = (Eterm**) (void *) &MSO(p).mso; - *uptr += offs; - } - - if (MSO(p).funs && in_area((Eterm *)MSO(p).funs, area, area_size)) { - Eterm** uptr = (Eterm**) (void *) &MSO(p).funs; - *uptr += offs; - } - - if (MSO(p).externals && in_area((Eterm *)MSO(p).externals, area, area_size)) { - Eterm** uptr = (Eterm**) (void *) &MSO(p).externals; + if (MSO(p).first && in_area((Eterm *)MSO(p).first, area, area_size)) { + Eterm** uptr = (Eterm**) (void *) &MSO(p).first; *uptr += offs; } } @@ -2542,7 +2416,7 @@ within2(Eterm *ptr, Process *p, Eterm *real_htop) return 1; } while (bp != NULL) { - if (bp->mem <= ptr && ptr < bp->mem + bp->size) { + if (bp->mem <= ptr && ptr < bp->mem + bp->used_size) { return 1; } bp = bp->next; @@ -2556,7 +2430,7 @@ within2(Eterm *ptr, Process *p, Eterm *real_htop) hfp = erts_dist_ext_trailer(mp->data.dist_ext); else hfp = NULL; - if (hfp && hfp->mem <= ptr && ptr < hfp->mem + hfp->size) + if (hfp && hfp->mem <= ptr && ptr < hfp->mem + hfp->used_size) return 1; } mp = mp->next; @@ -2582,8 +2456,8 @@ do { \ __FILE__, __LINE__, #EXP); \ } while (0) -#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST -# define ERTS_EXTERNAL_VISITED_BIT ((Eterm) 1 << 31) +#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST +# define ERTS_OFFHEAP_VISITED_BIT ((Eterm) 1 << 31) #endif @@ -2593,62 +2467,45 @@ erts_check_off_heap2(Process *p, Eterm *htop) Eterm *oheap = (Eterm *) OLD_HEAP(p); Eterm *ohtop = (Eterm *) OLD_HTOP(p); int old; - ProcBin *pb; - ErlFunThing *eft; - ExternalThing *et; + union erl_off_heap_ptr u; old = 0; - for (pb = MSO(p).mso; pb; pb = pb->next) { - Eterm *ptr = (Eterm *) pb; - long refc = erts_refc_read(&pb->val->refc, 1); + for (u.hdr = MSO(p).first; u.hdr; u.hdr = u.hdr->next) { + long refc; + switch (thing_subtag(u.hdr->thing_word)) { + case REFC_BINARY_SUBTAG: + refc = erts_refc_read(&u.pb->val->refc, 1); + break; + case FUN_SUBTAG: + refc = erts_refc_read(&u.fun->fe->refc, 1); + break; + case EXTERNAL_PID_SUBTAG: + case EXTERNAL_PORT_SUBTAG: + case EXTERNAL_REF_SUBTAG: + refc = erts_refc_read(&u.ext->node->refc, 1); + break; + default: + ASSERT(!!"erts_check_off_heap2: Invalid thing_word"); + } ERTS_CHK_OFFHEAP_ASSERT(refc >= 1); +#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST + ERTS_CHK_OFFHEAP_ASSERT(!(u.hdr->thing_word & ERTS_EXTERNAL_VISITED_BIT)); + u.hdr->thing_word |= ERTS_OFFHEAP_VISITED_BIT; +#endif if (old) { - ERTS_CHK_OFFHEAP_ASSERT(oheap <= ptr && ptr < ohtop); + ERTS_CHK_OFFHEAP_ASSERT(oheap <= u.ep && u.ep < ohtop); } - else if (oheap <= ptr && ptr < ohtop) + else if (oheap <= u.ep && u.ep < ohtop) old = 1; else { - ERTS_CHK_OFFHEAP_ASSERT(within2(ptr, p, htop)); + ERTS_CHK_OFFHEAP_ASSERT(within2(u.ep, p, htop)); } } - old = 0; - for (eft = MSO(p).funs; eft; eft = eft->next) { - Eterm *ptr = (Eterm *) eft; - long refc = erts_refc_read(&eft->fe->refc, 1); - ERTS_CHK_OFFHEAP_ASSERT(refc >= 1); - if (old) - ERTS_CHK_OFFHEAP_ASSERT(oheap <= ptr && ptr < ohtop); - else if (oheap <= ptr && ptr < ohtop) - old = 1; - else - ERTS_CHK_OFFHEAP_ASSERT(within2(ptr, p, htop)); - } - - old = 0; - for (et = MSO(p).externals; et; et = et->next) { - Eterm *ptr = (Eterm *) et; - long refc = erts_refc_read(&et->node->refc, 1); - ERTS_CHK_OFFHEAP_ASSERT(refc >= 1); -#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST - ERTS_CHK_OFFHEAP_ASSERT(!(et->header & ERTS_EXTERNAL_VISITED_BIT)); -#endif - if (old) - ERTS_CHK_OFFHEAP_ASSERT(oheap <= ptr && ptr < ohtop); - else if (oheap <= ptr && ptr < ohtop) - old = 1; - else - ERTS_CHK_OFFHEAP_ASSERT(within2(ptr, p, htop)); -#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST - et->header |= ERTS_EXTERNAL_VISITED_BIT; -#endif - } - #ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST - for (et = MSO(p).externals; et; et = et->next) - et->header &= ~ERTS_EXTERNAL_VISITED_BIT; + for (u.hdr = MSO(p).first; u.hdr; u.hdr = u.hdr->next) + u.hdr->thing_word &= ~ERTS_OFFHEAP_VISITED_BIT; #endif - } void diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index af55b6363f..807ef8ae8d 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2007-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2007-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -22,11 +22,12 @@ /* GC declarations shared by beam/erl_gc.c and hipe/hipe_gc.c */ -#ifdef DEBUG +#if defined(DEBUG) && !ERTS_GLB_INLINE_INCL_FUNC_DEF # define HARDDEBUG 1 #endif -#define IS_MOVED(x) (!is_header((x))) +#define IS_MOVED_BOXED(x) (!is_header((x))) +#define IS_MOVED_CONS(x) (is_non_value((x))) #define MOVE_CONS(PTR,CAR,HTOP,ORIG) \ do { \ @@ -69,4 +70,28 @@ extern Uint erts_test_long_gc_sleep; int within(Eterm *ptr, Process *p); #endif +ERTS_GLB_INLINE Eterm follow_moved(Eterm term); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE Eterm follow_moved(Eterm term) +{ + Eterm* ptr; + switch (primary_tag(term)) { + case TAG_PRIMARY_IMMED1: + break; + case TAG_PRIMARY_BOXED: + ptr = boxed_val(term); + if (IS_MOVED_BOXED(*ptr)) term = *ptr; + break; + case TAG_PRIMARY_LIST: + ptr = list_val(term); + if (IS_MOVED_CONS(ptr[0])) term = ptr[1]; + break; + default: + ASSERT(!"strange tag in follow_moved"); + } + return term; +} +#endif + #endif /* __ERL_GC_H__ */ diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index ea2ba4d55c..76b206d76f 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2003-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2003-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -49,30 +49,30 @@ #define MIN_MBC_FIRST_FREE_SZ (4*1024) #define MAX_SUB_MASK_IX \ - ((((Uint)1) << (NO_OF_BKT_IX_BITS - SUB_MASK_IX_SHIFT)) - 1) -#define MAX_SUB_BKT_IX ((((Uint)1) << SUB_MASK_IX_SHIFT) - 1) + ((((UWord)1) << (NO_OF_BKT_IX_BITS - SUB_MASK_IX_SHIFT)) - 1) +#define MAX_SUB_BKT_IX ((((UWord)1) << SUB_MASK_IX_SHIFT) - 1) #define MAX_BKT_IX (NO_OF_BKTS - 1) -#define MIN_BLK_SZ UNIT_CEILING(sizeof(GFFreeBlock_t) + sizeof(Uint)) +#define MIN_BLK_SZ UNIT_CEILING(sizeof(GFFreeBlock_t) + sizeof(UWord)) -#define IX2SBIX(IX) ((IX) & (~(~((Uint)0) << SUB_MASK_IX_SHIFT))) +#define IX2SBIX(IX) ((IX) & (~(~((UWord)0) << SUB_MASK_IX_SHIFT))) #define IX2SMIX(IX) ((IX) >> SUB_MASK_IX_SHIFT) #define MAKE_BKT_IX(SMIX, SBIX) \ - ((((Uint)(SMIX)) << SUB_MASK_IX_SHIFT) | ((Uint)(SBIX))) + ((((UWord)(SMIX)) << SUB_MASK_IX_SHIFT) | ((UWord)(SBIX))) #define SET_BKT_MASK_IX(BM, IX) \ do { \ int sub_mask_ix__ = IX2SMIX((IX)); \ - (BM).main |= (((Uint) 1) << sub_mask_ix__); \ - (BM).sub[sub_mask_ix__] |= (((Uint)1) << IX2SBIX((IX))); \ + (BM).main |= (((UWord) 1) << sub_mask_ix__); \ + (BM).sub[sub_mask_ix__] |= (((UWord)1) << IX2SBIX((IX))); \ } while (0) #define UNSET_BKT_MASK_IX(BM, IX) \ do { \ int sub_mask_ix__ = IX2SMIX((IX)); \ - (BM).sub[sub_mask_ix__] &= ~(((Uint)1) << IX2SBIX((IX))); \ + (BM).sub[sub_mask_ix__] &= ~(((UWord)1) << IX2SBIX((IX))); \ if (!(BM).sub[sub_mask_ix__]) \ - (BM).main &= ~(((Uint)1) << sub_mask_ix__); \ + (BM).main &= ~(((UWord)1) << sub_mask_ix__); \ } while (0) /* Buckets ... */ @@ -263,8 +263,8 @@ find_bucket(BucketMask_t *bmask, int min_index) while(max != min) { \ mid = ((max - min) >> 1) + min; \ if((BitMask) \ - & (~(~((Uint) 0) << (mid + 1))) \ - & (~((Uint) 0) << min)) \ + & (~(~((UWord) 0) << (mid + 1))) \ + & (~((UWord) 0) << min)) \ max = mid; \ else \ min = mid + 1; \ @@ -272,21 +272,21 @@ find_bucket(BucketMask_t *bmask, int min_index) (MinBit) = min - ASSERT(bmask->main < (((Uint) 1) << (MAX_SUB_MASK_IX+1))); + ASSERT(bmask->main < (((UWord) 1) << (MAX_SUB_MASK_IX+1))); sub_mask_ix = IX2SMIX(min_index); - if ((bmask->main & (~((Uint) 0) << sub_mask_ix)) == 0) + if ((bmask->main & (~((UWord) 0) << sub_mask_ix)) == 0) return -1; /* There exists a non empty bucket; find it... */ - if (bmask->main & (((Uint) 1) << sub_mask_ix)) { + if (bmask->main & (((UWord) 1) << sub_mask_ix)) { sub_bkt_ix = IX2SBIX(min_index); - if ((bmask->sub[sub_mask_ix] & (~((Uint) 0) << sub_bkt_ix)) == 0) { + if ((bmask->sub[sub_mask_ix] & (~((UWord) 0) << sub_bkt_ix)) == 0) { sub_mask_ix++; sub_bkt_ix = 0; - if ((bmask->main & (~((Uint) 0)<< sub_mask_ix)) == 0) + if ((bmask->main & (~((UWord) 0)<< sub_mask_ix)) == 0) return -1; } else @@ -299,17 +299,17 @@ find_bucket(BucketMask_t *bmask, int min_index) ASSERT(sub_mask_ix <= MAX_SUB_MASK_IX); /* Has to be a bit > sub_mask_ix */ - ASSERT(bmask->main & (~((Uint) 0) << (sub_mask_ix))); + ASSERT(bmask->main & (~((UWord) 0) << (sub_mask_ix))); GET_MIN_BIT(sub_mask_ix, bmask->main, sub_mask_ix, MAX_SUB_MASK_IX); find_sub_bkt_ix: ASSERT(sub_mask_ix <= MAX_SUB_MASK_IX); ASSERT(sub_bkt_ix <= MAX_SUB_BKT_IX); - if ((bmask->sub[sub_mask_ix] & (((Uint) 1) << sub_bkt_ix)) == 0) { + if ((bmask->sub[sub_mask_ix] & (((UWord) 1) << sub_bkt_ix)) == 0) { ASSERT(sub_mask_ix + 1 <= MAX_SUB_BKT_IX); /* Has to be a bit > sub_bkt_ix */ - ASSERT(bmask->sub[sub_mask_ix] & (~((Uint) 0) << sub_bkt_ix)); + ASSERT(bmask->sub[sub_mask_ix] & (~((UWord) 0) << sub_bkt_ix)); GET_MIN_BIT(sub_bkt_ix, bmask->sub[sub_mask_ix], @@ -336,7 +336,7 @@ search_bucket(Allctr_t *allctr, int ix, Uint size) Uint min_sz; Uint blk_sz; Uint cand_sz = 0; - Uint max_blk_search; + UWord max_blk_search; GFFreeBlock_t *blk; GFFreeBlock_t *cand = NULL; int blk_on_lambc; @@ -615,9 +615,9 @@ check_block(Allctr_t *allctr, Block_t * blk, int free_block) Uint blk_sz = BLK_SZ(blk); bi = BKT_IX(gfallctr, blk_sz); - ASSERT(gfallctr->bucket_mask.main & (((Uint) 1) << IX2SMIX(bi))); + ASSERT(gfallctr->bucket_mask.main & (((UWord) 1) << IX2SMIX(bi))); ASSERT(gfallctr->bucket_mask.sub[IX2SMIX(bi)] - & (((Uint) 1) << IX2SBIX(bi))); + & (((UWord) 1) << IX2SBIX(bi))); found = 0; for (fblk = gfallctr->buckets[bi]; fblk; fblk = fblk->next) @@ -648,9 +648,9 @@ check_mbc(Allctr_t *allctr, Carrier_t *mbc) int bi; for(bi = 0; bi < NO_OF_BKTS; bi++) { - if ((gfallctr->bucket_mask.main & (((Uint) 1) << IX2SMIX(bi))) + if ((gfallctr->bucket_mask.main & (((UWord) 1) << IX2SMIX(bi))) && (gfallctr->bucket_mask.sub[IX2SMIX(bi)] - & (((Uint) 1) << IX2SBIX(bi)))) { + & (((UWord) 1) << IX2SBIX(bi)))) { ASSERT(gfallctr->buckets[bi] != NULL); } else { diff --git a/erts/emulator/beam/erl_goodfit_alloc.h b/erts/emulator/beam/erl_goodfit_alloc.h index 3d1b8c01f6..a554a6f466 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.h +++ b/erts/emulator/beam/erl_goodfit_alloc.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2003-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2003-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -28,7 +28,7 @@ typedef struct GFAllctr_t_ GFAllctr_t; typedef struct { - Uint mbsd; + UWord mbsd; } GFAllctrInit_t; #define ERTS_DEFAULT_GF_ALLCTR_INIT { \ @@ -49,18 +49,18 @@ Allctr_t *erts_gfalc_start(GFAllctr_t *, GFAllctrInit_t *, AllctrInit_t *); #include "erl_alloc_util.h" #define NO_OF_BKT_IX_BITS (8) -#ifdef ARCH_64 +#if defined(ARCH_64) # define SUB_MASK_IX_SHIFT (6) #else # define SUB_MASK_IX_SHIFT (5) #endif -#define NO_OF_BKTS (((Uint) 1) << NO_OF_BKT_IX_BITS) -#define NO_OF_SUB_MASKS (NO_OF_BKTS/(((Uint) 1) << SUB_MASK_IX_SHIFT)) +#define NO_OF_BKTS (((UWord) 1) << NO_OF_BKT_IX_BITS) +#define NO_OF_SUB_MASKS (NO_OF_BKTS/(((UWord) 1) << SUB_MASK_IX_SHIFT)) typedef struct { - Uint main; - Uint sub[NO_OF_SUB_MASKS]; -} BucketMask_t; + UWord main; + UWord sub[NO_OF_SUB_MASKS]; +} BucketMask_t; typedef struct GFFreeBlock_t_ GFFreeBlock_t; struct GFFreeBlock_t_ { @@ -74,11 +74,11 @@ struct GFAllctr_t_ { char * last_aux_mbc_start; char * last_aux_mbc_end; - Uint bkt_max_size_d; - Uint bkt_intrvl_d; + UWord bkt_max_size_d; + UWord bkt_intrvl_d; BucketMask_t bucket_mask; GFFreeBlock_t * buckets[NO_OF_BKTS]; - Uint max_blk_search; + UWord max_blk_search; }; diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index e97ab328cd..14bd10b42c 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -78,6 +78,7 @@ static erts_tid_t main_thread; erts_cpu_info_t *erts_cpuinfo; +int erts_reader_groups; int erts_use_sender_punish; /* @@ -110,6 +111,7 @@ int erts_compat_rel; static int use_multi_run_queue; static int no_schedulers; static int no_schedulers_online; +static int max_reader_groups; #ifdef DEBUG Uint32 verbose; /* See erl_debug.h for information about verbose */ @@ -281,6 +283,7 @@ erl_init(void) init_load(); erts_init_bif(); erts_init_bif_chksum(); + erts_init_bif_binary(); erts_init_bif_re(); erts_init_unicode(); /* after RE to get access to PCRE unicode */ erts_delay_trap = erts_export_put(am_erlang, am_delay_trap, 2); @@ -292,9 +295,6 @@ erl_init(void) #ifdef HIPE hipe_mode_switch_init(); /* Must be after init_load/beam_catches/init */ #endif -#ifdef _OSE_ - erl_sys_init_final(); -#endif packet_parser_init(); erl_nif_init(); } @@ -338,59 +338,6 @@ init_shared_memory(int argc, char **argv) #endif } - -/* - * Create the very first process. - */ - -void -erts_first_process(Eterm modname, void* code, unsigned size, int argc, char** argv) -{ - int i; - Eterm args; - Eterm pid; - Eterm* hp; - Process parent; - Process* p; - ErlSpawnOpts so; - - if (erts_find_function(modname, am_start, 1) == NULL) { - char sbuf[256]; - Atom* ap; - - ap = atom_tab(atom_val(modname)); - memcpy(sbuf, ap->name, ap->len); - sbuf[ap->len] = '\0'; - erl_exit(5, "No function %s:start/1\n", sbuf); - } - - /* - * We need a dummy parent process to be able to call erl_create_process(). - */ - erts_init_empty_process(&parent); - hp = HAlloc(&parent, argc*2 + 4); - args = NIL; - for (i = argc-1; i >= 0; i--) { - int len = sys_strlen(argv[i]); - args = CONS(hp, new_binary(&parent, (byte*)argv[i], len), args); - hp += 2; - } - args = CONS(hp, new_binary(&parent, code, size), args); - hp += 2; - args = CONS(hp, args, NIL); - - so.flags = 0; - pid = erl_create_process(&parent, modname, am_start, args, &so); - p = process_tab[internal_pid_index(pid)]; - p->group_leader = pid; - - erts_cleanup_empty_process(&parent); -} - -/* - * XXX Old way of starting. Hopefully soon obsolete. - */ - static void erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** argv) { @@ -560,6 +507,7 @@ void erts_usage(void) ERTS_MIN_COMPAT_REL, this_rel_num()); erts_fprintf(stderr, "-r force ets memory block to be moved on realloc\n"); + erts_fprintf(stderr, "-rg amount set reader groups limit\n"); erts_fprintf(stderr, "-sbt type set scheduler bind type, valid types are:\n"); erts_fprintf(stderr, " u|ns|ts|ps|s|nnts|nnps|tnnps|db\n"); erts_fprintf(stderr, "-sct cput set cpu topology,\n"); @@ -593,6 +541,50 @@ void erts_usage(void) erl_exit(-1, ""); } +#ifdef USE_THREADS +/* + * allocators for thread lib + */ + +static void *ethr_std_alloc(size_t size) +{ + return erts_alloc_fnf(ERTS_ALC_T_ETHR_STD, (Uint) size); +} +static void *ethr_std_realloc(void *ptr, size_t size) +{ + return erts_realloc_fnf(ERTS_ALC_T_ETHR_STD, ptr, (Uint) size); +} +static void ethr_std_free(void *ptr) +{ + erts_free(ERTS_ALC_T_ETHR_STD, ptr); +} +static void *ethr_sl_alloc(size_t size) +{ + return erts_alloc_fnf(ERTS_ALC_T_ETHR_SL, (Uint) size); +} +static void *ethr_sl_realloc(void *ptr, size_t size) +{ + return erts_realloc_fnf(ERTS_ALC_T_ETHR_SL, ptr, (Uint) size); +} +static void ethr_sl_free(void *ptr) +{ + erts_free(ERTS_ALC_T_ETHR_SL, ptr); +} +static void *ethr_ll_alloc(size_t size) +{ + return erts_alloc_fnf(ERTS_ALC_T_ETHR_LL, (Uint) size); +} +static void *ethr_ll_realloc(void *ptr, size_t size) +{ + return erts_realloc_fnf(ERTS_ALC_T_ETHR_LL, ptr, (Uint) size); +} +static void ethr_ll_free(void *ptr) +{ + erts_free(ERTS_ALC_T_ETHR_LL, ptr); +} + +#endif + static void early_init(int *argc, char **argv) /* * Only put things here which are @@ -670,9 +662,15 @@ early_init(int *argc, char **argv) /* ? ncpuavail : (ncpuonln > 0 ? ncpuonln : no_schedulers)); +#ifdef ERTS_SMP + erts_max_main_threads = no_schedulers_online; +#endif + schdlrs = no_schedulers; schdlrs_onln = no_schedulers_online; + max_reader_groups = ERTS_MAX_READER_GROUPS; + if (argc && argv) { int i = 1; while (i < *argc) { @@ -682,6 +680,24 @@ early_init(int *argc, char **argv) /* } if (argv[i][0] == '-') { switch (argv[i][1]) { + case 'r': { + char *sub_param = argv[i]+2; + if (has_prefix("g", sub_param)) { + char *arg = get_arg(sub_param+1, argv[i+1], &i); + if (sscanf(arg, "%d", &max_reader_groups) != 1) { + erts_fprintf(stderr, + "bad reader groups limit: %s\n", arg); + erts_usage(); + } + if (max_reader_groups < 0) { + erts_fprintf(stderr, + "bad reader groups limit: %d\n", + max_reader_groups); + erts_usage(); + } + } + break; + } case 'S' : { int tot, onln; char *arg = get_arg(argv[i]+2, argv[i+1], &i); @@ -754,9 +770,43 @@ early_init(int *argc, char **argv) /* erts_early_init_scheduling(); /* Require allocators */ erts_init_utils(); /* Require allocators */ +#ifdef USE_THREADS + { + erts_thr_late_init_data_t elid = ERTS_THR_LATE_INIT_DATA_DEF_INITER; + elid.mem.std.alloc = ethr_std_alloc; + elid.mem.std.realloc = ethr_std_realloc; + elid.mem.std.free = ethr_std_free; + elid.mem.sl.alloc = ethr_sl_alloc; + elid.mem.sl.realloc = ethr_sl_realloc; + elid.mem.sl.free = ethr_sl_free; + elid.mem.ll.alloc = ethr_ll_alloc; + elid.mem.ll.realloc = ethr_ll_realloc; + elid.mem.ll.free = ethr_ll_free; + +#ifdef ERTS_SMP + elid.main_threads = erts_max_main_threads; +#else + elid.main_threads = 1; +#endif + elid.reader_groups = (elid.main_threads > 1 + ? elid.main_threads + : 0); + if (max_reader_groups <= 1) + elid.reader_groups = 0; + if (elid.reader_groups > max_reader_groups) + elid.reader_groups = max_reader_groups; + erts_reader_groups = elid.reader_groups; + + erts_thr_late_init(&elid); + } +#endif #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_late_init(); #endif + +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_late_init(); +#endif #if defined(HIPE) hipe_signal_init(); /* must be done very early */ @@ -819,7 +869,13 @@ erl_start(int argc, char **argv) if (erts_sys_getenv("ERL_THREAD_POOL_SIZE", envbuf, &envbufsz) == 0) { async_max_threads = atoi(envbuf); } - + +#if (defined(__APPLE__) && defined(__MACH__)) || defined(__DARWIN__) + /* + * The default stack size on MacOS X is too small for pcre. + */ + erts_sched_thread_suggested_stack_size = 256; +#endif #ifdef DEBUG verbose = DEBUG_DEFAULT; @@ -1238,9 +1294,17 @@ erl_start(int argc, char **argv) erts_async_thread_suggested_stack_size)); break; - case 'r': - erts_ets_realloc_always_moves = 1; + case 'r': { + char *sub_param = argv[i]+2; + if (has_prefix("g", sub_param)) { + get_arg(sub_param+1, argv[i+1], &i); + /* already handled */ + } + else { + erts_ets_realloc_always_moves = 1; + } break; + } case 'n': /* XXX obsolete */ break; case 'c': @@ -1325,6 +1389,7 @@ erl_start(int argc, char **argv) erts_sys_main_thread(); /* May or may not return! */ #else + erts_thr_set_main_status(1, 1); set_main_stack_size(); process_main(); #endif @@ -1398,7 +1463,7 @@ system_cleanup(int exit_code) erts_cleanup_incgc(); #endif -#if defined(USE_THREADS) && !defined(ERTS_SMP) +#if defined(USE_THREADS) exit_async(); #endif #if HAVE_ERTS_MSEG @@ -1447,13 +1512,7 @@ __decl_noreturn void erl_exit0(char *file, int line, int n, char *fmt,...) if (fmt != NULL && *fmt != '\0') erl_error(fmt, args); /* Print error message. */ va_end(args); -#ifdef __WIN32__ - if(n > 0) ConWaitForExit(); - else ConNormalExit(); -#endif -#if !defined(__WIN32__) && !defined(VXWORKS) && !defined(_OSE_) - sys_tty_reset(); -#endif + sys_tty_reset(n); if (n == ERTS_INTR_EXIT) exit(0); @@ -1493,13 +1552,7 @@ __decl_noreturn void erl_exit(int n, char *fmt,...) if (fmt != NULL && *fmt != '\0') erl_error(fmt, args); /* Print error message. */ va_end(args); -#ifdef __WIN32__ - if(n > 0) ConWaitForExit(); - else ConNormalExit(); -#endif -#if !defined(__WIN32__) && !defined(VXWORKS) && !defined(_OSE_) - sys_tty_reset(); -#endif + sys_tty_reset(n); if (n == ERTS_INTR_EXIT) exit(0); diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c index 3f022f92b8..f3f3c22933 100644 --- a/erts/emulator/beam/erl_instrument.c +++ b/erts/emulator/beam/erl_instrument.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2003-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2003-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -540,18 +540,18 @@ static void dump_memory_map_to_stream(FILE *fp) if (is_internal_pid(bp->pid)) fprintf(fp, "{%lu, %lu, %lu, {%lu,%lu,%lu}}.\n", - (Uint) bp->type_no, - (Uint) bp->mem, - (Uint) bp->size, - (Uint) pid_channel_no(bp->pid), - (Uint) pid_number(bp->pid), - (Uint) pid_serial(bp->pid)); + (UWord) bp->type_no, + (UWord) bp->mem, + (UWord) bp->size, + (UWord) pid_channel_no(bp->pid), + (UWord) pid_number(bp->pid), + (UWord) pid_serial(bp->pid)); else fprintf(fp, "{%lu, %lu, %lu, undefined}.\n", - (Uint) bp->type_no, - (Uint) bp->mem, - (Uint) bp->size); + (UWord) bp->type_no, + (UWord) bp->mem, + (UWord) bp->size); } if (lock) @@ -638,7 +638,7 @@ Eterm erts_instr_get_memory_map(Process *proc) hsz += 4; } - if ((Uint) bp->mem > MAX_SMALL) + if ((UWord) bp->mem > MAX_SMALL) hsz += BIG_UINT_HEAP_SIZE; if (bp->size > MAX_SMALL) hsz += BIG_UINT_HEAP_SIZE; @@ -749,12 +749,12 @@ Eterm erts_instr_get_memory_map(Process *proc) #endif type = make_small(bp->type_no); - if ((Uint) bp->mem > MAX_SMALL) { - ptr = uint_to_big((Uint) bp->mem, hp); + if ((UWord) bp->mem > MAX_SMALL) { + ptr = uint_to_big((UWord) bp->mem, hp); hp += BIG_UINT_HEAP_SIZE; } else - ptr = make_small((Uint) bp->mem); + ptr = make_small((UWord) bp->mem); if (bp->size > MAX_SMALL) { size = uint_to_big(bp->size, hp); @@ -962,12 +962,12 @@ dump_stat_to_stream(FILE *fp, int begin_max_period) fprintf(fp, "{total,[{total,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}]}.\n", - stats->tot.size, - stats->tot.max_size, - stats->tot.max_size_ever, - stats->tot.blocks, - stats->tot.max_blocks, - stats->tot.max_blocks_ever); + (UWord) stats->tot.size, + (UWord) stats->tot.max_size, + (UWord) stats->tot.max_size_ever, + (UWord) stats->tot.blocks, + (UWord) stats->tot.max_blocks, + (UWord) stats->tot.max_blocks_ever); a_max = 0; a_min = ~0; @@ -992,12 +992,12 @@ dump_stat_to_stream(FILE *fp, int begin_max_period) "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s", i == a_min ? "{allocators,\n [" : " ", ERTS_ALC_A2AD(i), - stats->a[i].size, - stats->a[i].max_size, - stats->a[i].max_size_ever, - stats->a[i].blocks, - stats->a[i].max_blocks, - stats->a[i].max_blocks_ever, + (UWord) stats->a[i].size, + (UWord) stats->a[i].max_size, + (UWord) stats->a[i].max_size_ever, + (UWord) stats->a[i].blocks, + (UWord) stats->a[i].max_blocks, + (UWord) stats->a[i].max_blocks_ever, i == a_max ? "]}.\n" : ",\n"); } } @@ -1009,12 +1009,12 @@ dump_stat_to_stream(FILE *fp, int begin_max_period) "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s", i == ERTS_ALC_C_MIN ? "{classes,\n [" : " ", ERTS_ALC_C2CD(i), - stats->c[i].size, - stats->c[i].max_size, - stats->c[i].max_size_ever, - stats->c[i].blocks, - stats->c[i].max_blocks, - stats->c[i].max_blocks_ever, + (UWord) stats->c[i].size, + (UWord) stats->c[i].max_size, + (UWord) stats->c[i].max_size_ever, + (UWord) stats->c[i].blocks, + (UWord) stats->c[i].max_blocks, + (UWord) stats->c[i].max_blocks_ever, i == ERTS_ALC_C_MAX ? "]}.\n" : ",\n" ); } @@ -1025,12 +1025,12 @@ dump_stat_to_stream(FILE *fp, int begin_max_period) "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s", i == ERTS_ALC_N_MIN ? "{types,\n [" : " ", ERTS_ALC_N2TD(i), - stats->n[i].size, - stats->n[i].max_size, - stats->n[i].max_size_ever, - stats->n[i].blocks, - stats->n[i].max_blocks, - stats->n[i].max_blocks_ever, + (UWord) stats->n[i].size, + (UWord) stats->n[i].max_size, + (UWord) stats->n[i].max_size_ever, + (UWord) stats->n[i].blocks, + (UWord) stats->n[i].max_blocks, + (UWord) stats->n[i].max_blocks_ever, i == ERTS_ALC_N_MAX ? "]}.\n" : ",\n" ); } diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 074b08ea57..d6138fa4e4 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -96,16 +96,15 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "proc_status", "pid" }, { "proc_tab", NULL }, { "ports_snapshot", NULL }, - { "db_tab", "address" }, - { "db_tab_fix", "address" }, { "meta_name_tab", "address" }, { "meta_main_tab_slot", "address" }, + { "db_tab", "address" }, + { "db_tab_fix", "address" }, { "meta_main_tab_main", NULL }, { "db_hash_slot", "address" }, { "node_table", NULL }, { "dist_table", NULL }, { "sys_tracers", NULL }, - { "trace_pattern", NULL }, { "module_tab", NULL }, { "export_tab", NULL }, { "fun_tab", NULL }, @@ -120,9 +119,9 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "child_status", NULL }, #endif #ifdef __WIN32__ - { "sys_driver_data_lock", NULL }, + { "sys_driver_data_lock", NULL }, #endif - { "drv_ev_state_grow", NULL, }, + { "drv_ev_state_grow", NULL, }, { "drv_ev_state", "address" }, { "safe_hash", "address" }, { "pollset_rm_list", NULL }, @@ -154,7 +153,11 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "instr", NULL }, { "fix_alloc", "index" }, { "alcu_allocator", "index" }, + { "alcu_delayed_free", "index" }, { "mseg", NULL }, +#ifdef HALFWORD_HEAP + { "pmmap", NULL }, +#endif #ifdef ERTS_SMP { "port_task_pre_alloc_lock", "address" }, { "port_taskq_pre_alloc_lock", "address" }, @@ -175,17 +178,19 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "pix_lock", "address" }, { "run_queues_lists", NULL }, { "sched_stat", NULL }, + { "run_queue_sleep_list", "address" }, #endif { "alloc_thr_ix_lock", NULL }, #ifdef ERTS_SMP - { "proc_lck_wtr_alloc", NULL }, + { "proc_lck_qs_alloc", NULL }, #endif #ifdef __WIN32__ #ifdef DEBUG { "save_ops_lock", NULL }, #endif #endif - { "mtrace_buf", NULL } + { "mtrace_buf", NULL }, + { "erts_alloc_hard_debug", NULL } }; #define ERTS_LOCK_ORDER_SIZE \ @@ -197,6 +202,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { & ERTS_LC_FLG_LT_ALL \ & ~(ERTS_LC_FLG_LT_SPINLOCK|ERTS_LC_FLG_LT_RWSPINLOCK))) +static __decl_noreturn void __noreturn lc_abort(void); + static char * lock_type(Uint16 flags) { @@ -220,7 +227,7 @@ rw_op_str(Uint16 flags) return " (r)"; case ERTS_LC_FLG_LO_WRITE: erts_fprintf(stderr, "\nInternal error\n"); - abort(); + lc_abort(); default: break; } @@ -269,28 +276,18 @@ static erts_lc_free_block_t *free_blocks; #define ERTS_LC_FB_CHUNK_SIZE 10 #endif -#ifdef ETHR_HAVE_NATIVE_LOCKS static ethr_spinlock_t free_blocks_lock; -#define ERTS_LC_LOCK ethr_spin_lock -#define ERTS_LC_UNLOCK ethr_spin_unlock -#else -static ethr_mutex free_blocks_lock; -#define ERTS_LC_LOCK ethr_mutex_lock -#define ERTS_LC_UNLOCK ethr_mutex_unlock -#endif static ERTS_INLINE void lc_lock(void) { - if (ERTS_LC_LOCK(&free_blocks_lock) != 0) - abort(); + ethr_spin_lock(&free_blocks_lock); } static ERTS_INLINE void lc_unlock(void) { - if (ERTS_LC_UNLOCK(&free_blocks_lock) != 0) - abort(); + ethr_spin_unlock(&free_blocks_lock); } static ERTS_INLINE void lc_free(void *p) @@ -311,7 +308,7 @@ static void *lc_core_alloc(void) { lc_unlock(); erts_fprintf(stderr, "Lock checker out of memory!\n"); - abort(); + lc_abort(); } #else @@ -325,7 +322,7 @@ static void *lc_core_alloc(void) * ERTS_LC_FB_CHUNK_SIZE); if (!fbs) { erts_fprintf(stderr, "Lock checker failed to allocate memory!\n"); - abort(); + lc_abort(); } for (i = 1; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) { #ifdef DEBUG @@ -365,11 +362,11 @@ create_locked_locks(char *thread_name) { erts_lc_locked_locks_t *l_lcks = malloc(sizeof(erts_lc_locked_locks_t)); if (!l_lcks) - abort(); + lc_abort(); l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown"); if (!l_lcks->thread_name) - abort(); + lc_abort(); l_lcks->tid = erts_thr_self(); l_lcks->required.first = NULL; @@ -511,7 +508,7 @@ uninitialized_lock(void) { erts_fprintf(stderr, "Performing operations on uninitialized lock!\n"); print_curr_locks(get_my_locked_locks()); - abort(); + lc_abort(); } static void @@ -521,7 +518,7 @@ lock_twice(char *prefix, erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck, erts_fprintf(stderr, "%s%s", prefix, rw_op_str(op_flags)); print_lock(" ", lck, " lock which is already locked by thread!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -531,7 +528,7 @@ unlock_op_mismatch(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck, erts_fprintf(stderr, "Unlocking%s ", rw_op_str(op_flags)); print_lock("", lck, " lock which mismatch previous lock operation!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -539,7 +536,7 @@ unlock_of_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) { print_lock("Unlocking ", lck, " lock which is not locked by thread!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -548,7 +545,7 @@ lock_order_violation(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) print_lock("Lock order violation occured when locking ", lck, "!\n"); print_curr_locks(l_lcks); print_lock_order(); - abort(); + lc_abort(); } static void @@ -559,7 +556,7 @@ type_order_violation(char *op, erts_lc_locked_locks_t *l_lcks, print_lock(op, lck, "!\n"); ASSERT(l_lcks); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -611,7 +608,7 @@ lock_mismatch(erts_lc_locked_locks_t *l_lcks, int exact, } } print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -619,7 +616,7 @@ unlock_of_required_lock(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) { print_lock("Unlocking required ", lck, " lock!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -627,7 +624,7 @@ unrequire_of_not_required_lock(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *l { print_lock("Unrequire on ", lck, " lock not required!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -635,7 +632,7 @@ require_twice(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) { print_lock("Require on ", lck, " lock already required!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -643,7 +640,7 @@ required_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) { print_lock("Required ", lck, " lock not locked!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } @@ -656,13 +653,23 @@ thread_exit_handler(void) erts_fprintf(stderr, "Thread exiting while having locked locks!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } destroy_locked_locks(l_lcks); /* erts_tsd_set(locks_key, NULL); */ } } +static __decl_noreturn void +lc_abort(void) +{ +#ifdef __WIN32__ + DebugBreak(); +#else + abort(); +#endif +} + void erts_lc_set_thread_name(char *thread_name) { @@ -674,7 +681,7 @@ erts_lc_set_thread_name(char *thread_name) free((void *) l_lcks->thread_name); l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown"); if (!l_lcks->thread_name) - abort(); + lc_abort(); } } @@ -684,7 +691,7 @@ erts_lc_assert_failed(char *file, int line, char *assertion) erts_fprintf(stderr, "%s:%d: Lock check assertion \"%s\" failed!\n", file, line, assertion); print_curr_locks(get_my_locked_locks()); - abort(); + lc_abort(); return 0; } @@ -697,7 +704,7 @@ void erts_lc_fail(char *fmt, ...) va_end(args); erts_fprintf(stderr, "\n"); print_curr_locks(get_my_locked_locks()); - abort(); + lc_abort(); } @@ -717,7 +724,7 @@ erts_lc_get_lock_order_id(char *name) "(update erl_lock_check.c)\n", name); } - abort(); + lc_abort(); return (Sint16) -1; } @@ -893,6 +900,25 @@ erts_lc_check_exact(erts_lc_lock_t *have, int have_len) } } +void +erts_lc_check_no_locked_of_type(Uint16 flags) +{ + erts_lc_locked_locks_t *l_lcks = get_my_locked_locks(); + if (l_lcks) { + erts_lc_locked_lock_t *l_lck = l_lcks->locked.first; + for (l_lck = l_lcks->locked.first; l_lck; l_lck = l_lck->next) { + if (l_lck->flags & flags) { + erts_fprintf(stderr, + "Locked lock of type %s found which isn't " + "allowed here!\n", + lock_type(l_lck->flags)); + print_curr_locks(l_lcks); + lc_abort(); + } + } + } +} + int erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags) { @@ -1231,6 +1257,7 @@ void erts_lc_init_lock(erts_lc_lock_t *lck, char *name, Uint16 flags) { lck->id = erts_lc_get_lock_order_id(name); + lck->extra = make_boxed(&lck->extra); lck->flags = flags; lck->inited = ERTS_LC_INITITALIZED; @@ -1279,13 +1306,8 @@ erts_lc_init(void) free_blocks = NULL; #endif /* #ifdef ERTS_LC_STATIC_ALLOC */ -#ifdef ETHR_HAVE_NATIVE_LOCKS if (ethr_spinlock_init(&free_blocks_lock) != 0) - abort(); -#else - if (ethr_mutex_init(&free_blocks_lock) != 0) - abort(); -#endif + lc_abort(); erts_tsd_key_create(&locks_key); } diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h index d5e2ede9ac..0372e6850d 100644 --- a/erts/emulator/beam/erl_lock_check.h +++ b/erts/emulator/beam/erl_lock_check.h @@ -77,6 +77,7 @@ void erts_lc_check(erts_lc_lock_t *have, int have_len, void erts_lc_check_exact(erts_lc_lock_t *have, int have_len); void erts_lc_have_locks(int *resv, erts_lc_lock_t *lcks, int len); void erts_lc_have_lock_ids(int *resv, int *ids, int len); +void erts_lc_check_no_locked_of_type(Uint16 flags); int erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags); void erts_lc_trylock_flg(int locked, erts_lc_lock_t *lck, Uint16 op_flags); void erts_lc_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags); diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index 0d7e1335c1..239773f366 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -166,8 +166,8 @@ static void print_lock_x(erts_lcnt_lock_t *lock, Uint16 flag, char *action, char int i; type = lcnt_lock_type(lock->flag); - ethr_atomic_read(&lock->r_state, &r_state); - ethr_atomic_read(&lock->w_state, &w_state); + r_state = ethr_atomic_read(&lock->r_state); + w_state = ethr_atomic_read(&lock->w_state); if (lock->flag & flag) { @@ -257,6 +257,10 @@ void erts_lcnt_init() { erts_lcnt_clear_counters(); } +void erts_lcnt_late_init() { + erts_thr_install_exit_handler(erts_lcnt_thread_exit_handler); +} + /* list operations */ /* BEGIN ASSUMPTION: lcnt_data_lock taken */ @@ -390,10 +394,10 @@ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { ASSERT(eltd); - ethr_atomic_read(&lock->w_state, &w_state); + w_state = ethr_atomic_read(&lock->w_state); if (option & ERTS_LCNT_LO_WRITE) { - ethr_atomic_read(&lock->r_state, &r_state); + r_state = ethr_atomic_read(&lock->r_state); ethr_atomic_inc( &lock->w_state); } if (option & ERTS_LCNT_LO_READ) { @@ -419,7 +423,7 @@ void erts_lcnt_lock(erts_lcnt_lock_t *lock) { if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - ethr_atomic_read(&lock->w_state, &w_state); + w_state = ethr_atomic_read(&lock->w_state); ethr_atomic_inc( &lock->w_state); eltd = lcnt_get_thread_data(); @@ -474,7 +478,7 @@ void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line #ifdef DEBUG if (!(lock->flag & (ERTS_LCNT_LT_RWMUTEX | ERTS_LCNT_LT_RWSPINLOCK))) { - ethr_atomic_read(&lock->flowstate, &flowstate); + flowstate = ethr_atomic_read(&lock->flowstate); ASSERT(flowstate == 0); ethr_atomic_inc( &lock->flowstate); } @@ -518,12 +522,12 @@ void erts_lcnt_unlock(erts_lcnt_lock_t *lock) { if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; #ifdef DEBUG /* flowstate */ - ethr_atomic_read(&lock->flowstate, &flowstate); + flowstate = ethr_atomic_read(&lock->flowstate); ASSERT(flowstate == 1); ethr_atomic_dec( &lock->flowstate); /* write state */ - ethr_atomic_read(&lock->w_state, &w_state); + w_state = ethr_atomic_read(&lock->w_state); ASSERT(w_state > 0) #endif ethr_atomic_dec(&lock->w_state); @@ -554,7 +558,7 @@ void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) { if (res != EBUSY) { #ifdef DEBUG - ethr_atomic_read(&lock->flowstate, &flowstate); + flowstate = ethr_atomic_read(&lock->flowstate); ASSERT(flowstate == 0); ethr_atomic_inc( &lock->flowstate); #endif @@ -570,36 +574,26 @@ void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) { /* thread operations */ -static void *lcnt_thr_init(erts_lcnt_thread_data_t *eltd) { - void *(*function)(void *); - void *argument; - void *res; - function = eltd->function; - argument = eltd->argument; - - ethr_tsd_set(lcnt_thr_data_key, eltd); - - res = (void *)function(argument); - free(eltd); - return (void *)res; -} - - - -int erts_lcnt_thr_create(ethr_tid *tid, void * (*function)(void *), void *arg, ethr_thr_opts *opts) { +void erts_lcnt_thread_setup(void) { erts_lcnt_thread_data_t *eltd; - + lcnt_lock(); /* lock for thread id global update */ eltd = lcnt_thread_data_alloc(); lcnt_unlock(); - - eltd->function = function; - eltd->argument = arg; - - return ethr_thr_create(tid, (void *)lcnt_thr_init, (void *)eltd, opts); + ASSERT(eltd); + ethr_tsd_set(lcnt_thr_data_key, eltd); } +void erts_lcnt_thread_exit_handler() { + erts_lcnt_thread_data_t *eltd; + + eltd = ethr_tsd_get(lcnt_thr_data_key); + + if (eltd) { + free(eltd); + } +} /* bindings for bifs */ diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h index e3044c371f..6306580ae4 100644 --- a/erts/emulator/beam/erl_lock_count.h +++ b/erts/emulator/beam/erl_lock_count.h @@ -155,11 +155,6 @@ typedef struct { erts_lcnt_time_t timer; /* timer */ int timer_set; /* bool */ int lock_in_conflict; /* bool */ - - /* function pointer */ - void *(*function)(void *); - void *argument; - } erts_lcnt_thread_data_t; /* globals */ @@ -169,6 +164,11 @@ extern Uint16 erts_lcnt_rt_options; /* function declerations */ void erts_lcnt_init(void); +void erts_lcnt_late_init(void); + +/* thread operations */ +void erts_lcnt_thread_setup(void); +void erts_lcnt_thread_exit_handler(void); /* list operations (local) */ erts_lcnt_lock_list_t *erts_lcnt_list_init(void); @@ -194,12 +194,7 @@ void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option); void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option); void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res); -/* thread operations */ - -int erts_lcnt_thr_create(ethr_tid *tid, void * (*function)(void *), void *arg, ethr_thr_opts *opts); - /* bif interface */ - Uint16 erts_lcnt_set_rt_opt(Uint16 opt); Uint16 erts_lcnt_clear_rt_opt(Uint16 opt); void erts_lcnt_clear_counters(void); diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index a056fce0c5..82f272d28a 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -30,6 +30,7 @@ #include "erl_message.h" #include "erl_process.h" #include "erl_nmgc.h" +#include "erl_binary.h" ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message, ErlMessage, @@ -42,6 +43,15 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message, #undef HARD_DEBUG #endif + + + +static ERTS_INLINE int in_heapfrag(const Eterm* ptr, const ErlHeapFragment *bp) +{ + return ((unsigned)(ptr - bp->mem) < bp->used_size); +} + + void init_message(void) { @@ -81,9 +91,12 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, #endif ErlHeapFragment* nbp; + /* ToDo: Make use of 'used_size' to avoid realloc + when shrinking just a few words */ + #ifdef DEBUG { - Uint off_sz = size < bp->size ? size : bp->size; + Uint off_sz = size < bp->used_size ? size : bp->used_size; for (i = 0; i < brefs_size; i++) { Eterm *ptr; if (is_immed(brefs[i])) @@ -95,12 +108,12 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, } #endif - if (size == bp->size) + if (size == bp->used_size) return bp; #ifdef HARD_DEBUG dbg_brefs = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(Eterm *)*brefs_size); - dbg_bp = new_message_buffer(bp->size); + dbg_bp = new_message_buffer(bp->used_size); dbg_hp = dbg_bp->mem; dbg_tot_size = 0; for (i = 0; i < brefs_size; i++) { @@ -109,15 +122,15 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, dbg_brefs[i] = copy_struct(brefs[i], dbg_size, &dbg_hp, &dbg_bp->off_heap); } - ASSERT(dbg_tot_size == (size < bp->size ? size : bp->size)); + ASSERT(dbg_tot_size == (size < bp->used_size ? size : bp->used_size)); #endif nbp = (ErlHeapFragment*) ERTS_HEAP_REALLOC(ERTS_ALC_T_HEAP_FRAG, (void *) bp, - ERTS_HEAP_FRAG_SIZE(bp->size), + ERTS_HEAP_FRAG_SIZE(bp->alloc_size), ERTS_HEAP_FRAG_SIZE(size)); if (bp != nbp) { - Uint off_sz = size < nbp->size ? size : nbp->size; + Uint off_sz = size < nbp->used_size ? size : nbp->used_size; Eterm *sp = &bp->mem[0]; Eterm *ep = sp + off_sz; Sint offs = &nbp->mem[0] - sp; @@ -135,7 +148,7 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, } #endif } - nbp->size = size; + nbp->alloc_size = size; nbp->used_size = size; #ifdef HARD_DEBUG @@ -152,26 +165,40 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, void erts_cleanup_offheap(ErlOffHeap *offheap) { - if (offheap->mso) { - erts_cleanup_mso(offheap->mso); - } -#ifndef HYBRID /* FIND ME! */ - if (offheap->funs) { - erts_cleanup_funs(offheap->funs); - } -#endif - if (offheap->externals) { - erts_cleanup_externals(offheap->externals); + union erl_off_heap_ptr u; + + for (u.hdr = offheap->first; u.hdr; u.hdr = u.hdr->next) { + switch (thing_subtag(u.hdr->thing_word)) { + case REFC_BINARY_SUBTAG: + if (erts_refc_dectest(&u.pb->val->refc, 0) == 0) { + erts_bin_free(u.pb->val); + } + break; + case FUN_SUBTAG: + if (erts_refc_dectest(&u.fun->fe->refc, 0) == 0) { + erts_erase_fun_entry(u.fun->fe); + } + break; + default: + ASSERT(is_external_header(u.hdr->thing_word)); + erts_deref_node_entry(u.ext->node); + break; + } } } void free_message_buffer(ErlHeapFragment* bp) { - erts_cleanup_offheap(&bp->off_heap); - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, - (void *) bp, - ERTS_HEAP_FRAG_SIZE(bp->size)); + ASSERT(bp != NULL); + do { + ErlHeapFragment* next_bp = bp->next; + + erts_cleanup_offheap(&bp->off_heap); + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp, + ERTS_HEAP_FRAG_SIZE(bp->size)); + bp = next_bp; + }while (bp != NULL); } static ERTS_INLINE void @@ -181,43 +208,19 @@ link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp) /* Link the message buffer */ bp->next = MBUF(proc); MBUF(proc) = bp; - MBUF_SIZE(proc) += bp->size; + MBUF_SIZE(proc) += bp->used_size; FLAGS(proc) |= F_FORCE_GC; - /* Move any binaries into the process */ - if (bp->off_heap.mso != NULL) { - ProcBin** next_p = &bp->off_heap.mso; - while (*next_p != NULL) { - next_p = &((*next_p)->next); - } - *next_p = MSO(proc).mso; - MSO(proc).mso = bp->off_heap.mso; - bp->off_heap.mso = NULL; - MSO(proc).overhead += bp->off_heap.overhead; - } - - /* Move any funs into the process */ -#ifndef HYBRID - if (bp->off_heap.funs != NULL) { - ErlFunThing** next_p = &bp->off_heap.funs; + /* Move any off_heap's into the process */ + if (bp->off_heap.first != NULL) { + struct erl_off_heap_header** next_p = &bp->off_heap.first; while (*next_p != NULL) { next_p = &((*next_p)->next); } - *next_p = MSO(proc).funs; - MSO(proc).funs = bp->off_heap.funs; - bp->off_heap.funs = NULL; - } -#endif - - /* Move any external things into the process */ - if (bp->off_heap.externals != NULL) { - ExternalThing** next_p = &bp->off_heap.externals; - while (*next_p != NULL) { - next_p = &((*next_p)->next); - } - *next_p = MSO(proc).externals; - MSO(proc).externals = bp->off_heap.externals; - bp->off_heap.externals = NULL; + *next_p = MSO(proc).first; + MSO(proc).first = bp->off_heap.first; + bp->off_heap.first = NULL; + OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead); } } } @@ -242,7 +245,7 @@ erts_msg_distext2heap(Process *pp, goto decode_error; if (is_not_nil(*tokenp)) { ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp); - tok_sz = heap_frag->size; + tok_sz = heap_frag->used_size; sz += tok_sz; } if (pp) @@ -283,12 +286,13 @@ erts_msg_distext2heap(Process *pp, erts_cleanup_offheap(&heap_frag->off_heap); } erts_free_dist_ext_copy(dist_extp); - if (*bpp) + if (*bpp) { free_message_buffer(*bpp); + *bpp = NULL; + } else if (hp) { HRelease(pp, hp_end, hp); } - *bpp = NULL; return THE_NON_VALUE; } @@ -436,11 +440,10 @@ erts_queue_message(Process* receiver, ERL_MESSAGE_TERM(mp) = message; ERL_MESSAGE_TOKEN(mp) = seq_trace_token; mp->next = NULL; + mp->data.heap_frag = bp; #ifdef ERTS_SMP if (*receiver_locks & ERTS_PROC_LOCK_MAIN) { - mp->data.heap_frag = bp; - /* * We move 'in queue' to 'private queue' and place * message at the end of 'private queue' in order @@ -453,11 +456,9 @@ erts_queue_message(Process* receiver, LINK_MESSAGE_PRIVQ(receiver, mp); } else { - mp->data.heap_frag = bp; LINK_MESSAGE(receiver, mp); } #else - mp->data.heap_frag = bp; LINK_MESSAGE(receiver, mp); #endif @@ -491,19 +492,7 @@ erts_link_mbuf_to_proc(struct process *proc, ErlHeapFragment *bp) void erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) { - /* Unions for typecasts avoids warnings about type-punned pointers and aliasing */ - union { - Uint** upp; - ProcBin **pbpp; - ErlFunThing **efpp; - ExternalThing **etpp; - } oh_list_pp, oh_el_next_pp; - union { - Uint *up; - ProcBin *pbp; - ErlFunThing *efp; - ExternalThing *etp; - } oh_el_p; + struct erl_off_heap_header* oh; Eterm term, token, *fhp, *hp; Sint offs; Uint sz; @@ -530,40 +519,33 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) #ifdef HARD_DEBUG dbg_term_sz = size_object(term); dbg_token_sz = size_object(token); - ASSERT(bp->size == dbg_term_sz + dbg_token_sz); - - dbg_bp = new_message_buffer(bp->size); + /*ASSERT(dbg_term_sz + dbg_token_sz == erts_msg_used_frag_sz(msg)); + Copied size may be smaller due to removed SubBins's or garbage. + Copied size may be larger due to duplicated shared terms. + */ + dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz); dbg_hp = dbg_bp->mem; dbg_term = copy_struct(term, dbg_term_sz, &dbg_hp, &dbg_bp->off_heap); dbg_token = copy_struct(token, dbg_token_sz, &dbg_hp, &dbg_bp->off_heap); dbg_thp_start = *hpp; #endif - ASSERT(bp); - msg->data.attached = NULL; - - off_heap->overhead += bp->off_heap.overhead; - sz = bp->size; - -#ifdef DEBUG - if (is_not_immed(term)) { - ASSERT(bp->mem <= ptr_val(term)); - ASSERT(bp->mem + bp->size > ptr_val(term)); + if (bp->next != NULL) { + move_multi_frags(hpp, off_heap, bp, msg->m, 2); + goto copy_done; } - if (is_not_immed(token)) { - ASSERT(bp->mem <= ptr_val(token)); - ASSERT(bp->mem + bp->size > ptr_val(token)); - } -#endif + OH_OVERHEAD(off_heap, bp->off_heap.overhead); + sz = bp->used_size; + + ASSERT(is_immed(term) || in_heapfrag(ptr_val(term),bp)); + ASSERT(is_immed(token) || in_heapfrag(ptr_val(token),bp)); fhp = bp->mem; hp = *hpp; offs = hp - fhp; - oh_list_pp.upp = NULL; - oh_el_next_pp.upp = NULL; /* Shut up compiler warning */ - oh_el_p.up = NULL; /* Shut up compiler warning */ + oh = NULL; while (sz--) { Uint cpy_sz; Eterm val = *fhp++; @@ -574,8 +556,7 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) break; case TAG_PRIMARY_LIST: case TAG_PRIMARY_BOXED: - ASSERT(bp->mem <= ptr_val(val)); - ASSERT(bp->mem + bp->size > ptr_val(val)); + ASSERT(in_heapfrag(ptr_val(val), bp)); *hp++ = offset_ptr(val, offs); break; case TAG_PRIMARY_HEADER: @@ -584,31 +565,18 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) case ARITYVAL_SUBTAG: break; case REFC_BINARY_SUBTAG: - oh_list_pp.pbpp = &off_heap->mso; - oh_el_p.up = (hp-1); - oh_el_next_pp.pbpp = &(oh_el_p.pbp)->next; - cpy_sz = thing_arityval(val); - goto cpy_words; case FUN_SUBTAG: -#ifndef HYBRID - oh_list_pp.efpp = &off_heap->funs; - oh_el_p.up = (hp-1); - oh_el_next_pp.efpp = &(oh_el_p.efp)->next; -#endif - cpy_sz = thing_arityval(val); - goto cpy_words; case EXTERNAL_PID_SUBTAG: case EXTERNAL_PORT_SUBTAG: case EXTERNAL_REF_SUBTAG: - oh_list_pp.etpp = &off_heap->externals; - oh_el_p.up = (hp-1); - oh_el_next_pp.etpp = &(oh_el_p.etp)->next; + oh = (struct erl_off_heap_header*) (hp-1); cpy_sz = thing_arityval(val); goto cpy_words; default: cpy_sz = header_arity(val); cpy_words: + ASSERT(sz >= cpy_sz); sz -= cpy_sz; while (cpy_sz >= 8) { cpy_sz -= 8; @@ -631,44 +599,13 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) case 1: *hp++ = *fhp++; default: break; } - if (oh_list_pp.upp) { -#ifdef HARD_DEBUG - Uint *dbg_old_oh_list_p = *oh_list_pp.upp; -#endif + if (oh) { /* Add to offheap list */ - *oh_el_next_pp.upp = *oh_list_pp.upp; - *oh_list_pp.upp = oh_el_p.up; - ASSERT(*hpp <= oh_el_p.up); - ASSERT(hp > oh_el_p.up); -#ifdef HARD_DEBUG - switch (val & _HEADER_SUBTAG_MASK) { - case REFC_BINARY_SUBTAG: - ASSERT(off_heap->mso == *oh_list_pp.pbpp); - ASSERT(off_heap->mso->next - == (ProcBin *) dbg_old_oh_list_p); - break; -#ifndef HYBRID - case FUN_SUBTAG: - ASSERT(off_heap->funs == *oh_list_pp.efpp); - ASSERT(off_heap->funs->next - == (ErlFunThing *) dbg_old_oh_list_p); - break; -#endif - case EXTERNAL_PID_SUBTAG: - case EXTERNAL_PORT_SUBTAG: - case EXTERNAL_REF_SUBTAG: - ASSERT(off_heap->externals - == *oh_list_pp.etpp); - ASSERT(off_heap->externals->next - == (ExternalThing *) dbg_old_oh_list_p); - break; - default: - ASSERT(0); - } -#endif - oh_list_pp.upp = NULL; - - + oh->next = off_heap->first; + off_heap->first = oh; + ASSERT(*hpp <= (Eterm*)oh); + ASSERT(hp > (Eterm*)oh); + oh = NULL; } break; } @@ -676,12 +613,11 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) } } - ASSERT(bp->size == hp - *hpp); + ASSERT(bp->used_size == hp - *hpp); *hpp = hp; if (is_not_immed(token)) { - ASSERT(bp->mem <= ptr_val(token)); - ASSERT(bp->mem + bp->size > ptr_val(token)); + ASSERT(in_heapfrag(ptr_val(token), bp)); ERL_MESSAGE_TOKEN(msg) = offset_ptr(token, offs); #ifdef HARD_DEBUG ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TOKEN(msg))); @@ -690,8 +626,7 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) } if (is_not_immed(term)) { - ASSERT(bp->mem <= ptr_val(term)); - ASSERT(bp->mem + bp->size > ptr_val(term)); + ASSERT(in_heapfrag(ptr_val(term),bp)); ERL_MESSAGE_TERM(msg) = offset_ptr(term, offs); #ifdef HARD_DEBUG ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TERM(msg))); @@ -699,10 +634,12 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) #endif } +copy_done: #ifdef HARD_DEBUG { int i, j; + ErlHeapFragment* frag; { ProcBin *mso = off_heap->mso; i = j = 0; @@ -710,10 +647,12 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) mso = mso->next; i++; } - mso = bp->off_heap.mso; - while (mso) { - mso = mso->next; - j++; + for (frag=bp; frag; frag=frag->next) { + mso = frag->off_heap.mso; + while (mso) { + mso = mso->next; + j++; + } } ASSERT(i == j); } @@ -724,10 +663,12 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) fun = fun->next; i++; } - fun = bp->off_heap.funs; - while (fun) { - fun = fun->next; - j++; + for (frag=bp; frag; frag=frag->next) { + fun = frag->off_heap.funs; + while (fun) { + fun = fun->next; + j++; + } } ASSERT(i == j); } @@ -738,10 +679,12 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) external = external->next; i++; } - external = bp->off_heap.externals; - while (external) { - external = external->next; - j++; + for (frag=bp; frag; frag=frag->next) { + external = frag->off_heap.externals; + while (external) { + external = external->next; + j++; + } } ASSERT(i == j); } @@ -749,12 +692,9 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) #endif - bp->off_heap.mso = NULL; -#ifndef HYBRID - bp->off_heap.funs = NULL; -#endif - bp->off_heap.externals = NULL; + bp->off_heap.first = NULL; free_message_buffer(bp); + msg->data.heap_frag = NULL; #ifdef HARD_DEBUG ASSERT(eq(ERL_MESSAGE_TERM(msg), dbg_term)); @@ -764,6 +704,7 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) } + Uint erts_msg_attached_data_size_aux(ErlMessage *msg) { @@ -789,7 +730,7 @@ erts_msg_attached_data_size_aux(ErlMessage *msg) if (is_not_nil(msg->m[1])) { ErlHeapFragment *heap_frag; heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); - sz += heap_frag->size; + sz += heap_frag->used_size; } return sz; } @@ -805,7 +746,7 @@ erts_move_msg_attached_data_to_heap(Eterm **hpp, ErlOffHeap *ohp, ErlMessage *ms ErlHeapFragment *heap_frag; heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); ERL_MESSAGE_TOKEN(msg) = copy_struct(ERL_MESSAGE_TOKEN(msg), - heap_frag->size, + heap_frag->used_size, hpp, ohp); erts_cleanup_offheap(&heap_frag->off_heap); @@ -1062,3 +1003,4 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, erts_queue_message(to, to_locksp, bp, save, NIL); } } + diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 5cf7c209bd..5aca0db6fe 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -28,13 +28,22 @@ struct external_thing_; * but is stored outside of any heap. */ -typedef struct erl_off_heap { - struct proc_bin* mso; /* List of associated binaries. */ -#ifndef HYBRID /* FIND ME! */ - struct erl_fun_thing* funs; /* List of funs. */ +struct erl_off_heap_header { + Eterm thing_word; + Uint size; +#if HALFWORD_HEAP + void* dummy_ptr_padding__; #endif - struct external_thing_* externals; /* List of external things. */ - int overhead; /* Administrative overhead (used to force GC). */ + struct erl_off_heap_header* next; +}; + +#define OH_OVERHEAD(oh, size) do { \ + (oh)->overhead += size; \ +} while(0) + +typedef struct erl_off_heap { + struct erl_off_heap_header* first; + Uint64 overhead; /* Administrative overhead (used to force GC). */ } ErlOffHeap; #include "external.h" @@ -49,7 +58,7 @@ typedef struct erl_heap_fragment ErlHeapFragment; struct erl_heap_fragment { ErlHeapFragment* next; /* Next heap fragment */ ErlOffHeap off_heap; /* Offset heap data. */ - unsigned size; /* Size in words of mem */ + unsigned alloc_size; /* Size in (half)words of mem */ unsigned used_size; /* With terms to be moved to heap by GC */ Eterm mem[1]; /* Data */ }; @@ -75,6 +84,13 @@ typedef struct { ErlMessage** last; /* point to the last next pointer */ ErlMessage** save; int len; /* queue length */ + + /* + * The following two fields are used by the recv_mark/1 and + * recv_set/1 instructions. + */ + BeamInstr* mark; /* address to rec_loop/2 instruction */ + ErlMessage** saved_last; /* saved last pointer */ } ErlMessageQueue; #ifdef ERTS_SMP @@ -137,6 +153,7 @@ do { \ (p)->msg.len--; \ if (__mp == NULL) \ (p)->msg.last = (p)->msg.save; \ + (p)->msg.mark = 0; \ } while(0) /* Reset message save point (after receive match) */ @@ -191,11 +208,9 @@ do { \ #define ERTS_INIT_HEAP_FRAG(HEAP_FRAG_P, DATA_WORDS) \ do { \ (HEAP_FRAG_P)->next = NULL; \ - (HEAP_FRAG_P)->size = (DATA_WORDS); \ + (HEAP_FRAG_P)->alloc_size = (DATA_WORDS); \ (HEAP_FRAG_P)->used_size = (DATA_WORDS); \ - (HEAP_FRAG_P)->off_heap.mso = NULL; \ - (HEAP_FRAG_P)->off_heap.funs = NULL; \ - (HEAP_FRAG_P)->off_heap.externals = NULL; \ + (HEAP_FRAG_P)->off_heap.first = NULL; \ (HEAP_FRAG_P)->off_heap.overhead = 0; \ } while (0) @@ -219,14 +234,25 @@ void erts_move_msg_attached_data_to_heap(Eterm **, ErlOffHeap *, ErlMessage *); Eterm erts_msg_distext2heap(Process *, ErtsProcLocks *, ErlHeapFragment **, Eterm *, ErtsDistExternal *); +ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg); ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg); #if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg) +{ + const ErlHeapFragment *bp; + Uint sz = 0; + for (bp = msg->data.heap_frag; bp!=NULL; bp=bp->next) { + sz += bp->used_size; + } + return sz; +} + ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg) { ASSERT(msg->data.attached); if (is_value(ERL_MESSAGE_TERM(msg))) - return msg->data.heap_frag->size; + return erts_msg_used_frag_sz(msg); else if (msg->data.dist_ext->heap_size < 0) return erts_msg_attached_data_size_aux(msg); else { @@ -234,7 +260,7 @@ ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg) if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) { ErlHeapFragment *heap_frag; heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); - sz += heap_frag->size; + sz += heap_frag->used_size; } return sz; } diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c index 8b8ac2ec80..b1478758a1 100644 --- a/erts/emulator/beam/erl_mtrace.c +++ b/erts/emulator/beam/erl_mtrace.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2003-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2003-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -79,7 +79,7 @@ static erts_mtx_t mtrace_buf_mutex; #define UI16_SZ (2) #define UI32_SZ (4) #define UI64_SZ (8) -#ifdef ARCH_64 +#ifdef ARCH_64 /* XXX:PaN Halfword? (whole file...) */ # define UI_SZ UI64_SZ #else # define UI_SZ UI32_SZ @@ -188,7 +188,7 @@ check_alloc_entry(byte *sp, byte *ep, byte tag, Uint16 ct_no, int ct_no_n, Uint16 type, int type_n, - Uint res, int res_n, + UWord res, int res_n, Uint size, int size_n, Uint32 ti,int ti_n); void @@ -196,8 +196,8 @@ check_realloc_entry(byte *sp, byte *ep, byte tag, Uint16 ct_no, int ct_no_n, Uint16 type, int type_n, - Uint res, int res_n, - Uint ptr, int ptr_n, + UWord res, int res_n, + UWord ptr, int ptr_n, Uint size, int size_n, Uint32 ti,int ti_n); void @@ -205,7 +205,7 @@ check_free_entry(byte *sp, byte *ep, byte tag, Uint16 ct_no, int ct_no_n, Uint16 t_no, int t_no_n, - Uint ptr, int ptr_n, + UWord ptr, int ptr_n, Uint32 ti,int ti_n); void check_time_inc_entry(byte *sp, byte *ep, @@ -585,9 +585,7 @@ void erts_mtrace_init(char *receiver, char *nodename) Uint16 port; erts_mtx_init(&mtrace_buf_mutex, "mtrace_buf"); - erts_mtx_set_forksafe(&mtrace_buf_mutex); erts_mtx_init(&mtrace_op_mutex, "mtrace_op"); - erts_mtx_set_forksafe(&mtrace_op_mutex); socket_desc = erts_sock_open(); if (socket_desc == ERTS_SOCK_INVALID_SOCKET) { @@ -785,7 +783,7 @@ write_alloc_entry(byte tag, tag, ct_no, ct_no_n, t_no, t_no_n, - (Uint) res, res_n, + (UWord) res, res_n, size, size_n, ti, ti_n); #endif @@ -865,8 +863,8 @@ write_realloc_entry(byte tag, tag, ct_no, ct_no_n, t_no, t_no_n, - (Uint) res, res_n, - (Uint) ptr, ptr_n, + (UWord) res, res_n, + (UWord) ptr, ptr_n, size, size_n, ti, ti_n); #endif @@ -934,7 +932,7 @@ write_free_entry(byte tag, tag, ct_no, ct_no_n, t_no, t_no_n, - (Uint) ptr, ptr_n, + (UWord) ptr, ptr_n, ti, ti_n); #endif } @@ -1135,7 +1133,7 @@ check_alloc_entry(byte *sp, byte *ep, byte tag, Uint16 ct_no, int ct_no_n, Uint16 t_no, int t_no_n, - Uint res, int res_n, + UWord res, int res_n, Uint size, int size_n, Uint32 ti,int ti_n) { @@ -1163,8 +1161,8 @@ check_realloc_entry(byte *sp, byte *ep, byte tag, Uint16 ct_no, int ct_no_n, Uint16 t_no, int t_no_n, - Uint res, int res_n, - Uint ptr, int ptr_n, + UWord res, int res_n, + UWord ptr, int ptr_n, Uint size, int size_n, Uint32 ti,int ti_n) { @@ -1193,7 +1191,7 @@ check_free_entry(byte *sp, byte *ep, byte tag, Uint16 ct_no, int ct_no_n, Uint16 t_no, int t_no_n, - Uint ptr, int ptr_n, + UWord ptr, int ptr_n, Uint32 ti,int ti_n) { byte *p = sp; diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 585a6c1fdf..1dd9c8bd4a 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -48,9 +48,23 @@ struct erl_module_nif { struct enif_entry_t* entry; erts_refc_t rt_cnt; /* number of resource types */ erts_refc_t rt_dtor_cnt; /* number of resource types with destructors */ - int is_orphan; /* if erlang module has been purged */ + Module* mod; /* Can be NULL if orphan with dtor-resources left */ }; +#ifdef DEBUG +# define READONLY_CHECK +#endif +#ifdef READONLY_CHECK +# define ADD_READONLY_CHECK(ENV,PTR,SIZE) add_readonly_check(ENV,PTR,SIZE) +static void add_readonly_check(ErlNifEnv*, unsigned char* ptr, unsigned sz); +#else +# define ADD_READONLY_CHECK(ENV,PTR,SIZE) ((void)0) +#endif + +#ifdef DEBUG +static int is_offheap(const ErlOffHeap* off_heap); +#endif + #define MIN_HEAP_FRAG_SZ 200 static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp); @@ -74,7 +88,8 @@ static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp) HEAP_TOP(env->proc) = env->hp; } else { - HRelease(env->proc, env->hp_end, env->hp); + env->heap_frag->used_size = hp - env->heap_frag->mem; + ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); } frag_sz = need + MIN_HEAP_FRAG_SZ; hp = erts_heap_alloc(env->proc, frag_sz); @@ -106,6 +121,13 @@ static void pre_nif_noproc(ErlNifEnv* env, struct erl_module_nif* mod_nif) env->tmp_obj_list = NULL; } +/* Temporary object header, auto-deallocated when NIF returns. */ +struct enif_tmp_obj_t { + struct enif_tmp_obj_t* next; + void (*dtor)(struct enif_tmp_obj_t*); + /*char data[];*/ +}; + static ERTS_INLINE void free_tmp_objs(ErlNifEnv* env) { while (env->tmp_obj_list != NULL) { @@ -126,8 +148,9 @@ void erts_post_nif(ErlNifEnv* env) } else { ASSERT(env->hp_end != HEAP_LIMIT(env->proc)); - ASSERT(env->hp_end - env->hp <= env->heap_frag->size); - HRelease(env->proc, env->hp_end, env->hp); + ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); + env->heap_frag->used_size = env->hp - env->heap_frag->mem; + ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); } free_tmp_objs(env); } @@ -141,7 +164,7 @@ static void post_nif_noproc(ErlNifEnv* env) /* Flush out our cached heap pointers to allow an ordinary HAlloc */ -static void enable_halloc(ErlNifEnv* env) +static void flush_env(ErlNifEnv* env) { if (env->heap_frag == NULL) { ASSERT(env->hp_end == HEAP_LIMIT(env->proc)); @@ -151,14 +174,15 @@ static void enable_halloc(ErlNifEnv* env) } else { ASSERT(env->hp_end != HEAP_LIMIT(env->proc)); - ASSERT(env->hp_end - env->hp <= env->heap_frag->size); - HRelease(env->proc, env->hp_end, env->hp); + ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); + env->heap_frag->used_size = env->hp - env->heap_frag->mem; + ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); } } -/* Restore cached heap pointers +/* Restore cached heap pointers to allow alloc_heap again. */ -static void disable_halloc(ErlNifEnv* env) +static void cache_env(ErlNifEnv* env) { if (env->heap_frag == NULL) { ASSERT(env->hp_end == HEAP_LIMIT(env->proc)); @@ -168,35 +192,190 @@ static void disable_halloc(ErlNifEnv* env) } else { ASSERT(env->hp_end != HEAP_LIMIT(env->proc)); - ASSERT(env->hp_end - env->hp <= env->heap_frag->size); + ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); env->heap_frag = MBUF(env->proc); ASSERT(env->heap_frag != NULL); env->hp = env->heap_frag->mem + env->heap_frag->used_size; - env->hp_end = env->heap_frag->mem + env->heap_frag->size; + env->hp_end = env->heap_frag->mem + env->heap_frag->alloc_size; } } - - void* enif_priv_data(ErlNifEnv* env) { return env->mod_nif->priv_data; } -void* enif_alloc(ErlNifEnv* env, size_t size) +void* enif_alloc(size_t size) { return erts_alloc_fnf(ERTS_ALC_T_NIF, (Uint) size); } -void* enif_realloc(ErlNifEnv* env, void* ptr, size_t size) +void* enif_realloc(void* ptr, size_t size) { return erts_realloc_fnf(ERTS_ALC_T_NIF, ptr, size); } -void enif_free(ErlNifEnv* env, void* ptr) +void enif_free(void* ptr) { erts_free(ERTS_ALC_T_NIF, ptr); } +struct enif_msg_environment_t +{ + ErlNifEnv env; + Process phony_proc; +}; + +ErlNifEnv* enif_alloc_env(void) +{ + struct enif_msg_environment_t* msg_env = + erts_alloc_fnf(ERTS_ALC_T_NIF, sizeof(struct enif_msg_environment_t)); + Eterm* phony_heap = (Eterm*) msg_env; /* dummy non-NULL ptr */ + + msg_env->env.hp = phony_heap; + msg_env->env.hp_end = phony_heap; + msg_env->env.heap_frag = NULL; + msg_env->env.mod_nif = NULL; + msg_env->env.tmp_obj_list = (struct enif_tmp_obj_t*) 1; /* invalid non-NULL */ + msg_env->env.proc = &msg_env->phony_proc; + memset(&msg_env->phony_proc, 0, sizeof(Process)); + HEAP_START(&msg_env->phony_proc) = phony_heap; + HEAP_TOP(&msg_env->phony_proc) = phony_heap; + HEAP_LIMIT(&msg_env->phony_proc) = phony_heap; + HEAP_END(&msg_env->phony_proc) = phony_heap; + MBUF(&msg_env->phony_proc) = NULL; + msg_env->phony_proc.id = ERTS_INVALID_PID; +#ifdef FORCE_HEAP_FRAGS + msg_env->phony_proc.space_verified = 0; + msg_env->phony_proc.space_verified_from = NULL; +#endif + return &msg_env->env; +} +void enif_free_env(ErlNifEnv* env) +{ + enif_clear_env(env); + erts_free(ERTS_ALC_T_NIF, env); +} + +static ERTS_INLINE void clear_offheap(ErlOffHeap* oh) +{ + oh->first = NULL; + oh->overhead = 0; +} + +void enif_clear_env(ErlNifEnv* env) +{ + struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)env; + Process* p = &menv->phony_proc; + ASSERT(p == menv->env.proc); + ASSERT(p->id == ERTS_INVALID_PID); + ASSERT(MBUF(p) == menv->env.heap_frag); + if (MBUF(p) != NULL) { + erts_cleanup_offheap(&MSO(p)); + clear_offheap(&MSO(p)); + free_message_buffer(MBUF(p)); + MBUF(p) = NULL; + menv->env.heap_frag = NULL; + } + ASSERT(HEAP_TOP(p) == HEAP_END(p)); + menv->env.hp = menv->env.hp_end = HEAP_TOP(p); + + ASSERT(!is_offheap(&MSO(p))); +} +int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, + ErlNifEnv* msg_env, ERL_NIF_TERM msg) +{ + struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)msg_env; + ErtsProcLocks rp_locks = 0; + Process* rp; + Process* c_p; + ErlHeapFragment* frags; +#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) + ErtsProcLocks rp_had_locks; +#endif + Eterm receiver = to_pid->pid; + int flush_me = 0; + + if (env != NULL) { + c_p = env->proc; + if (receiver == c_p->id) { + rp_locks = ERTS_PROC_LOCK_MAIN; + flush_me = 1; + } + } + else { +#ifdef ERTS_SMP + c_p = NULL; +#else + erl_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM"); +#endif + } + +#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) + rp_had_locks = rp_locks; +#endif + rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, + receiver, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC); + if (rp == NULL) { + ASSERT(env == NULL || receiver != c_p->id); + return 0; + } + flush_env(msg_env); + frags = menv->env.heap_frag; + ASSERT(frags == MBUF(&menv->phony_proc)); + if (frags != NULL) { + /* Move all offheap's from phony proc to the first fragment. + Quick and dirty, but erts_move_msg_mbuf_to_heap doesn't care. */ + ASSERT(!is_offheap(&frags->off_heap)); + frags->off_heap = MSO(&menv->phony_proc); + clear_offheap(&MSO(&menv->phony_proc)); + menv->env.heap_frag = NULL; + MBUF(&menv->phony_proc) = NULL; + } + ASSERT(!is_offheap(&MSO(&menv->phony_proc))); + + if (flush_me) { + flush_env(env); /* Needed for ERTS_HOLE_CHECK */ + } + erts_queue_message(rp, &rp_locks, frags, msg, am_undefined); + if (rp_locks) { + ERTS_SMP_LC_ASSERT(rp_locks == (rp_had_locks | (ERTS_PROC_LOCK_MSGQ | + ERTS_PROC_LOCK_STATUS))); + erts_smp_proc_unlock(rp, (ERTS_PROC_LOCK_MSGQ | ERTS_PROC_LOCK_STATUS)); + } + erts_smp_proc_dec_refc(rp); + if (flush_me) { + cache_env(env); + } + return 1; +} + +ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term) +{ + Uint sz; + Eterm* hp; + sz = size_object(src_term); + hp = alloc_heap(dst_env, sz); + return copy_struct(src_term, sz, &hp, &MSO(dst_env->proc)); +} + + +#ifdef DEBUG +static int is_offheap(const ErlOffHeap* oh) +{ + return oh->first != NULL; +} +#endif + +ErlNifPid* enif_self(ErlNifEnv* caller_env, ErlNifPid* pid) +{ + pid->pid = caller_env->proc->id; + return pid; +} +int enif_get_local_pid(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifPid* pid) +{ + return is_internal_pid(term) ? (pid->pid=term, 1) : 0; +} + int enif_is_atom(ErlNifEnv* env, ERL_NIF_TERM term) { return is_atom(term); @@ -232,9 +411,19 @@ int enif_is_ref(ErlNifEnv* env, ERL_NIF_TERM term) return is_ref(term); } +int enif_is_tuple(ErlNifEnv* env, ERL_NIF_TERM term) +{ + return is_tuple(term); +} + +int enif_is_list(ErlNifEnv* env, ERL_NIF_TERM term) +{ + return is_list(term) || is_nil(term); +} + static void aligned_binary_dtor(struct enif_tmp_obj_t* obj) { - erts_free_aligned_binary_bytes((byte*)obj); + erts_free_aligned_binary_bytes_extra((byte*)obj,ERTS_ALC_T_TMP); } int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) @@ -244,7 +433,7 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) byte* raw_ptr; }u; u.tmp = NULL; - bin->data = erts_get_aligned_binary_bytes_extra(bin_term, &u.raw_ptr, + bin->data = erts_get_aligned_binary_bytes_extra(bin_term, &u.raw_ptr, ERTS_ALC_T_TMP, sizeof(struct enif_tmp_obj_t)); if (bin->data == NULL) { return 0; @@ -257,6 +446,7 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) bin->bin_term = bin_term; bin->size = binary_size(bin_term); bin->ref_bin = NULL; + ADD_READONLY_CHECK(env, bin->data, bin->size); return 1; } @@ -293,10 +483,11 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) bin->bin_term = THE_NON_VALUE; bin->ref_bin = NULL; io_list_to_buf(term, (char*) bin->data, sz); + ADD_READONLY_CHECK(env, bin->data, bin->size); return 1; } -int enif_alloc_binary(ErlNifEnv* env, unsigned size, ErlNifBinary* bin) +int enif_alloc_binary(size_t size, ErlNifBinary* bin) { Binary* refbin; @@ -315,7 +506,7 @@ int enif_alloc_binary(ErlNifEnv* env, unsigned size, ErlNifBinary* bin) return 1; } -int enif_realloc_binary(ErlNifEnv* env, ErlNifBinary* bin, unsigned size) +int enif_realloc_binary(ErlNifBinary* bin, size_t size) { if (bin->ref_bin != NULL) { Binary* oldbin; @@ -333,15 +524,15 @@ int enif_realloc_binary(ErlNifEnv* env, ErlNifBinary* bin, unsigned size) } else { unsigned char* old_data = bin->data; - unsigned cpy_sz = (size < bin->size ? size : bin->size); - enif_alloc_binary(env, size, bin); + size_t cpy_sz = (size < bin->size ? size : bin->size); + enif_alloc_binary(size, bin); sys_memcpy(bin->data, old_data, cpy_sz); } return 1; } -void enif_release_binary(ErlNifEnv* env, ErlNifBinary* bin) +void enif_release_binary(ErlNifBinary* bin) { if (bin->ref_bin != NULL) { Binary* refbin = bin->ref_bin; @@ -357,12 +548,21 @@ void enif_release_binary(ErlNifEnv* env, ErlNifBinary* bin) #endif } -int enif_is_identical(ErlNifEnv* env, Eterm lhs, Eterm rhs) +unsigned char* enif_make_new_binary(ErlNifEnv* env, size_t size, + ERL_NIF_TERM* termp) +{ + flush_env(env); + *termp = new_binary(env->proc, NULL, size); + cache_env(env); + return binary_bytes(*termp); +} + +int enif_is_identical(Eterm lhs, Eterm rhs) { return EQ(lhs,rhs); } -int enif_compare(ErlNifEnv* env, Eterm lhs, Eterm rhs) +int enif_compare(Eterm lhs, Eterm rhs) { return cmp(lhs,rhs); } @@ -425,13 +625,13 @@ Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin) pb = (ProcBin *) alloc_heap(env, PROC_BIN_SIZE); pb->thing_word = HEADER_PROC_BIN; pb->size = bptr->orig_size; - pb->next = MSO(env->proc).mso; - MSO(env->proc).mso = pb; + pb->next = MSO(env->proc).first; + MSO(env->proc).first = (struct erl_off_heap_header*) pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; - MSO(env->proc).overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(&(MSO(env->proc)), pb->size / sizeof(Eterm)); bin_term = make_binary(pb); if (erts_refc_read(&bptr->refc, 1) == 1) { /* Total ownership transfer */ @@ -441,15 +641,15 @@ Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin) return bin_term; } else { - enable_halloc(env); + flush_env(env); bin->bin_term = new_binary(env->proc, bin->data, bin->size); - disable_halloc(env); + cache_env(env); return bin->bin_term; } } Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, - unsigned pos, unsigned size) + size_t pos, size_t size) { ErlSubBin* sb; Eterm orig; @@ -479,9 +679,11 @@ Eterm enif_make_badarg(ErlNifEnv* env) BIF_ERROR(env->proc, BADARG); } -int enif_get_atom(ErlNifEnv* env, Eterm atom, char* buf, unsigned len) +int enif_get_atom(ErlNifEnv* env, Eterm atom, char* buf, unsigned len, + ErlNifCharEncoding encoding) { Atom* ap; + ASSERT(encoding == ERL_NIF_LATIN1); if (is_not_atom(atom)) { return 0; } @@ -496,9 +698,9 @@ int enif_get_atom(ErlNifEnv* env, Eterm atom, char* buf, unsigned len) int enif_get_int(ErlNifEnv* env, Eterm term, int* ip) { -#if SIZEOF_INT == SIZEOF_VOID_P +#if SIZEOF_INT == ERTS_SIZEOF_ETERM return term_to_Sint(term, (Sint*)ip); -#elif SIZEOF_LONG == SIZEOF_VOID_P +#elif SIZEOF_LONG == ERTS_SIZEOF_ETERM Sint i; if (!term_to_Sint(term, &i) || i < INT_MIN || i > INT_MAX) { return 0; @@ -512,9 +714,9 @@ int enif_get_int(ErlNifEnv* env, Eterm term, int* ip) int enif_get_uint(ErlNifEnv* env, Eterm term, unsigned* ip) { -#if SIZEOF_INT == SIZEOF_VOID_P +#if SIZEOF_INT == ERTS_SIZEOF_ETERM return term_to_Uint(term, (Uint*)ip); -#elif SIZEOF_LONG == SIZEOF_VOID_P +#elif SIZEOF_LONG == ERTS_SIZEOF_ETERM Uint i; if (!term_to_Uint(term, &i) || i > UINT_MAX) { return 0; @@ -526,8 +728,11 @@ int enif_get_uint(ErlNifEnv* env, Eterm term, unsigned* ip) int enif_get_long(ErlNifEnv* env, Eterm term, long* ip) { -#if SIZEOF_LONG == SIZEOF_VOID_P +#if SIZEOF_LONG == ERTS_SIZEOF_ETERM return term_to_Sint(term, ip); +#elif SIZEOF_INT == ERTS_SIZEOF_ETERM + Sint i; + return term_to_Sint(term, &i) ? (*ip = (long) i, 1) : 0; #else # error Unknown long word size #endif @@ -535,14 +740,29 @@ int enif_get_long(ErlNifEnv* env, Eterm term, long* ip) int enif_get_ulong(ErlNifEnv* env, Eterm term, unsigned long* ip) { -#if SIZEOF_LONG == SIZEOF_VOID_P +#if SIZEOF_LONG == ERTS_SIZEOF_ETERM return term_to_Uint(term, ip); +#elif SIZEOF_INT == ERTS_SIZEOF_ETERM + Uint u; + return term_to_Uint(term, &u) ? (*ip = (unsigned long) u, 1) : 0; #else # error Unknown long word size #endif } -int enif_get_double(ErlNifEnv* env, Eterm term, double* dp) +#if HAVE_INT64 && SIZEOF_LONG != 8 +int enif_get_int64(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifSInt64* ip) +{ + return term_to_Sint64(term, ip); +} + +int enif_get_uint64(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifUInt64* ip) +{ + return term_to_Uint64(term, ip); +} +#endif /* HAVE_INT64 && SIZEOF_LONG != 8 */ + +int enif_get_double(ErlNifEnv* env, ERL_NIF_TERM term, double* dp) { FloatDef f; if (is_not_float(term)) { @@ -553,6 +773,17 @@ int enif_get_double(ErlNifEnv* env, Eterm term, double* dp) return 1; } +int enif_get_atom_length(ErlNifEnv* env, Eterm atom, unsigned* len, + ErlNifCharEncoding enc) +{ + Atom* ap; + ASSERT(enc == ERL_NIF_LATIN1); + if (is_not_atom(atom)) return 0; + ap = atom_tab(atom_val(atom)); + *len = ap->len; + return 1; +} + int enif_get_list_cell(ErlNifEnv* env, Eterm term, Eterm* head, Eterm* tail) { Eterm* val; @@ -563,29 +794,33 @@ int enif_get_list_cell(ErlNifEnv* env, Eterm term, Eterm* head, Eterm* tail) return 1; } +int enif_get_list_length(ErlNifEnv* env, Eterm term, unsigned* len) +{ + if (is_not_list(term) && is_not_nil(term)) return 0; + *len = list_length(term); + return 1; +} + ERL_NIF_TERM enif_make_int(ErlNifEnv* env, int i) { -#if SIZEOF_INT == SIZEOF_VOID_P +#if SIZEOF_INT == ERTS_SIZEOF_ETERM return IS_SSMALL(i) ? make_small(i) : small_to_big(i,alloc_heap(env,2)); -#elif SIZEOF_LONG == SIZEOF_VOID_P +#elif SIZEOF_LONG == ERTS_SIZEOF_ETERM return make_small(i); #endif } ERL_NIF_TERM enif_make_uint(ErlNifEnv* env, unsigned i) { -#if SIZEOF_INT == SIZEOF_VOID_P +#if SIZEOF_INT == ERTS_SIZEOF_ETERM return IS_USMALL(0,i) ? make_small(i) : uint_to_big(i,alloc_heap(env,2)); -#elif SIZEOF_LONG == SIZEOF_VOID_P +#elif SIZEOF_LONG == ERTS_SIZEOF_ETERM return make_small(i); #endif } ERL_NIF_TERM enif_make_long(ErlNifEnv* env, long i) { -#if SIZEOF_LONG != SIZEOF_VOID_P -# error Unknown long word size -#endif return IS_SSMALL(i) ? make_small(i) : small_to_big(i, alloc_heap(env,2)); } @@ -594,6 +829,26 @@ ERL_NIF_TERM enif_make_ulong(ErlNifEnv* env, unsigned long i) return IS_USMALL(0,i) ? make_small(i) : uint_to_big(i,alloc_heap(env,2)); } +#if HAVE_INT64 && SIZEOF_LONG != 8 +ERL_NIF_TERM enif_make_int64(ErlNifEnv* env, ErlNifSInt64 i) +{ + Uint* hp; + Uint need = 0; + erts_bld_sint64(NULL, &need, i); + hp = alloc_heap(env, need); + return erts_bld_sint64(&hp, NULL, i); +} + +ERL_NIF_TERM enif_make_uint64(ErlNifEnv* env, ErlNifUInt64 i) +{ + Uint* hp; + Uint need = 0; + erts_bld_uint64(NULL, &need, i); + hp = alloc_heap(env, need); + return erts_bld_uint64(&hp, NULL, i); +} +#endif /* HAVE_INT64 && SIZEOF_LONG != 8 */ + ERL_NIF_TERM enif_make_double(ErlNifEnv* env, double d) { Eterm* hp = alloc_heap(env,FLOAT_SIZE_OBJECT); @@ -605,12 +860,25 @@ ERL_NIF_TERM enif_make_double(ErlNifEnv* env, double d) ERL_NIF_TERM enif_make_atom(ErlNifEnv* env, const char* name) { - return am_atom_put(name, sys_strlen(name)); + return enif_make_atom_len(env, name, sys_strlen(name)); } -int enif_make_existing_atom(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom) +ERL_NIF_TERM enif_make_atom_len(ErlNifEnv* env, const char* name, size_t len) { - return erts_atom_get(name, sys_strlen(name), atom); + return am_atom_put(name, len); +} + +int enif_make_existing_atom(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom, + ErlNifCharEncoding enc) +{ + return enif_make_existing_atom_len(env, name, sys_strlen(name), atom, enc); +} + +int enif_make_existing_atom_len(ErlNifEnv* env, const char* name, size_t len, + ERL_NIF_TERM* atom, ErlNifCharEncoding encoding) +{ + ASSERT(encoding == ERL_NIF_LATIN1); + return erts_atom_get(name, len, atom); } ERL_NIF_TERM enif_make_tuple(ErlNifEnv* env, unsigned cnt, ...) @@ -689,11 +957,16 @@ ERL_NIF_TERM enif_make_list_from_array(ErlNifEnv* env, const ERL_NIF_TERM arr[], ERL_NIF_TERM enif_make_string(ErlNifEnv* env, const char* string, ErlNifCharEncoding encoding) -{ - Sint n = sys_strlen(string); - Eterm* hp = alloc_heap(env,n*2); +{ + return enif_make_string_len(env, string, sys_strlen(string), encoding); +} + +ERL_NIF_TERM enif_make_string_len(ErlNifEnv* env, const char* string, + size_t len, ErlNifCharEncoding encoding) +{ + Eterm* hp = alloc_heap(env,len*2); ASSERT(encoding == ERL_NIF_LATIN1); - return erts_bld_string_n(&hp,NULL,string,n); + return erts_bld_string_n(&hp,NULL,string,len); } ERL_NIF_TERM enif_make_ref(ErlNifEnv* env) @@ -764,7 +1037,8 @@ struct enif_resource_type_t ErlNifResourceDtor* dtor; /* user destructor function */ erts_refc_t refc; /* num of resources of this type (HOTSPOT warning) +1 for active erl_module_nif */ - char name[1]; + Eterm module; + Eterm name; }; /* dummy node in circular list */ @@ -782,14 +1056,14 @@ typedef struct enif_resource_t #define SIZEOF_ErlNifResource(SIZE) (offsetof(ErlNifResource,data) + (SIZE)) #define DATA_TO_RESOURCE(PTR) ((ErlNifResource*)((char*)(PTR) - offsetof(ErlNifResource,data))) -static ErlNifResourceType* find_resource_type(const char* name) +static ErlNifResourceType* find_resource_type(Eterm module, Eterm name) { ErlNifResourceType* type; for (type = resource_type_list.next; type != &resource_type_list; type = type->next) { - if (sys_strcmp(type->name, name) == 0) { + if (type->module == module && type->name == name) { return type; } } @@ -822,33 +1096,42 @@ static void steal_resource_type(ErlNifResourceType* type) if (type->dtor != NULL && erts_refc_dectest(&lib->rt_dtor_cnt, 0) == 0 - && lib->is_orphan) { + && lib->mod == NULL) { /* last type with destructor gone, close orphan lib */ close_lib(lib); } if (erts_refc_dectest(&lib->rt_cnt, 0) == 0 - && lib->is_orphan) { + && lib->mod == NULL) { erts_free(ERTS_ALC_T_NIF, lib); } } ErlNifResourceType* -enif_open_resource_type(ErlNifEnv* env, const char* type_name, - ErlNifResourceDtor* dtor, - enum ErlNifResourceFlags flags, - enum ErlNifResourceFlags* tried) +enif_open_resource_type(ErlNifEnv* env, + const char* module_str, + const char* name_str, + ErlNifResourceDtor* dtor, + ErlNifResourceFlags flags, + ErlNifResourceFlags* tried) { - ErlNifResourceType* type = find_resource_type(type_name); - enum ErlNifResourceFlags op = flags; + ErlNifResourceType* type = NULL; + ErlNifResourceFlags op = flags; + Eterm module_am, name_am; + ASSERT(erts_smp_is_system_blocked(0)); + ASSERT(module_str == NULL); /* for now... */ + module_am = make_atom(env->mod_nif->mod->module); + name_am = enif_make_atom(env, name_str); + + type = find_resource_type(module_am, name_am); if (type == NULL) { if (flags & ERL_NIF_RT_CREATE) { type = erts_alloc(ERTS_ALC_T_NIF, - sizeof(struct enif_resource_type_t) - + sys_strlen(type_name)); + sizeof(struct enif_resource_type_t)); type->dtor = dtor; - sys_strcpy(type->name, type_name); + type->module = module_am; + type->name = name_am; erts_refc_init(&type->refc, 1); type->owner = env->mod_nif; type->prev = &resource_type_list; @@ -896,13 +1179,13 @@ static void nif_resource_dtor(Binary* bin) if (erts_refc_dectest(&type->refc, 0) == 0) { ASSERT(type->next == NULL); ASSERT(type->owner != NULL); - ASSERT(type->owner->is_orphan); + ASSERT(type->owner->mod == NULL); steal_resource_type(type); erts_free(ERTS_ALC_T_NIF, type); } } -void* enif_alloc_resource(ErlNifEnv* env, ErlNifResourceType* type, unsigned size) +void* enif_alloc_resource(ErlNifResourceType* type, size_t size) { Binary* bin = erts_create_magic_binary(SIZEOF_ErlNifResource(size), &nif_resource_dtor); ErlNifResource* resource = ERTS_MAGIC_BIN_DATA(bin); @@ -915,7 +1198,7 @@ void* enif_alloc_resource(ErlNifEnv* env, ErlNifResourceType* type, unsigned siz return resource->data; } -void enif_release_resource(ErlNifEnv* env, void* obj) +void enif_release_resource(void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource); @@ -929,6 +1212,18 @@ void enif_release_resource(ErlNifEnv* env, void* obj) } } +void enif_keep_resource(void* obj) +{ + ErlNifResource* resource = DATA_TO_RESOURCE(obj); + ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource); + + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); +#ifdef DEBUG + erts_refc_inc(&resource->nif_refc, 1); +#endif + erts_refc_inc(&bin->binary.refc, 2); +} + ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); @@ -937,15 +1232,30 @@ ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj) return erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary); } +ERL_NIF_TERM enif_make_resource_binary(ErlNifEnv* env, void* obj, + const void* data, size_t size) +{ + Eterm bin = enif_make_resource(env, obj); + ProcBin* pb = (ProcBin*) binary_val(bin); + pb->bytes = (byte*) data; + pb->size = size; + return bin; +} + int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* type, void** objp) { + ProcBin* pb; Binary* mbin; ErlNifResource* resource; if (!ERTS_TERM_IS_MAGIC_BINARY(term)) { return 0; } - mbin = ((ProcBin*) binary_val(term))->val; + pb = (ProcBin*) binary_val(term); + /*if (pb->size != 0) { + return 0; / * Or should we allow "resource binaries" as handles? * / + }*/ + mbin = pb->val; resource = (ErlNifResource*) ERTS_MAGIC_BIN_DATA(mbin); if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor || resource->type != type) { @@ -955,7 +1265,7 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ return 1; } -unsigned enif_sizeof_resource(ErlNifEnv* env, void* obj) +size_t enif_sizeof_resource(void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); Binary* bin = &ERTS_MAGIC_BIN_FROM_DATA(resource)->binary; @@ -967,37 +1277,22 @@ unsigned enif_sizeof_resource(ErlNifEnv* env, void* obj) ***************************************************************************/ -static Uint** get_func_pp(Eterm* mod_code, Eterm f_atom, unsigned arity) +static BeamInstr** get_func_pp(BeamInstr* mod_code, Eterm f_atom, unsigned arity) { int n = (int) mod_code[MI_NUM_FUNCTIONS]; int j; for (j = 0; j < n; ++j) { - Uint* code_ptr = (Uint*) mod_code[MI_FUNCTIONS+j]; - ASSERT(code_ptr[0] == (Uint) BeamOp(op_i_func_info_IaaI)); + BeamInstr* code_ptr = (BeamInstr*) mod_code[MI_FUNCTIONS+j]; + ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); if (f_atom == ((Eterm) code_ptr[3]) && arity == ((unsigned) code_ptr[4])) { - return (Uint**) &mod_code[MI_FUNCTIONS+j]; + return (BeamInstr**) &mod_code[MI_FUNCTIONS+j]; } } return NULL; } -/*static void refresh_cached_nif_data(Eterm* mod_code, - struct erl_module_nif* mod_nif) -{ - int i; - for (i=0; i < mod_nif->entry->num_of_funcs; i++) { - Eterm f_atom; - ErlNifFunc* func = &mod_nif->entry->funcs[i]; - Uint* code_ptr; - - erts_atom_get(func->name, sys_strlen(func->name), &f_atom); - code_ptr = *get_func_pp(mod_code, f_atom, func->arity); - code_ptr[5+2] = (Uint) mod_nif->priv_data; - } -}*/ - static Eterm mkatom(const char *str) { return am_atom_put(str, sys_strlen(str)); @@ -1089,7 +1384,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) static const char bad_lib[] = "bad_lib"; static const char reload[] = "reload"; static const char upgrade[] = "upgrade"; - char lib_name[256]; /* BUGBUG: Max-length? */ + char* lib_name = NULL; void* handle = NULL; void* init_func; ErlNifEntry* entry = NULL; @@ -1098,15 +1393,20 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) Module* mod; Eterm mod_atom; Eterm f_atom; - Eterm* caller; + BeamInstr* caller; ErtsSysDdllError errdesc = ERTS_SYS_DDLL_ERROR_INIT; Eterm ret = am_ok; int veto; struct erl_module_nif* lib = NULL; - len = intlist_to_buf(BIF_ARG_1, lib_name, sizeof(lib_name)-1); - if (len < 1) { - /*erts_fprintf(stderr, "Invalid library path name '%T'\r\n", BIF_ARG_1);*/ + len = list_length(BIF_ARG_1); + if (len < 0) { + BIF_ERROR(BIF_P, BADARG); + } + lib_name = (char *) erts_alloc(ERTS_ALC_T_TMP, len + 1); + + if (intlist_to_buf(BIF_ARG_1, lib_name, len) != len) { + erts_free(ERTS_ALC_T_TMP, lib_name); BIF_ERROR(BIF_P, BADARG); } lib_name[len] = '\0'; @@ -1165,7 +1465,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) /*erts_fprintf(stderr, "Found module %T\r\n", mod_atom);*/ for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) { - Uint** code_pp; + BeamInstr** code_pp; ErlNifFunc* f = &entry->funcs[i]; if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom) || (code_pp = get_func_pp(mod->code, f_atom, f->arity))==NULL) { @@ -1195,7 +1495,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) lib->entry = entry; erts_refc_init(&lib->rt_cnt, 0); erts_refc_init(&lib->rt_dtor_cnt, 0); - lib->is_orphan = 0; + lib->mod = mod; env.mod_nif = lib; if (mod->nif != NULL) { /* Reload */ int k; @@ -1268,22 +1568,23 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) mod->nif = lib; for (i=0; i < entry->num_of_funcs; i++) { - Uint* code_ptr; + BeamInstr* code_ptr; erts_atom_get(entry->funcs[i].name, sys_strlen(entry->funcs[i].name), &f_atom); code_ptr = *get_func_pp(mod->code, f_atom, entry->funcs[i].arity); if (code_ptr[1] == 0) { - code_ptr[5+0] = (Uint) BeamOp(op_call_nif); + code_ptr[5+0] = (BeamInstr) BeamOp(op_call_nif); } else { /* Function traced, patch the original instruction word */ - BpData* bp = (BpData*) code_ptr[1]; - bp->orig_instr = (Uint) BeamOp(op_call_nif); + BpData** bps = (BpData**) code_ptr[1]; + BpData* bp = (BpData*) bps[bp_sched2ix()]; + bp->orig_instr = (BeamInstr) BeamOp(op_call_nif); } - code_ptr[5+1] = (Uint) entry->funcs[i].fptr; - code_ptr[5+2] = (Uint) lib; + code_ptr[5+1] = (BeamInstr) entry->funcs[i].fptr; + code_ptr[5+2] = (BeamInstr) lib; } } - else { + else { error: ASSERT(ret != am_ok); if (lib != NULL) { @@ -1297,6 +1598,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) erts_smp_release_system(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_free(ERTS_ALC_T_TMP, lib_name); BIF_RET(ret); } @@ -1308,7 +1610,7 @@ erts_unload_nif(struct erl_module_nif* lib) ErlNifResourceType* next; ASSERT(erts_smp_is_system_blocked(0)); ASSERT(lib != NULL); - ASSERT(!lib->is_orphan); + ASSERT(lib->mod != NULL); for (rt = resource_type_list.next; rt != &resource_type_list; rt = next) { @@ -1338,7 +1640,7 @@ erts_unload_nif(struct erl_module_nif* lib) else { ASSERT(erts_refc_read(&lib->rt_cnt, 1) > 0); } - lib->is_orphan = 1; + lib->mod = NULL; /* orphan lib */ } void erl_nif_init() @@ -1347,6 +1649,53 @@ void erl_nif_init() resource_type_list.prev = &resource_type_list; resource_type_list.dtor = NULL; resource_type_list.owner = NULL; - resource_type_list.name[0] = '\0'; + resource_type_list.module = THE_NON_VALUE; + resource_type_list.name = THE_NON_VALUE; } +#ifdef READONLY_CHECK +/* Use checksums to assert that NIFs do not write into inspected binaries +*/ +static void readonly_check_dtor(struct enif_tmp_obj_t*); +static unsigned calc_checksum(unsigned char* ptr, unsigned size); + +struct readonly_check_t +{ + struct enif_tmp_obj_t hdr; + unsigned char* ptr; + unsigned size; + unsigned checksum; +}; +static void add_readonly_check(ErlNifEnv* env, unsigned char* ptr, unsigned sz) +{ + struct readonly_check_t* obj = erts_alloc(ERTS_ALC_T_TMP, + sizeof(struct readonly_check_t)); + obj->hdr.next = env->tmp_obj_list; + env->tmp_obj_list = &obj->hdr; + obj->hdr.dtor = &readonly_check_dtor; + obj->ptr = ptr; + obj->size = sz; + obj->checksum = calc_checksum(ptr, sz); +} +static void readonly_check_dtor(struct enif_tmp_obj_t* o) +{ + struct readonly_check_t* obj = (struct readonly_check_t*) o; + unsigned chksum = calc_checksum(obj->ptr, obj->size); + if (chksum != obj->checksum) { + fprintf(stderr, "\r\nReadonly data written by NIF, checksums differ" + " %x != %x\r\nABORTING\r\n", chksum, obj->checksum); + abort(); + } + erts_free(ERTS_ALC_T_TMP, obj); +} +static unsigned calc_checksum(unsigned char* ptr, unsigned size) +{ + unsigned i, sum = 0; + for (i=0; i<size; i++) { + sum ^= ptr[i] << ((i % 4)*8); + } + return sum; +} + +#endif /* READONLY_CHECK */ + diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 1ccf00293e..ee3a7cd5f4 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -23,18 +23,67 @@ #ifndef __ERL_NIF_H__ #define __ERL_NIF_H__ + #include "erl_drv_nif.h" /* Version history: ** 0.1: R13B03 ** 1.0: R13B04 +** 2.0: R14A */ -#define ERL_NIF_MAJOR_VERSION 1 +#define ERL_NIF_MAJOR_VERSION 2 #define ERL_NIF_MINOR_VERSION 0 #include <stdlib.h> +#ifdef SIZEOF_CHAR +# define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR +# undef SIZEOF_CHAR +#endif +#ifdef SIZEOF_SHORT +# define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT +# undef SIZEOF_SHORT +#endif +#ifdef SIZEOF_INT +# define SIZEOF_INT_SAVED__ SIZEOF_INT +# undef SIZEOF_INT +#endif +#ifdef SIZEOF_LONG +# define SIZEOF_LONG_SAVED__ SIZEOF_LONG +# undef SIZEOF_LONG +#endif +#ifdef SIZEOF_LONG_LONG +# define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG +# undef SIZEOF_LONG_LONG +#endif +#ifdef HALFWORD_HEAP_EMULATOR +# define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR +# undef HALFWORD_HEAP_EMULATOR +#endif +#include "erl_int_sizes_config.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) +typedef unsigned __int64 ErlNifUInt64; +typedef __int64 ErlNifSInt64; +#elif SIZEOF_LONG == 8 +typedef unsigned long ErlNifUInt64; +typedef long ErlNifSInt64; +#elif SIZEOF_LONG_LONG == 8 +typedef unsigned long long ErlNifUInt64; +typedef long long ErlNifSInt64; +#else +#error No 64-bit integer type +#endif + +#ifdef HALFWORD_HEAP_EMULATOR +typedef unsigned int ERL_NIF_TERM; +#else typedef unsigned long ERL_NIF_TERM; +#endif struct enif_environment_t; typedef struct enif_environment_t ErlNifEnv; @@ -63,7 +112,7 @@ typedef struct enif_entry_t typedef struct { - unsigned size; + size_t size; unsigned char* data; /* Internals (avert your eyes) */ @@ -73,17 +122,22 @@ typedef struct typedef struct enif_resource_type_t ErlNifResourceType; typedef void ErlNifResourceDtor(ErlNifEnv*, void*); -enum ErlNifResourceFlags +typedef enum { ERL_NIF_RT_CREATE = 1, ERL_NIF_RT_TAKEOVER = 2 -}; +}ErlNifResourceFlags; typedef enum { ERL_NIF_LATIN1 = 1 }ErlNifCharEncoding; +typedef struct +{ + ERL_NIF_TERM pid; /* internal, may change */ +}ErlNifPid; + typedef ErlDrvSysInfo ErlNifSysInfo; typedef struct ErlDrvTid_ *ErlNifTid; @@ -116,8 +170,6 @@ extern TWinDynNifCallbacks WinDynNifCallbacks; #endif - - #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) # define ERL_NIF_INIT_GLOB TWinDynNifCallbacks WinDynNifCallbacks; # define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* nif_init(TWinDynNifCallbacks* callbacks) @@ -133,7 +185,18 @@ extern TWinDynNifCallbacks WinDynNifCallbacks; #endif +#ifdef __cplusplus +} +# define ERL_NIF_INIT_PROLOGUE extern "C" { +# define ERL_NIF_INIT_EPILOGUE } +#else +# define ERL_NIF_INIT_PROLOGUE +# define ERL_NIF_INIT_EPILOGUE +#endif + + #define ERL_NIF_INIT(NAME, FUNCS, LOAD, RELOAD, UPGRADE, UNLOAD) \ +ERL_NIF_INIT_PROLOGUE \ ERL_NIF_INIT_GLOB \ ERL_NIF_INIT_DECL(NAME) \ { \ @@ -148,7 +211,9 @@ ERL_NIF_INIT_DECL(NAME) \ }; \ ERL_NIF_INIT_BODY; \ return &entry; \ -} +} \ +ERL_NIF_INIT_EPILOGUE + #endif /* __ERL_NIF_H__ */ diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index ec07a976b2..eca506593d 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -23,29 +23,29 @@ #ifdef ERL_NIF_API_FUNC_DECL ERL_NIF_API_FUNC_DECL(void*,enif_priv_data,(ErlNifEnv*)); -ERL_NIF_API_FUNC_DECL(void*,enif_alloc,(ErlNifEnv*, size_t size)); -ERL_NIF_API_FUNC_DECL(void,enif_free,(ErlNifEnv*, void* ptr)); +ERL_NIF_API_FUNC_DECL(void*,enif_alloc,(size_t size)); +ERL_NIF_API_FUNC_DECL(void,enif_free,(void* ptr)); ERL_NIF_API_FUNC_DECL(int,enif_is_atom,(ErlNifEnv*, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(int,enif_is_binary,(ErlNifEnv*, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(int,enif_is_ref,(ErlNifEnv*, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(int,enif_inspect_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, ErlNifBinary* bin)); -ERL_NIF_API_FUNC_DECL(int,enif_alloc_binary,(ErlNifEnv*, unsigned size, ErlNifBinary* bin)); -ERL_NIF_API_FUNC_DECL(int,enif_realloc_binary,(ErlNifEnv*, ErlNifBinary* bin, unsigned size)); -ERL_NIF_API_FUNC_DECL(void,enif_release_binary,(ErlNifEnv*, ErlNifBinary* bin)); +ERL_NIF_API_FUNC_DECL(int,enif_alloc_binary,(size_t size, ErlNifBinary* bin)); +ERL_NIF_API_FUNC_DECL(int,enif_realloc_binary,(ErlNifBinary* bin, size_t size)); +ERL_NIF_API_FUNC_DECL(void,enif_release_binary,(ErlNifBinary* bin)); ERL_NIF_API_FUNC_DECL(int,enif_get_int,(ErlNifEnv*, ERL_NIF_TERM term, int* ip)); ERL_NIF_API_FUNC_DECL(int,enif_get_ulong,(ErlNifEnv*, ERL_NIF_TERM term, unsigned long* ip)); ERL_NIF_API_FUNC_DECL(int,enif_get_double,(ErlNifEnv*, ERL_NIF_TERM term, double* dp)); ERL_NIF_API_FUNC_DECL(int,enif_get_list_cell,(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM* head, ERL_NIF_TERM* tail)); ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int* arity, const ERL_NIF_TERM** array)); -ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ErlNifEnv* env, ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); -ERL_NIF_API_FUNC_DECL(int,enif_compare,(ErlNifEnv* env, ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); +ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); +ERL_NIF_API_FUNC_DECL(int,enif_compare,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_ulong,(ErlNifEnv* env, unsigned long i)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_double,(ErlNifEnv* env, double d)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_atom,(ErlNifEnv* env, const char* name)); -ERL_NIF_API_FUNC_DECL(int,enif_make_existing_atom,(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom)); +ERL_NIF_API_FUNC_DECL(int,enif_make_existing_atom,(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom, ErlNifCharEncoding)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_tuple,(ErlNifEnv* env, unsigned cnt, ...)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list,(ErlNifEnv* env, unsigned cnt, ...)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list_cell,(ErlNifEnv* env, ERL_NIF_TERM car, ERL_NIF_TERM cdr)); @@ -82,13 +82,13 @@ ERL_NIF_API_FUNC_DECL(int,enif_equal_tids,(ErlNifTid tid1, ErlNifTid tid2)); ERL_NIF_API_FUNC_DECL(void,enif_thread_exit,(void *resp)); ERL_NIF_API_FUNC_DECL(int,enif_thread_join,(ErlNifTid, void **respp)); -ERL_NIF_API_FUNC_DECL(void*,enif_realloc,(ErlNifEnv*, void* ptr, size_t size)); +ERL_NIF_API_FUNC_DECL(void*,enif_realloc,(void* ptr, size_t size)); ERL_NIF_API_FUNC_DECL(void,enif_system_info,(ErlNifSysInfo *sip, size_t si_size)); ERL_NIF_API_FUNC_DECL(int,enif_fprintf,(void/* FILE* */ *filep, const char *format, ...)); ERL_NIF_API_FUNC_DECL(int,enif_inspect_iolist_as_binary,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifBinary* bin)); -ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_sub_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, unsigned pos, unsigned size)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_sub_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, size_t pos, size_t size)); ERL_NIF_API_FUNC_DECL(int,enif_get_string,(ErlNifEnv*, ERL_NIF_TERM list, char* buf, unsigned len, ErlNifCharEncoding)); -ERL_NIF_API_FUNC_DECL(int,enif_get_atom,(ErlNifEnv*, ERL_NIF_TERM atom, char* buf, unsigned len)); +ERL_NIF_API_FUNC_DECL(int,enif_get_atom,(ErlNifEnv*, ERL_NIF_TERM atom, char* buf, unsigned len, ErlNifCharEncoding)); ERL_NIF_API_FUNC_DECL(int,enif_is_fun,(ErlNifEnv*, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(int,enif_is_pid,(ErlNifEnv*, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(int,enif_is_port,(ErlNifEnv*, ERL_NIF_TERM term)); @@ -99,12 +99,36 @@ ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_long,(ErlNifEnv*, long i)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_tuple_from_array,(ErlNifEnv*, const ERL_NIF_TERM arr[], unsigned cnt)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list_from_array,(ErlNifEnv*, const ERL_NIF_TERM arr[], unsigned cnt)); ERL_NIF_API_FUNC_DECL(int,enif_is_empty_list,(ErlNifEnv*, ERL_NIF_TERM term)); -ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type,(ErlNifEnv*, const char* type_name, void (*dtor)(ErlNifEnv*,void *), enum ErlNifResourceFlags flags, enum ErlNifResourceFlags* tried)); -ERL_NIF_API_FUNC_DECL(void*,enif_alloc_resource,(ErlNifEnv*, ErlNifResourceType* type, unsigned size)); -ERL_NIF_API_FUNC_DECL(void,enif_release_resource,(ErlNifEnv*, void* obj)); +ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type,(ErlNifEnv*, const char* module_str, const char* name_str, void (*dtor)(ErlNifEnv*,void *), ErlNifResourceFlags flags, ErlNifResourceFlags* tried)); +ERL_NIF_API_FUNC_DECL(void*,enif_alloc_resource,(ErlNifResourceType* type, size_t size)); +ERL_NIF_API_FUNC_DECL(void,enif_release_resource,(void* obj)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_resource,(ErlNifEnv*, void* obj)); ERL_NIF_API_FUNC_DECL(int,enif_get_resource,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifResourceType* type, void** objp)); -ERL_NIF_API_FUNC_DECL(unsigned,enif_sizeof_resource,(ErlNifEnv*, void* obj)); +ERL_NIF_API_FUNC_DECL(size_t,enif_sizeof_resource,(void* obj)); +ERL_NIF_API_FUNC_DECL(unsigned char*,enif_make_new_binary,(ErlNifEnv*,size_t size,ERL_NIF_TERM* termp)); +ERL_NIF_API_FUNC_DECL(int,enif_is_list,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_is_tuple,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_get_atom_length,(ErlNifEnv*, ERL_NIF_TERM atom, unsigned* len, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(int,enif_get_list_length,(ErlNifEnv* env, ERL_NIF_TERM term, unsigned* len)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_atom_len,(ErlNifEnv* env, const char* name, size_t len)); +ERL_NIF_API_FUNC_DECL(int, enif_make_existing_atom_len,(ErlNifEnv* env, const char* name, size_t len, ERL_NIF_TERM* atom, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_string_len,(ErlNifEnv* env, const char* string, size_t len, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(ErlNifEnv*,enif_alloc_env,(void)); +ERL_NIF_API_FUNC_DECL(void,enif_free_env,(ErlNifEnv* env)); +ERL_NIF_API_FUNC_DECL(void,enif_clear_env,(ErlNifEnv* env)); +ERL_NIF_API_FUNC_DECL(int,enif_send,(ErlNifEnv* env, const ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_copy,(ErlNifEnv* dst_env, ERL_NIF_TERM src_term)); +ERL_NIF_API_FUNC_DECL(ErlNifPid*,enif_self,(ErlNifEnv* caller_env, ErlNifPid* pid)); +ERL_NIF_API_FUNC_DECL(int,enif_get_local_pid,(ErlNifEnv* env, ERL_NIF_TERM, ErlNifPid* pid)); +ERL_NIF_API_FUNC_DECL(void,enif_keep_resource,(void* obj)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_resource_binary,(ErlNifEnv*,void* obj,const void* data, size_t size)); +#if SIZEOF_LONG != 8 +ERL_NIF_API_FUNC_DECL(int,enif_get_int64,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifSInt64* ip)); +ERL_NIF_API_FUNC_DECL(int,enif_get_uint64,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifUInt64* ip)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int64,(ErlNifEnv*, ErlNifSInt64)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_uint64,(ErlNifEnv*, ErlNifUInt64)); +#endif + /* ** Add last to keep compatibility on Windows!!! */ @@ -195,6 +219,29 @@ ERL_NIF_API_FUNC_DECL(unsigned,enif_sizeof_resource,(ErlNifEnv*, void* obj)); # define enif_make_resource ERL_NIF_API_FUNC_MACRO(enif_make_resource) # define enif_get_resource ERL_NIF_API_FUNC_MACRO(enif_get_resource) # define enif_sizeof_resource ERL_NIF_API_FUNC_MACRO(enif_sizeof_resource) +# define enif_make_new_binary ERL_NIF_API_FUNC_MACRO(enif_make_new_binary) +# define enif_is_list ERL_NIF_API_FUNC_MACRO(enif_is_list) +# define enif_is_tuple ERL_NIF_API_FUNC_MACRO(enif_is_tuple) +# define enif_get_atom_length ERL_NIF_API_FUNC_MACRO(enif_get_atom_length) +# define enif_get_list_length ERL_NIF_API_FUNC_MACRO(enif_get_list_length) +# define enif_make_atom_len ERL_NIF_API_FUNC_MACRO(enif_make_atom_len) +# define enif_make_existing_atom_len ERL_NIF_API_FUNC_MACRO(enif_make_existing_atom_len) +# define enif_make_string_len ERL_NIF_API_FUNC_MACRO(enif_make_string_len) +# define enif_alloc_env ERL_NIF_API_FUNC_MACRO(enif_alloc_env) +# define enif_free_env ERL_NIF_API_FUNC_MACRO(enif_free_env) +# define enif_clear_env ERL_NIF_API_FUNC_MACRO(enif_clear_env) +# define enif_send ERL_NIF_API_FUNC_MACRO(enif_send) +# define enif_make_copy ERL_NIF_API_FUNC_MACRO(enif_make_copy) +# define enif_self ERL_NIF_API_FUNC_MACRO(enif_self) +# define enif_get_local_pid ERL_NIF_API_FUNC_MACRO(enif_get_local_pid) +# define enif_keep_resource ERL_NIF_API_FUNC_MACRO(enif_keep_resource) +# define enif_make_resource_binary ERL_NIF_API_FUNC_MACRO(enif_make_resource_binary) +#if SIZEOF_LONG != 8 +# define enif_get_int64 ERL_NIF_API_FUNC_MACRO(enif_get_int64) +# define enif_get_uint64 ERL_NIF_API_FUNC_MACRO(enif_get_uint64) +# define enif_make_int64 ERL_NIF_API_FUNC_MACRO(enif_make_int64) +# define enif_make_uint64 ERL_NIF_API_FUNC_MACRO(enif_make_uint64) +#endif #endif @@ -217,9 +264,15 @@ ERL_NIF_API_FUNC_DECL(unsigned,enif_sizeof_resource,(ErlNifEnv*, void* obj)); # define enif_make_tuple7(ENV,E1,E2,E3,E4,E5,E6,E7) enif_make_tuple(ENV,7,E1,E2,E3,E4,E5,E6,E7) # define enif_make_tuple8(ENV,E1,E2,E3,E4,E5,E6,E7,E8) enif_make_tuple(ENV,8,E1,E2,E3,E4,E5,E6,E7,E8) # define enif_make_tuple9(ENV,E1,E2,E3,E4,E5,E6,E7,E8,E9) enif_make_tuple(ENV,9,E1,E2,E3,E4,E5,E6,E7,E8,E9) + +# define enif_make_pid(ENV, PID) ((const ERL_NIF_TERM)((PID)->pid)) + +#if SIZEOF_LONG == 8 +# define enif_get_int64 enif_get_long +# define enif_get_uint64 enif_get_ulong +# define enif_make_int64 enif_make_long +# define enif_make_uint64 enif_make_ulong #endif -#ifndef enif_get_data -# define enif_get_data enif_priv_data /* deprecated */ #endif diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h index 87dbfc2a04..ae1316eba2 100644 --- a/erts/emulator/beam/erl_node_container_utils.h +++ b/erts/emulator/beam/erl_node_container_utils.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2001-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -251,7 +251,7 @@ extern int erts_use_r9_pids_ports; * Refs * \* */ -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP #define internal_ref_no_of_numbers(x) \ (internal_ref_data((x))[0]) diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 42b28d987c..e430b4ad77 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2001-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -80,6 +80,8 @@ dist_table_alloc(void *dep_tmpl) Eterm chnl_nr; Eterm sysname; DistEntry *dep; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; if(((DistEntry *) dep_tmpl) == erts_this_dist_entry) return dep_tmpl; @@ -92,7 +94,7 @@ dist_table_alloc(void *dep_tmpl) dep->prev = NULL; erts_refc_init(&dep->refc, -1); - erts_smp_rwmtx_init_x(&dep->rwmtx, "dist_entry", chnl_nr); + erts_smp_rwmtx_init_opt_x(&dep->rwmtx, &rwmtx_opt, "dist_entry", chnl_nr); dep->sysname = sysname; dep->cid = NIL; dep->connection_id = 0; @@ -580,6 +582,18 @@ ErlNode *erts_find_or_insert_node(Eterm sysname, Uint creation) ErlNode ne; ne.sysname = sysname; ne.creation = creation; + + erts_smp_rwmtx_rlock(&erts_node_table_rwmtx); + res = hash_get(&erts_node_table, (void *) &ne); + if (res && res != erts_this_node) { + long refc = erts_refc_inctest(&res->refc, 0); + if (refc < 2) /* New or pending delete */ + erts_refc_inc(&res->refc, 1); + } + erts_smp_rwmtx_runlock(&erts_node_table_rwmtx); + if (res) + return res; + erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); res = hash_put(&erts_node_table, (void *) &ne); ASSERT(res); @@ -696,8 +710,12 @@ erts_set_this_node(Eterm sysname, Uint creation) void erts_init_node_tables(void) { + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; HashFunctions f; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + f.hash = (H_FUN) dist_table_hash; f.cmp = (HCMP_FUN) dist_table_cmp; f.alloc = (HALLOC_FUN) dist_table_alloc; @@ -719,9 +737,10 @@ void erts_init_node_tables(void) erts_this_dist_entry->prev = NULL; erts_refc_init(&erts_this_dist_entry->refc, 1); /* erts_this_node */ - erts_smp_rwmtx_init_x(&erts_this_dist_entry->rwmtx, - "dist_entry", - make_small(ERST_INTERNAL_CHANNEL_NO)); + erts_smp_rwmtx_init_opt_x(&erts_this_dist_entry->rwmtx, + &rwmtx_opt, + "dist_entry", + make_small(ERST_INTERNAL_CHANNEL_NO)); erts_this_dist_entry->sysname = am_Noname; erts_this_dist_entry->cid = NIL; erts_this_dist_entry->connection_id = 0; @@ -772,8 +791,8 @@ void erts_init_node_tables(void) (void) hash_put(&erts_node_table, (void *) erts_this_node); - erts_smp_rwmtx_init(&erts_node_table_rwmtx, "node_table"); - erts_smp_rwmtx_init(&erts_dist_table_rwmtx, "dist_table"); + erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table"); + erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table"); references_atoms_need_init = 1; } @@ -1087,49 +1106,62 @@ insert_offheap2(ErlOffHeap *oh, void *arg) static void insert_offheap(ErlOffHeap *oh, int type, Eterm id) { - if(oh->externals) { - ExternalThing *etp = oh->externals; - while (etp) { - insert_node(etp->node, type, id); - etp = etp->next; - } - } + union erl_off_heap_ptr u; + struct insert_offheap2_arg a; + a.type = BIN_REF; - if(oh->mso) { - ProcBin *pb; - struct insert_offheap2_arg a; - a.type = BIN_REF; - for(pb = oh->mso; pb; pb = pb->next) { - if(IsMatchProgBinary(pb->val)) { + for (u.hdr = oh->first; u.hdr; u.hdr = u.hdr->next) { + switch (thing_subtag(u.hdr->thing_word)) { + case REFC_BINARY_SUBTAG: + if(IsMatchProgBinary(u.pb->val)) { InsertedBin *ib; int insert_bin = 1; for (ib = inserted_bins; ib; ib = ib->next) - if(ib->bin_val == pb->val) { + if(ib->bin_val == u.pb->val) { insert_bin = 0; break; } if (insert_bin) { - Uint id_heap[BIG_UINT_HEAP_SIZE]; +#if HALFWORD_HEAP + UWord val = (UWord) u.pb->val; + DeclareTmpHeapNoproc(id_heap,BIG_UINT_HEAP_SIZE*2); /* extra place allocated */ +#else + DeclareTmpHeapNoproc(id_heap,BIG_UINT_HEAP_SIZE); +#endif Uint *hp = &id_heap[0]; InsertedBin *nib; - a.id = erts_bld_uint(&hp, NULL, (Uint) pb->val); - erts_match_prog_foreach_offheap(pb->val, +#if HALFWORD_HEAP + int actual_need = BIG_UWORD_HEAP_SIZE(val); + ASSERT(actual_need <= (BIG_UINT_HEAP_SIZE*2)); + UseTmpHeapNoproc(actual_need); + a.id = erts_bld_uword(&hp, NULL, (UWord) val); +#else + UseTmpHeapNoproc(BIG_UINT_HEAP_SIZE); + a.id = erts_bld_uint(&hp, NULL, (Uint) u.pb->val); +#endif + erts_match_prog_foreach_offheap(u.pb->val, insert_offheap2, (void *) &a); nib = erts_alloc(ERTS_ALC_T_NC_TMP, sizeof(InsertedBin)); - nib->bin_val = pb->val; + nib->bin_val = u.pb->val; nib->next = inserted_bins; inserted_bins = nib; +#if HALFWORD_HEAP + UnUseTmpHeapNoproc(actual_need); +#else + UnUseTmpHeapNoproc(BIG_UINT_HEAP_SIZE); +#endif } - } + } + break; + case FUN_SUBTAG: + break; /* No need to */ + default: + ASSERT(is_external_header(u.hdr->thing_word)); + insert_node(u.ext->node, type, id); + break; } } - -#if 0 - if(oh->funs) { - /* No need to */ - } -#endif } static void doit_insert_monitor(ErtsMonitor *monitor, void *p) @@ -1190,12 +1222,15 @@ static void insert_bif_timer(Eterm receiver, Eterm msg, ErlHeapFragment *bp, void *arg) { if (bp) { - Eterm heap[3]; + DeclareTmpHeapNoproc(heap,3); + + UseTmpHeapNoproc(3); insert_offheap(&bp->off_heap, TIMER_REF, (is_internal_pid(receiver) ? receiver : TUPLE2(&heap[0], AM_process, receiver))); + UnUseTmpHeapNoproc(3); } } @@ -1230,7 +1265,7 @@ setup_reference_table(void) DistEntry *dep; HashInfo hi; int i; - Eterm heap[3]; + DeclareTmpHeapNoproc(heap,3); inserted_bins = NULL; @@ -1251,6 +1286,7 @@ setup_reference_table(void) /* Go through the hole system, and build a table of all references to ErlNode and DistEntry structures */ + UseTmpHeapNoproc(3); insert_node(erts_this_node, SYSTEM_REF, TUPLE2(&heap[0], AM_system, am_undefined)); @@ -1261,11 +1297,13 @@ setup_reference_table(void) HEAP_REF, TUPLE2(&heap[0], AM_processes, am_undefined)); #endif + UnUseTmpHeapNoproc(3); /* Insert all processes */ for (i = 0; i < erts_max_processes; i++) if (process_tab[i]) { ErlMessage *msg; + /* Insert Heap */ insert_offheap(&(process_tab[i]->off_heap), HEAP_REF, @@ -1352,21 +1390,22 @@ setup_reference_table(void) { /* Add binaries stored elsewhere ... */ ErlOffHeap oh; - ProcBin pb[2] = {{0},{0}}; - ProcBin *mso = NULL; + ProcBin pb[2]; int i = 0; Binary *default_match_spec; Binary *default_meta_match_spec; - /* Only the ProcBin members val and next will be inspected + oh.first = NULL; + /* Only the ProcBin members thing_word, val and next will be inspected (by insert_offheap()) */ #undef ADD_BINARY -#define ADD_BINARY(Bin) \ - if ((Bin)) { \ - pb[i].val = (Bin); \ - pb[i].next = mso; \ - mso = &pb[i]; \ - i++; \ +#define ADD_BINARY(Bin) \ + if ((Bin)) { \ + pb[i].thing_word = REFC_BINARY_SUBTAG; \ + pb[i].val = (Bin); \ + pb[i].next = oh.first; \ + oh.first = (struct erl_off_heap_header*) &pb[i]; \ + i++; \ } erts_get_default_trace_pattern(NULL, @@ -1378,11 +1417,6 @@ setup_reference_table(void) ADD_BINARY(default_match_spec); ADD_BINARY(default_meta_match_spec); - oh.mso = mso; - oh.externals = NULL; -#ifndef HYBRID /* FIND ME! */ - oh.funs = NULL; -#endif insert_offheap(&oh, BIN_REF, AM_match_spec); #undef ADD_BINARY } diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index c48dac6219..eb759b87e9 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2001-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -61,7 +61,7 @@ #define ERTS_DE_QFLGS_ALL (ERTS_DE_QFLG_BUSY \ | ERTS_DE_QFLG_EXIT) -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP #define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713f713f713UL) #else #define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713) diff --git a/erts/emulator/beam/erl_obsolete.c b/erts/emulator/beam/erl_obsolete.c deleted file mode 100644 index 9c5a7c7ff9..0000000000 --- a/erts/emulator/beam/erl_obsolete.c +++ /dev/null @@ -1,186 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "sys.h" -#include "erl_driver.h" - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ - * * - * ------------------------- OBSOLETE! DO NOT USE! ------------------------- * - * * -\* */ - -/* cut from ../obsolete/driver.h (since it doesn't mix well with other - * headers from the emulator). - */ -#ifdef __WIN32__ -#ifdef CONST -# undef CONST -#endif -#endif - -#if ((defined(__STDC__) || defined(SABER)) && !defined(NO_PROTOTYPE)) || defined(__cplusplus) || defined(USE_PROTOTYPE) -# define _USING_PROTOTYPES_ 1 -# define _ANSI_ARGS_(x) x -# define CONST const -#else -# define _ANSI_ARGS_(x) () -# define CONST -#endif - -typedef void* erl_mutex_t; -typedef void* erl_cond_t; -typedef void* erl_thread_t; - -EXTERN erl_mutex_t erts_mutex_create _ANSI_ARGS_((void)); -EXTERN int erts_mutex_destroy _ANSI_ARGS_((erl_mutex_t)); -EXTERN int erts_mutex_lock _ANSI_ARGS_((erl_mutex_t)); -EXTERN int erts_mutex_unlock _ANSI_ARGS_((erl_mutex_t)); - -EXTERN erl_cond_t erts_cond_create _ANSI_ARGS_((void)); -EXTERN int erts_cond_destroy _ANSI_ARGS_((erl_cond_t)); -EXTERN int erts_cond_signal _ANSI_ARGS_((erl_cond_t)); -EXTERN int erts_cond_broadcast _ANSI_ARGS_((erl_cond_t)); -EXTERN int erts_cond_wait _ANSI_ARGS_((erl_cond_t, erl_mutex_t)); -EXTERN int erts_cond_timedwait _ANSI_ARGS_((erl_cond_t, erl_mutex_t, long)); - -EXTERN int erts_thread_create _ANSI_ARGS_((erl_thread_t*, - void* (*func)(void*), - void* arg, - int detached)); -EXTERN erl_thread_t erts_thread_self _ANSI_ARGS_((void)); -EXTERN void erts_thread_exit _ANSI_ARGS_((void*)); -EXTERN int erts_thread_join _ANSI_ARGS_((erl_thread_t, void**)); -EXTERN int erts_thread_kill _ANSI_ARGS_((erl_thread_t)); - -/* - * These functions implement the thread interface in ../obsolete/driver.h. - * Do *not* use this interface! Within the emulator, use the erl_threads.h, - * erl_smp.h, or ethread.h interface. From a driver use the thread interface - * in erl_driver.h. - */ - -erl_mutex_t -erts_mutex_create(void) -{ - return (erl_mutex_t) erl_drv_mutex_create(NULL); -} - -int -erts_mutex_destroy(erl_mutex_t mtx) -{ - erl_drv_mutex_destroy((ErlDrvMutex *) mtx); - return 0; -} - -int -erts_mutex_lock(erl_mutex_t mtx) -{ - erl_drv_mutex_lock((ErlDrvMutex *) mtx); - return 0; -} - -int -erts_mutex_unlock(erl_mutex_t mtx) -{ - erl_drv_mutex_unlock((ErlDrvMutex *) mtx); - return 0; -} - -erl_cond_t -erts_cond_create(void) -{ - return (erl_cond_t) erl_drv_cond_create(NULL); -} - -int -erts_cond_destroy(erl_cond_t cnd) -{ - erl_drv_cond_destroy((ErlDrvCond *) cnd); - return 0; -} - - -int -erts_cond_signal(erl_cond_t cnd) -{ - erl_drv_cond_signal((ErlDrvCond *) cnd); - return 0; -} - -int -erts_cond_broadcast(erl_cond_t cnd) -{ - erl_drv_cond_broadcast((ErlDrvCond *) cnd); - return 0; -} - - -int -erts_cond_wait(erl_cond_t cnd, erl_mutex_t mtx) -{ - erl_drv_cond_wait((ErlDrvCond *) cnd, (ErlDrvMutex *) mtx); - return 0; -} - -int -erts_cond_timedwait(erl_cond_t cnd, erl_mutex_t mtx, long ms) -{ - return ENOTSUP; -} - -int -erts_thread_create(erl_thread_t *tid, - void* (*func)(void*), - void* arg, - int detached) -{ - if (detached) - return ENOTSUP; - return erl_drv_thread_create(NULL, (ErlDrvTid *) tid, func, arg, NULL); -} - -erl_thread_t -erts_thread_self(void) -{ - return (erl_thread_t) erl_drv_thread_self(); -} - -void -erts_thread_exit(void *res) -{ - erl_drv_thread_exit(res); -} - -int -erts_thread_join(erl_thread_t tid, void **respp) -{ - return erl_drv_thread_join((ErlDrvTid) tid, respp); -} - -int -erts_thread_kill(erl_thread_t tid) -{ - return ENOTSUP; -} - diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 0b6bb0d8e9..c10724b951 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -625,6 +625,7 @@ erts_port_task_schedule(Eterm id, if (!enq_port) { ERTS_PT_CHK_PRES_PORTQ(runq, pp); + erts_smp_runq_unlock(runq); } else { enqueue_port(runq, pp); @@ -634,9 +635,10 @@ erts_port_task_schedule(Eterm id, profile_runnable_port(pp, am_active); } + erts_smp_runq_unlock(runq); + erts_smp_notify_inc_runq(runq); } - erts_smp_runq_unlock(runq); return 0; } @@ -944,8 +946,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) enqueue_port(xrunq, pp); ASSERT(pp->sched.exe_taskq); pp->sched.exe_taskq = NULL; - erts_smp_notify_inc_runq(xrunq); erts_smp_runq_unlock(xrunq); + erts_smp_notify_inc_runq(xrunq); } #endif port_was_enqueued = 1; @@ -969,11 +971,11 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) erts_port_release(pp); #else { - long refc = erts_smp_atomic_dectest(&pp->refc); + long refc; + erts_smp_mtx_unlock(pp->lock); + refc = erts_smp_atomic_dectest(&pp->refc); ASSERT(refc >= 0); - if (refc > 0) - erts_smp_mtx_unlock(pp->lock); - else { + if (refc == 0) { erts_smp_runq_unlock(runq); erts_port_cleanup(pp); /* Might aquire runq lock */ erts_smp_runq_lock(runq); @@ -1112,7 +1114,6 @@ erts_port_migrate(Port *prt, int *prt_locked, dequeue_port(from_rq, prt); erts_smp_atomic_set(&prt->run_queue, (long) to_rq); enqueue_port(to_rq, prt); - erts_smp_notify_inc_runq(to_rq); return ERTS_MIGRATE_SUCCESS; } diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index 7fe3f3bca5..d9f132f067 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -246,7 +246,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) if (is_CP(obj)) { PRINT_STRING(res, fn, arg, "<cp/header:"); - PRINT_POINTER(res, fn, arg, obj); + PRINT_POINTER(res, fn, arg, cp_val(obj)); PRINT_CHAR(res, fn, arg, '>'); return res; } @@ -406,7 +406,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) break; case EXPORT_DEF: { - Export* ep = (Export *) (export_val(obj))[1]; + Export* ep = *((Export **) (export_val(obj) + 1)); Atom* module = atom_tab(atom_val(ep->code[0])); Atom* name = atom_tab(atom_val(ep->code[1])); @@ -438,7 +438,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) break; default: PRINT_STRING(res, fn, arg, "<unknown:"); - PRINT_POINTER(res, fn, arg, obj); + PRINT_POINTER(res, fn, arg, (UWord) obj); PRINT_CHAR(res, fn, arg, '>'); break; } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 996806fc75..761096e9ad 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -38,6 +38,7 @@ #include "erl_instrument.h" #include "erl_threads.h" #include "erl_binary.h" +#include "beam_bp.h" #define ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED (2000*CONTEXT_REDS) #define ERTS_RUNQ_CALL_CHECK_BALANCE_REDS \ @@ -45,7 +46,13 @@ #define ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST (CONTEXT_REDS/10) -#define ERTS_SCHED_SLEEP_SPINCOUNT 10000 +#define ERTS_SCHED_SPIN_UNTIL_YIELD 100 + +#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT 10 +#define ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT 1000 +#define ERTS_SCHED_TSE_SLEEP_SPINCOUNT \ + (ERTS_SCHED_SYS_SLEEP_SPINCOUNT*ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT) +#define ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT 0 #define ERTS_WAKEUP_OTHER_LIMIT (100*CONTEXT_REDS/2) #define ERTS_WAKEUP_OTHER_DEC 10 @@ -91,9 +98,9 @@ do { \ #define ERTS_EMPTY_RUNQ(RQ) \ ((RQ)->len == 0 && (RQ)->misc.start == NULL) -extern Eterm beam_apply[]; -extern Eterm beam_exit[]; -extern Eterm beam_continue_exit[]; +extern BeamInstr beam_apply[]; +extern BeamInstr beam_exit[]; +extern BeamInstr beam_continue_exit[]; static Sint p_last; static Sint p_next; @@ -105,6 +112,10 @@ Uint erts_no_schedulers; Uint erts_max_processes = ERTS_DEFAULT_MAX_PROCESSES; Uint erts_process_tab_index_mask; +#ifdef ERTS_SMP +Uint erts_max_main_threads; +#endif + int erts_sched_thread_suggested_stack_size = -1; #ifdef ERTS_ENABLE_LOCK_CHECK @@ -115,16 +126,34 @@ ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE]; int erts_disable_proc_not_running_opt; -#define ERTS_SCHED_CHANGING_ONLINE 1 -#define ERTS_SCHED_CHANGING_MULTI_SCHED 2 +#define ERTS_SCHDLR_SSPND_CHNG_WAITER (((long) 1) << 0) +#define ERTS_SCHDLR_SSPND_CHNG_MSB (((long) 1) << 1) +#define ERTS_SCHDLR_SSPND_CHNG_ONLN (((long) 1) << 2) + +#ifndef DEBUG + +#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ + erts_smp_atomic_set(&schdlr_sspnd.changing, (VAL)) + +#else + +#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ +do { \ + long old_val__ = erts_smp_atomic_xchg(&schdlr_sspnd.changing, \ + (VAL)); \ + ASSERT(old_val__ == (OLD_VAL)); \ +} while (0) + +#endif + static struct { erts_smp_mtx_t mtx; erts_smp_cnd_t cnd; - int changing; int online; int curr_online; int wait_curr_online; + erts_smp_atomic_t changing; erts_smp_atomic_t active; struct { erts_smp_atomic_t ongoing; @@ -180,6 +209,7 @@ static ErtsCpuBindData *scheduler2cpu_map; erts_smp_rwmtx_t erts_cpu_bind_rwmtx; typedef enum { + ERTS_CPU_BIND_UNDEFINED, ERTS_CPU_BIND_SPREAD, ERTS_CPU_BIND_PROCESSOR_SPREAD, ERTS_CPU_BIND_THREAD_SPREAD, @@ -190,6 +220,9 @@ typedef enum { ERTS_CPU_BIND_NONE } ErtsCpuBindOrder; +#define ERTS_CPU_BIND_DEFAULT_BIND \ + ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD + ErtsCpuBindOrder cpu_bind_order; static erts_cpu_topology_t *user_cpudata; @@ -226,6 +259,17 @@ typedef union { ErtsAlignedSchedulerData *erts_aligned_scheduler_data; +#ifdef ERTS_SMP + +typedef union { + ErtsSchedulerSleepInfo ssi; + char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerSleepInfo))]; +} ErtsAlignedSchedulerSleepInfo; + +static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info; + +#endif + #ifndef BM_COUNTERS static int processes_busy; #endif @@ -283,8 +327,15 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist, 200, ERTS_ALC_T_PROC_LIST) -#define ERTS_RUNQ_IX(IX) (&erts_aligned_run_queues[(IX)].runq) -#define ERTS_SCHEDULER_IX(IX) (&erts_aligned_scheduler_data[(IX)].esd) +#define ERTS_RUNQ_IX(IX) \ + (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_run_queues), \ + &erts_aligned_run_queues[(IX)].runq) +#define ERTS_SCHEDULER_IX(IX) \ + (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers), \ + &erts_aligned_scheduler_data[(IX)].esd) +#define ERTS_SCHED_SLEEP_INFO_IX(IX) \ + (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers), \ + &aligned_sched_sleep_info[(IX)].ssi) #define ERTS_FOREACH_RUNQ(RQVAR, DO) \ do { \ @@ -334,7 +385,7 @@ do { \ static void init_processes_bif(void); static void save_terminating_process(Process *p); static void exec_misc_ops(ErtsRunQueue *); -static void print_function_from_pc(int to, void *to_arg, Eterm* x); +static void print_function_from_pc(int to, void *to_arg, BeamInstr* x); static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg); #ifdef ERTS_SMP @@ -348,6 +399,11 @@ static void signal_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size #endif +static int reader_group_lookup(int logical); +static void create_tmp_cpu_topology_copy(erts_cpu_topology_t **cpudata, + int *cpudata_size); +static void destroy_tmp_cpu_topology_copy(erts_cpu_topology_t *cpudata); + static void early_cpu_bind_init(void); static void late_cpu_bind_init(void); @@ -388,7 +444,12 @@ erts_pre_init_process(void) erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].get_locks = ERTS_PSD_DIST_ENTRY_GET_LOCKS; erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].set_locks - = ERTS_PSD_DIST_ENTRY_GET_LOCKS; + = ERTS_PSD_DIST_ENTRY_SET_LOCKS; + + erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].get_locks + = ERTS_PSD_CALL_TIME_BP_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].set_locks + = ERTS_PSD_CALL_TIME_BP_SET_LOCKS; /* Check that we have locks for all entries */ for (ix = 0; ix < ERTS_PSD_SIZE; ix++) { @@ -572,6 +633,76 @@ erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data) #ifdef ERTS_SMP +void +erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, long flags) +{ + switch (flags & ERTS_SSI_FLGS_SLEEP_TYPE) { + case ERTS_SSI_FLG_POLL_SLEEPING: + erts_sys_schedule_interrupt(1); + break; + case ERTS_SSI_FLG_TSE_SLEEPING: + erts_tse_set(ssi->event); + break; + case 0: + break; + default: + erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n", + __FILE__, __LINE__); + break; + } +} + +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN +void +erts_smp_notify_check_children_needed(void) +{ + int i; + + for (i = 0; i < erts_no_schedulers; i++) { + long aux_work; + ErtsSchedulerSleepInfo *ssi; + ssi = ERTS_SCHED_SLEEP_INFO_IX(i); + aux_work = erts_smp_atomic_bor(&ssi->aux_work, + ERTS_SSI_AUX_WORK_CHECK_CHILDREN); + if (!(aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN)) + erts_sched_poke(ssi); + } +} +#endif + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK +static ERTS_INLINE long +blockable_aux_work(ErtsSchedulerData *esdp, + ErtsSchedulerSleepInfo *ssi, + long aux_work) +{ + if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN + if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) { + aux_work = erts_smp_atomic_band(&ssi->aux_work, + ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN); + aux_work &= ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN; + erts_check_children(); + } +#endif + } + return aux_work; +} + +#endif + +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK +static ERTS_INLINE long +nonblockable_aux_work(ErtsSchedulerData *esdp, + ErtsSchedulerSleepInfo *ssi, + long aux_work) +{ + if (aux_work & ERTS_SSI_NONBLOCKABLE_AUX_WORK_MASK) { + + } +} +#endif + static void prepare_for_block(void *vrq) { @@ -624,7 +755,31 @@ erts_active_schedulers(void) return as; } +static ERTS_INLINE int +prepare_for_sys_schedule(void) +{ #ifdef ERTS_SMP + while (!erts_port_task_have_outstanding_io_tasks() + && !erts_smp_atomic_xchg(&doing_sys_schedule, 1)) { + if (!erts_port_task_have_outstanding_io_tasks()) + return 1; + erts_smp_atomic_set(&doing_sys_schedule, 0); + } + return 0; +#else + return !erts_port_task_have_outstanding_io_tasks(); +#endif +} + +#ifdef ERTS_SMP + +static ERTS_INLINE void +sched_change_waiting_sys_to_waiting(Uint no, ErtsRunQueue *rq) +{ + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ASSERT(rq->waiting < 0); + rq->waiting *= -1; +} static ERTS_INLINE void sched_waiting(Uint no, ErtsRunQueue *rq) @@ -666,7 +821,11 @@ empty_runq(ErtsRunQueue *rq) if (oifls & ERTS_RUNQ_IFLG_NONEMPTY) { #ifdef DEBUG long empty = erts_smp_atomic_read(&no_empty_run_queues); - ASSERT(0 <= empty && empty < erts_no_run_queues); + /* + * For a short period of time no_empty_run_queues may have + * been increased twice for a specific run queue. + */ + ASSERT(0 <= empty && empty < 2*erts_no_run_queues); #endif erts_smp_atomic_inc(&no_empty_run_queues); } @@ -679,107 +838,314 @@ non_empty_runq(ErtsRunQueue *rq) if (!(oifls & ERTS_RUNQ_IFLG_NONEMPTY)) { #ifdef DEBUG long empty = erts_smp_atomic_read(&no_empty_run_queues); - ASSERT(0 < empty && empty <= erts_no_run_queues); + /* + * For a short period of time no_empty_run_queues may have + * been increased twice for a specific run queue. + */ + ASSERT(0 < empty && empty <= 2*erts_no_run_queues); #endif erts_smp_atomic_dec(&no_empty_run_queues); } } -static ERTS_INLINE int -sched_spin_wake(ErtsRunQueue *rq) +static long +sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi) { -#if ERTS_SCHED_SLEEP_SPINCOUNT == 0 - return 0; -#else - long val; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + long oflgs; + long nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING); + long xflgs = 0; - val = erts_smp_atomic_read(&rq->spin_waiter); - ASSERT(val >= 0); - if (val != 0) { - erts_smp_atomic_inc(&rq->spin_wake); - return 1; - } - return 0; -#endif + do { + oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + xflgs = oflgs; + } while (!(oflgs & ERTS_SSI_FLG_SUSPENDED)); + return oflgs; } -static ERTS_INLINE int -sched_spin_wake_all(ErtsRunQueue *rq) +static long +sched_prep_cont_spin_wait(ErtsSchedulerSleepInfo *ssi) { -#if ERTS_SCHED_SLEEP_SPINCOUNT == 0 - return 0; -#else - long val; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + long oflgs; + long nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING); + long xflgs = ERTS_SSI_FLG_WAITING; - val = erts_smp_atomic_read(&rq->spin_waiter); - ASSERT(val >= 0); - if (val != 0) - erts_smp_atomic_add(&rq->spin_wake, val); - return val; -#endif + do { + oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + xflgs = oflgs; + nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED; + } while (oflgs & ERTS_SSI_FLG_WAITING); + return oflgs; } +static long +sched_spin_wait(ErtsSchedulerSleepInfo *ssi, int spincount) +{ + long until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; + int sc = spincount; + long flgs; + + do { + flgs = erts_smp_atomic_read(&ssi->flags); + if ((flgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) + != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) { + break; + } + ERTS_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; + erts_thr_yield(); + } + } while (--sc > 0); + return flgs; +} + +static long +sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, long sleep_type) +{ + long oflgs; + long nflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING|sleep_type; + long xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING; + + if (sleep_type == ERTS_SSI_FLG_TSE_SLEEPING) + erts_tse_reset(ssi->event); + + while (1) { + oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + if ((oflgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) + != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) { + return oflgs; + } + xflgs = oflgs; + nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED; + } +} + +#define ERTS_SCHED_WAIT_WOKEN(FLGS) \ + (((FLGS) & (ERTS_SSI_FLG_WAITING|ERTS_SSI_FLG_SUSPENDED)) \ + != ERTS_SSI_FLG_WAITING) + static void -sched_sys_wait(Uint no, ErtsRunQueue *rq) +scheduler_wait(long *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) { - long dt; -#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 - int val; - int spincount = ERTS_SCHED_SLEEP_SPINCOUNT; + ErtsSchedulerSleepInfo *ssi = esdp->ssi; + int spincount; + long flgs; +#if defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) \ + || defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) + long aux_work; +#endif + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + erts_smp_spin_lock(&rq->sleepers.lock); + flgs = sched_prep_spin_wait(ssi); + if (flgs & ERTS_SSI_FLG_SUSPENDED) { + /* Go suspend instead... */ + erts_smp_spin_unlock(&rq->sleepers.lock); + return; + } + + ssi->prev = NULL; + ssi->next = rq->sleepers.list; + if (rq->sleepers.list) + rq->sleepers.list->prev = ssi; + rq->sleepers.list = ssi; + erts_smp_spin_unlock(&rq->sleepers.lock); + + /* + * If all schedulers are waiting, one of them *should* + * be waiting in erl_sys_schedule() + */ + + if (!prepare_for_sys_schedule()) { + + sched_waiting(esdp->no, rq); + + erts_smp_runq_unlock(rq); + + spincount = ERTS_SCHED_TSE_SLEEP_SPINCOUNT; + + tse_wait: + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); + tse_blockable_aux_work: + aux_work = blockable_aux_work(esdp, ssi, aux_work); #endif + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - sched_waiting_sys(no, rq); + while (1) { -#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 - erts_smp_atomic_inc(&rq->spin_waiter); - erts_smp_runq_unlock(rq); +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK +#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); +#endif + nonblockable_aux_work(esdp, ssi, aux_work); +#endif - erl_sys_schedule(1); /* Might give us something to do */ + flgs = sched_spin_wait(ssi, spincount); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + int res; + ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + } + } - dt = do_time_read_and_reset(); - if (dt) bump_timer(dt); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + break; + } + + flgs = sched_prep_cont_spin_wait(ssi); + spincount = ERTS_SCHED_TSE_SLEEP_SPINCOUNT; + + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + break; + } + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); + if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + goto tse_blockable_aux_work; + } +#endif - while (spincount-- > 0) { - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) { - erts_smp_runq_lock(rq); - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) - goto woken; - if (spincount == 0) - goto sleep; - erts_smp_runq_unlock(rq); } - } - erts_smp_runq_lock(rq); - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) { - woken: - erts_smp_atomic_dec(&rq->spin_wake); - ASSERT(erts_smp_atomic_read(&rq->spin_wake) >= 0); - erts_smp_atomic_dec(&rq->spin_waiter); - ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + + if (flgs & ~ERTS_SSI_FLG_SUSPENDED) + erts_smp_atomic_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + + erts_smp_runq_lock(rq); + sched_active(esdp->no, rq); + } else { - sleep: - erts_smp_atomic_dec(&rq->spin_waiter); - ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); + long dt; + + erts_smp_atomic_set(&function_calls, 0); + *fcalls = 0; + + sched_waiting_sys(esdp->no, rq); + + erts_smp_runq_unlock(rq); + + spincount = ERTS_SCHED_SYS_SLEEP_SPINCOUNT; + + while (spincount-- > 0) { + + sys_poll_aux_work: + + erl_sys_schedule(1); /* Might give us something to do */ + + dt = do_time_read_and_reset(); + if (dt) bump_timer(dt); + + sys_aux_work: + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); + aux_work = blockable_aux_work(esdp, ssi, aux_work); +#endif +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK +#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); +#endif + nonblockable_aux_work(esdp, ssi, aux_work); +#endif + + flgs = erts_smp_atomic_read(&ssi->flags); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + goto sys_woken; + } + if (!(flgs & ERTS_SSI_FLG_SLEEPING)) { + flgs = sched_prep_cont_spin_wait(ssi); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + goto sys_woken; + } + } + + /* + * If we got new I/O tasks we aren't allowed to + * call erl_sys_schedule() until it is handled. + */ + if (erts_port_task_have_outstanding_io_tasks()) { + erts_smp_atomic_set(&doing_sys_schedule, 0); + /* + * Got to check that we still got I/O tasks; otherwise + * we have to continue checking for I/O... + */ + if (!prepare_for_sys_schedule()) { + spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT; + goto tse_wait; + } + } + } + + erts_smp_runq_lock(rq); + /* * If we got new I/O tasks we aren't allowed to * sleep in erl_sys_schedule(). */ - if (!erts_port_task_have_outstanding_io_tasks()) { -#endif + if (erts_port_task_have_outstanding_io_tasks()) { + erts_smp_atomic_set(&doing_sys_schedule, 0); + /* + * Got to check that we still got I/O tasks; otherwise + * we have to wait in erl_sys_schedule() after all... + */ + if (prepare_for_sys_schedule()) + goto do_sys_schedule; + + /* + * Not allowed to wait in erl_sys_schedule; + * do tse wait instead... + */ + sched_change_waiting_sys_to_waiting(esdp->no, rq); + erts_smp_runq_unlock(rq); + spincount = 0; + goto tse_wait; + } + else { + do_sys_schedule: erts_sys_schedule_interrupt(0); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING); + if (!(flgs & ERTS_SSI_FLG_SLEEPING)) { + if (!(flgs & ERTS_SSI_FLG_WAITING)) + goto sys_locked_woken; + erts_smp_runq_unlock(rq); + flgs = sched_prep_cont_spin_wait(ssi); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + goto sys_woken; + } + ASSERT(!erts_port_task_have_outstanding_io_tasks()); + goto sys_poll_aux_work; + } + + ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + erts_smp_runq_unlock(rq); erl_sys_schedule(0); @@ -787,134 +1153,103 @@ sched_sys_wait(Uint no, ErtsRunQueue *rq) dt = do_time_read_and_reset(); if (dt) bump_timer(dt); - erts_smp_runq_lock(rq); + flgs = sched_prep_cont_spin_wait(ssi); + if (flgs & ERTS_SSI_FLG_WAITING) + goto sys_aux_work; -#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 + sys_woken: + erts_smp_runq_lock(rq); + sys_locked_woken: + erts_smp_atomic_set(&doing_sys_schedule, 0); + if (flgs & ~ERTS_SSI_FLG_SUSPENDED) + erts_smp_atomic_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + sched_active_sys(esdp->no, rq); } } -#endif - - sched_active_sys(no, rq); -} -static void -sched_cnd_wait(Uint no, ErtsRunQueue *rq) -{ -#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 - int val; - int spincount = ERTS_SCHED_SLEEP_SPINCOUNT; ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); -#endif - - sched_waiting(no, rq); - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - prepare_for_block, - resume_after_block, - (void *) rq); - -#if ERTS_SCHED_SLEEP_SPINCOUNT == 0 - erts_smp_cnd_wait(&rq->cnd, &rq->mtx); -#else - erts_smp_atomic_inc(&rq->spin_waiter); - erts_smp_mtx_unlock(&rq->mtx); - - while (spincount-- > 0) { - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) { - erts_smp_mtx_lock(&rq->mtx); - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) - goto woken; - if (spincount == 0) - goto sleep; - erts_smp_mtx_unlock(&rq->mtx); - } - } - - erts_smp_mtx_lock(&rq->mtx); - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val == 0) { - sleep: - erts_smp_atomic_dec(&rq->spin_waiter); - ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); - erts_smp_cnd_wait(&rq->cnd, &rq->mtx); - } - else { - woken: - erts_smp_atomic_dec(&rq->spin_wake); - ASSERT(erts_smp_atomic_read(&rq->spin_wake) >= 0); - erts_smp_atomic_dec(&rq->spin_waiter); - ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); - } -#endif - - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - prepare_for_block, - resume_after_block, - (void *) rq); - - sched_active(no, rq); } -static void -wake_one_scheduler(void) -{ - ASSERT(erts_common_run_queue); - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(erts_common_run_queue)); - if (erts_common_run_queue->waiting) { - if (!sched_spin_wake(erts_common_run_queue)) { - if (erts_common_run_queue->waiting == -1) /* One scheduler waiting - and doing so in - sys_schedule */ - erts_sys_schedule_interrupt(1); - else - erts_smp_cnd_signal(&erts_common_run_queue->cnd); - } +static ERTS_INLINE long +ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi) +{ + /* reset all flags but suspended */ + long oflgs; + long nflgs = 0; + long xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING; + while (1) { + oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return oflgs; + nflgs = oflgs & ERTS_SSI_FLG_SUSPENDED; + xflgs = oflgs; } } static void -wake_scheduler(ErtsRunQueue *rq, int incq) +wake_scheduler(ErtsRunQueue *rq, int incq, int one) { - ASSERT(!erts_common_run_queue); - ASSERT(-1 <= rq->waiting && rq->waiting <= 1); - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - if (rq->waiting && !rq->woken) { - if (!sched_spin_wake(rq)) { - if (rq->waiting < 0) - erts_sys_schedule_interrupt(1); - else - erts_smp_cnd_signal(&rq->cnd); + int res; + ErtsSchedulerSleepInfo *ssi; + ErtsSchedulerSleepList *sl; + + /* + * The unlocked run queue is not strictly necessary + * from a thread safety or deadlock prevention + * perspective. It will, however, cost us performance + * if it is locked during wakup of another scheduler, + * so all code *should* handle this without having + * the lock on the run queue. + */ + ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq)); + + sl = &rq->sleepers; + + erts_smp_spin_lock(&sl->lock); + ssi = sl->list; + if (!ssi) + erts_smp_spin_unlock(&sl->lock); + else if (one) { + long flgs; + if (ssi->prev) + ssi->prev->next = ssi->next; + else { + ASSERT(sl->list == ssi); + sl->list = ssi->next; } - rq->woken = 1; - if (incq) + if (ssi->next) + ssi->next->prev = ssi->prev; + + res = sl->list != NULL; + erts_smp_spin_unlock(&sl->lock); + + flgs = ssi_flags_set_wake(ssi); + erts_sched_finish_poke(ssi, flgs); + + if (incq && !erts_common_run_queue && (flgs & ERTS_SSI_FLG_WAITING)) non_empty_runq(rq); } + else { + sl->list = NULL; + erts_smp_spin_unlock(&sl->lock); + do { + ErtsSchedulerSleepInfo *wake_ssi = ssi; + ssi = ssi->next; + erts_sched_finish_poke(ssi, ssi_flags_set_wake(wake_ssi)); + } while (ssi); + } } static void wake_all_schedulers(void) { - if (erts_common_run_queue) { - erts_smp_runq_lock(erts_common_run_queue); - if (erts_common_run_queue->waiting) { - if (erts_common_run_queue->waiting < 0) - erts_sys_schedule_interrupt(1); - sched_spin_wake_all(erts_common_run_queue); - erts_smp_cnd_broadcast(&erts_common_run_queue->cnd); - } - erts_smp_runq_unlock(erts_common_run_queue); - } + if (erts_common_run_queue) + wake_scheduler(erts_common_run_queue, 0, 0); else { int ix; for (ix = 0; ix < erts_no_run_queues; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - erts_smp_runq_lock(rq); - wake_scheduler(rq, 0); - erts_smp_runq_unlock(rq); + wake_scheduler(rq, 0, 1); } } } @@ -929,14 +1264,14 @@ chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) wrq = ERTS_RUNQ_IX(ix); iflgs = erts_smp_atomic_read(&wrq->info_flags); if (!(iflgs & (ERTS_RUNQ_IFLG_SUSPENDED|ERTS_RUNQ_IFLG_NONEMPTY))) { - erts_smp_xrunq_lock(crq, wrq); if (activate) { if (ix == erts_smp_atomic_cmpxchg(&balance_info.active_runqs, ix+1, ix)) { + erts_smp_xrunq_lock(crq, wrq); wrq->flags &= ~ERTS_RUNQ_FLG_INACTIVE; + erts_smp_xrunq_unlock(crq, wrq); } } - wake_scheduler(wrq, 0); - erts_smp_xrunq_unlock(crq, wrq); + wake_scheduler(wrq, 0, 1); return 1; } return 0; @@ -982,15 +1317,13 @@ static ERTS_INLINE void smp_notify_inc_runq(ErtsRunQueue *runq) { #ifdef ERTS_SMP - if (erts_common_run_queue) - wake_one_scheduler(); - else - wake_scheduler(runq, 1); + if (runq) + wake_scheduler(runq, 1, 1); #endif } void -erts_smp_notify_inc_runq__(ErtsRunQueue *runq) +erts_smp_notify_inc_runq(ErtsRunQueue *runq) { smp_notify_inc_runq(runq); } @@ -1136,20 +1469,23 @@ static void evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) { Port *prt; + int notify_to_rq = 0; int prio; int prt_locked = 0; int rq_locked = 0; int evac_rq_locked = 1; + ErtsMigrateResult mres; erts_smp_runq_lock(evac_rq); + erts_smp_atomic_bor(&evac_rq->scheduler->ssi->flags, ERTS_SSI_FLG_SUSPENDED); + evac_rq->flags &= ~ERTS_RUNQ_FLGS_IMMIGRATE_QMASK; evac_rq->flags |= (ERTS_RUNQ_FLGS_EMIGRATE_QMASK | ERTS_RUNQ_FLGS_EVACUATE_QMASK | ERTS_RUNQ_FLG_SUSPENDED); erts_smp_atomic_bor(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED); - /* * Need to set up evacuation paths first since we * may release the run queue lock on evac_rq @@ -1177,9 +1513,11 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) /* Evacuate scheduled ports */ prt = evac_rq->ports.start; while (prt) { - (void) erts_port_migrate(prt, &prt_locked, + mres = erts_port_migrate(prt, &prt_locked, evac_rq, &evac_rq_locked, rq, &rq_locked); + if (mres == ERTS_MIGRATE_SUCCESS) + notify_to_rq = 1; if (prt_locked) erts_smp_port_unlock(prt); if (!evac_rq_locked) { @@ -1208,9 +1546,11 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) goto end_of_proc; } - (void) erts_proc_migrate(proc, &proc_locks, + mres = erts_proc_migrate(proc, &proc_locks, evac_rq, &evac_rq_locked, rq, &rq_locked); + if (mres == ERTS_MIGRATE_SUCCESS) + notify_to_rq = 1; if (proc_locks) erts_smp_proc_unlock(proc, proc_locks); if (!evac_rq_locked) { @@ -1242,10 +1582,13 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) if (rq_locked) erts_smp_runq_unlock(rq); - if (!evac_rq_locked) - erts_smp_runq_lock(evac_rq); - wake_scheduler(evac_rq, 0); - erts_smp_runq_unlock(evac_rq); + if (evac_rq_locked) + erts_smp_runq_unlock(evac_rq); + + if (notify_to_rq) + smp_notify_inc_runq(rq); + + wake_scheduler(evac_rq, 0, 1); } static int @@ -1473,31 +1816,6 @@ try_steal_task(ErtsRunQueue *rq) return res; } -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN -void -erts_smp_notify_check_children_needed(void) -{ - int i; - for (i = 0; i < erts_no_schedulers; i++) { - erts_smp_runq_lock(ERTS_SCHEDULER_IX(i)->run_queue); - ERTS_SCHEDULER_IX(i)->check_children = 1; - if (!erts_common_run_queue) - wake_scheduler(ERTS_SCHEDULER_IX(i)->run_queue, 0); - erts_smp_runq_unlock(ERTS_SCHEDULER_IX(i)->run_queue); - } - if (ongoing_multi_scheduling_block()) { - /* Also blocked schedulers need to check children */ - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - for (i = 0; i < erts_no_schedulers; i++) - ERTS_SCHEDULER_IX(i)->blocked_check_children = 1; - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - } - if (erts_common_run_queue) - wake_all_schedulers(); -} -#endif - /* Run queue balancing */ typedef struct { @@ -2078,16 +2396,20 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_aligned_run_queues = erts_alloc(ERTS_ALC_T_RUNQS, (sizeof(ErtsAlignedRunQueue)*(n+1))); - if ((((Uint) erts_aligned_run_queues) & ERTS_CACHE_LINE_MASK) == 0) + if ((((UWord) erts_aligned_run_queues) & ERTS_CACHE_LINE_MASK) != 0) erts_aligned_run_queues = ((ErtsAlignedRunQueue *) - ((((Uint) erts_aligned_run_queues) + ((((UWord) erts_aligned_run_queues) & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE)); + ASSERT((((UWord) erts_aligned_run_queues) & ERTS_CACHE_LINE_MASK) == 0); + #ifdef ERTS_SMP erts_smp_atomic_init(&no_empty_run_queues, 0); #endif + erts_no_run_queues = n; + for (ix = 0; ix < n; ix++) { int pix, rix; ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); @@ -2102,8 +2424,10 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_smp_mtx_init_x(&rq->mtx, "run_queue", make_small(ix + 1)); erts_smp_cnd_init(&rq->cnd); - erts_smp_atomic_init(&rq->spin_waiter, 0); - erts_smp_atomic_init(&rq->spin_wake, 0); +#ifdef ERTS_SMP + erts_smp_spinlock_init(&rq->sleepers.lock, "run_queue_sleep_list"); + rq->sleepers.list = NULL; +#endif rq->waiting = 0; rq->woken = 0; @@ -2154,7 +2478,6 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) } erts_common_run_queue = !mrq ? ERTS_RUNQ_IX(0) : NULL; - erts_no_run_queues = n; #ifdef ERTS_SMP @@ -2169,23 +2492,59 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) #endif + n = (int) no_schedulers; + erts_no_schedulers = n; + +#ifdef ERTS_SMP + /* Create and initialize scheduler sleep info */ + + aligned_sched_sleep_info = erts_alloc(ERTS_ALC_T_SCHDLR_SLP_INFO, + (sizeof(ErtsAlignedSchedulerSleepInfo) + *(n+1))); + if ((((Uint) aligned_sched_sleep_info) & ERTS_CACHE_LINE_MASK) == 0) + aligned_sched_sleep_info = ((ErtsAlignedSchedulerSleepInfo *) + ((((Uint) aligned_sched_sleep_info) + & ~ERTS_CACHE_LINE_MASK) + + ERTS_CACHE_LINE_SIZE)); + for (ix = 0; ix < n; ix++) { + ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); +#if 0 /* no need to initialize these... */ + ssi->next = NULL; + ssi->prev = NULL; +#endif + erts_smp_atomic_init(&ssi->flags, 0); + ssi->event = NULL; /* initialized in sched_thread_func */ + erts_smp_atomic_init(&ssi->aux_work, 0); + } +#endif + /* Create and initialize scheduler specific data */ - n = (int) no_schedulers; erts_aligned_scheduler_data = erts_alloc(ERTS_ALC_T_SCHDLR_DATA, (sizeof(ErtsAlignedSchedulerData) *(n+1))); - if ((((Uint) erts_aligned_scheduler_data) & ERTS_CACHE_LINE_MASK) == 0) + if ((((UWord) erts_aligned_scheduler_data) & ERTS_CACHE_LINE_MASK) != 0) erts_aligned_scheduler_data = ((ErtsAlignedSchedulerData *) - ((((Uint) erts_aligned_scheduler_data) + ((((UWord) erts_aligned_scheduler_data) & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE)); + + ASSERT((((UWord) erts_aligned_scheduler_data) & ERTS_CACHE_LINE_MASK) == 0); + for (ix = 0; ix < n; ix++) { ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix); #ifdef ERTS_SMP erts_bits_init_state(&esdp->erl_bits_state); esdp->match_pseudo_process = NULL; + esdp->ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); esdp->free_process = NULL; +#if HALFWORD_HEAP + /* Registers need to be heap allocated (correct memory range) for tracing to work */ + esdp->save_reg = erts_alloc(ERTS_ALC_T_BEAM_REGISTER, ERTS_X_REGS_ALLOCATED * sizeof(Eterm)); +#endif +#endif +#if !HEAP_ON_C_STACK + esdp->num_tmp_heap_used = 0; #endif esdp->no = (Uint) ix+1; esdp->current_process = NULL; @@ -2206,11 +2565,6 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) } #ifdef ERTS_SMP -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - esdp->check_children = 0; - esdp->blocked_check_children = 0; -#endif - erts_smp_atomic_init(&esdp->suspended, 0); erts_smp_atomic_init(&esdp->chk_cpu_bind, 0); #endif } @@ -2219,7 +2573,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd"); erts_smp_cnd_init(&schdlr_sspnd.cnd); - schdlr_sspnd.changing = 0; + erts_smp_atomic_init(&schdlr_sspnd.changing, 0); schdlr_sspnd.online = no_schedulers_online; schdlr_sspnd.curr_online = no_schedulers; erts_smp_atomic_init(&schdlr_sspnd.msb.ongoing, 0); @@ -2242,7 +2596,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) if (no_schedulers_online < no_schedulers) { if (erts_common_run_queue) { for (ix = no_schedulers_online; ix < no_schedulers; ix++) - erts_smp_atomic_set(&(ERTS_SCHEDULER_IX(ix)->suspended), 1); + erts_smp_atomic_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ERTS_SSI_FLG_SUSPENDED); } else { for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++) @@ -2253,7 +2608,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) schdlr_sspnd.wait_curr_online = no_schedulers_online; schdlr_sspnd.curr_online *= 2; /* Boot strapping... */ - schdlr_sspnd.changing = ERTS_SCHED_CHANGING_ONLINE; + ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); erts_smp_atomic_init(&doing_sys_schedule, 0); @@ -2401,13 +2757,115 @@ susp_sched_resume_block(void *unused) } static void +scheduler_ix_resume_wake(Uint ix) +{ + ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); + long xflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + long oflgs; + do { + oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, 0, xflgs); + if (oflgs == xflgs) { + erts_sched_finish_poke(ssi, oflgs); + break; + } + xflgs = oflgs; + } while (oflgs & ERTS_SSI_FLG_SUSPENDED); +} + +static long +sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, long xpct) +{ + long oflgs; + long nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + long xflgs = xpct; + + do { + oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + xflgs = oflgs; + } while (oflgs & ERTS_SSI_FLG_SUSPENDED); + + return oflgs; +} + +static long +sched_spin_suspended(ErtsSchedulerSleepInfo *ssi, int spincount) +{ + int until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; + int sc = spincount; + long flgs; + + do { + flgs = erts_smp_atomic_read(&ssi->flags); + if ((flgs & (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) + != (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + break; + } + ERTS_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; + erts_thr_yield(); + } + } while (--sc > 0); + return flgs; +} + +static long +sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi) +{ + long oflgs; + long nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + long xflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + + erts_tse_reset(ssi->event); + + while (1) { + oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + if ((oflgs & (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) + != (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + return oflgs; + } + xflgs = oflgs; + } +} + +static void suspend_scheduler(ErtsSchedulerData *esdp) { + long flgs; + int changing; long no = (long) esdp->no; ErtsRunQueue *rq = esdp->run_queue; + ErtsSchedulerSleepInfo *ssi = esdp->ssi; long active_schedulers; int curr_online = 1; int wake = 0; + int reset_read_group = 0; +#if defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) \ + || defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) + long aux_work; +#endif /* * Schedulers may be suspended in two different ways: @@ -2429,113 +2887,151 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (scheduler2cpu_map[esdp->no].bound_id >= 0 && erts_unbind_from_cpu(erts_cpuinfo) == 0) { esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1; + reset_read_group = 1; } erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); + if (reset_read_group) + erts_smp_rwmtx_set_reader_group(0); + + if (esdp->no <= erts_max_main_threads) + erts_thr_set_main_status(0, 0); + if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_inactive); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - active_schedulers = erts_smp_atomic_dectest(&schdlr_sspnd.active); - ASSERT(active_schedulers >= 1); - if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_MULTI_SCHED) { - if (active_schedulers == schdlr_sspnd.msb.wait_active) - wake = 1; - if (active_schedulers == 1) - schdlr_sspnd.changing = 0; - } + flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); + if (flgs & ERTS_SSI_FLG_SUSPENDED) { - while (1) { - -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - int check_children; - erts_smp_runq_lock(esdp->run_queue); - check_children = esdp->check_children; - esdp->check_children = 0; - erts_smp_runq_unlock(esdp->run_queue); - if (check_children) { - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - erts_check_children(); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + active_schedulers = erts_smp_atomic_dectest(&schdlr_sspnd.active); + ASSERT(active_schedulers >= 1); + changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) { + if (active_schedulers == schdlr_sspnd.msb.wait_active) + wake = 1; + if (active_schedulers == 1) { + changing = erts_smp_atomic_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_MSB); + changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB; + } } -#endif - if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE) { - int changed = 0; - if (no > schdlr_sspnd.online && curr_online) { - schdlr_sspnd.curr_online--; - curr_online = 0; - changed = 1; + while (1) { + if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) { + int changed = 0; + if (no > schdlr_sspnd.online && curr_online) { + schdlr_sspnd.curr_online--; + curr_online = 0; + changed = 1; + } + else if (no <= schdlr_sspnd.online && !curr_online) { + schdlr_sspnd.curr_online++; + curr_online = 1; + changed = 1; + } + if (changed + && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) + wake = 1; + if (schdlr_sspnd.online == schdlr_sspnd.curr_online) { + changing = erts_smp_atomic_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_ONLN); + changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN; + } } - else if (no <= schdlr_sspnd.online && !curr_online) { - schdlr_sspnd.curr_online++; - curr_online = 1; - changed = 1; + + if (wake) { + erts_smp_cnd_signal(&schdlr_sspnd.cnd); + wake = 0; } - if (changed - && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) - wake = 1; - if (schdlr_sspnd.online == schdlr_sspnd.curr_online) - schdlr_sspnd.changing = 0; - } - if (wake) { - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - wake = 0; - } + flgs = erts_smp_atomic_read(&ssi->flags); + if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) + break; + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - if (!(rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ|ERTS_RUNQ_FLG_SUSPENDED))) - break; - if ((rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) - && !erts_smp_atomic_read(&esdp->suspended)) - break; +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); + blockable_aux_work: + blockable_aux_work(esdp, ssi, aux_work); +#endif - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - while (1) { + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + while (1) { + long flgs; +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK +#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); +#endif + nonblockable_aux_work(esdp, ssi, aux_work); +#endif -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - if (esdp->blocked_check_children) - break; + flgs = sched_spin_suspended(ssi, ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + flgs = sched_set_suspended_sleeptype(ssi); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + int res; + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + } + } + + flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)); + if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) + break; + changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER) + break; + + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); + if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + goto blockable_aux_work; + } #endif - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + } - if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE) - break; + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - if (!(rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ - | ERTS_RUNQ_FLG_SUSPENDED))) - break; - if ((rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) - && !erts_smp_atomic_read(&esdp->suspended)) - break; + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + changing = erts_smp_atomic_read(&schdlr_sspnd.changing); } -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - esdp->blocked_check_children = 0; -#endif + active_schedulers = erts_smp_atomic_inctest(&schdlr_sspnd.active); + changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) + && schdlr_sspnd.online == active_schedulers) { + erts_smp_atomic_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_MSB); + } - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - } + ASSERT(no <= schdlr_sspnd.online); + ASSERT(!erts_smp_atomic_read(&schdlr_sspnd.msb.ongoing)); - active_schedulers = erts_smp_atomic_inctest(&schdlr_sspnd.active); - if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_MULTI_SCHED - && schdlr_sspnd.online == active_schedulers) { - schdlr_sspnd.changing = 0; } + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + ASSERT(curr_online); + if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_active); + if (esdp->no <= erts_max_main_threads) + erts_thr_set_main_status(1, (int) esdp->no); + erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); @@ -2600,8 +3096,10 @@ erts_schedulers_state(Uint *total, int yield_allowed) { int res; + long changing; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - if (yield_allowed && schdlr_sspnd.changing) + changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + if (yield_allowed && (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER)) res = ERTS_SCHDLR_SSPND_YIELD_RESTART; else { *active = *online = schdlr_sspnd.online; @@ -2621,6 +3119,7 @@ erts_set_schedulers_online(Process *p, Sint *old_no) { int ix, res, no, have_unlocked_plocks; + long changing; if (new_no < 1 || erts_no_schedulers < new_no) return ERTS_SCHDLR_SSPND_EINVAL; @@ -2630,7 +3129,8 @@ erts_set_schedulers_online(Process *p, have_unlocked_plocks = 0; no = (int) new_no; - if (schdlr_sspnd.changing) { + changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + if (changing) { res = ERTS_SCHDLR_SSPND_YIELD_RESTART; } else { @@ -2639,17 +3139,19 @@ erts_set_schedulers_online(Process *p, res = ERTS_SCHDLR_SSPND_DONE; } else { - schdlr_sspnd.changing = ERTS_SCHED_CHANGING_ONLINE; + ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); schdlr_sspnd.online = no; if (no > online) { int ix; schdlr_sspnd.wait_curr_online = no; - if (ongoing_multi_scheduling_block()) - /* No schedulers to resume */; + if (ongoing_multi_scheduling_block()) { + for (ix = online; ix < no; ix++) + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); + } else if (erts_common_run_queue) { for (ix = online; ix < no; ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, - 0); + scheduler_ix_resume_wake(ix); } else { if (plocks) { @@ -2663,6 +3165,7 @@ erts_set_schedulers_online(Process *p, erts_smp_runq_lock(rq); ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x5); erts_smp_runq_unlock(rq); + scheduler_ix_resume_wake(ix); } /* * Spread evacuation paths among all online @@ -2677,7 +3180,6 @@ erts_set_schedulers_online(Process *p, erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); } - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); res = ERTS_SCHDLR_SSPND_DONE; } else /* if (no < online) */ { @@ -2694,12 +3196,17 @@ erts_set_schedulers_online(Process *p, schdlr_sspnd.wait_curr_online = no+1; } - if (ongoing_multi_scheduling_block()) - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - else if (erts_common_run_queue) { + if (ongoing_multi_scheduling_block()) { for (ix = no; ix < online; ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, - 1); + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); + } + else if (erts_common_run_queue) { + for (ix = no; ix < online; ix++) { + ErtsSchedulerSleepInfo *ssi; + ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic_bor(&ssi->flags, + ERTS_SSI_FLG_SUSPENDED); + } wake_all_schedulers(); } else { @@ -2726,7 +3233,10 @@ erts_set_schedulers_online(Process *p, erts_smp_atomic_set(&balance_info.used_runqs, no); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - ERTS_FOREACH_OP_RUNQ(rq, wake_scheduler(rq, 0)); + for (ix = no; ix < online; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + wake_scheduler(rq, 0, 1); + } } } @@ -2740,6 +3250,12 @@ erts_set_schedulers_online(Process *p, susp_sched_prep_block, susp_sched_resume_block, NULL); + ASSERT(res != ERTS_SCHDLR_SSPND_DONE + ? (ERTS_SCHDLR_SSPND_CHNG_WAITER + & erts_smp_atomic_read(&schdlr_sspnd.changing)) + : (ERTS_SCHDLR_SSPND_CHNG_WAITER + == erts_smp_atomic_read(&schdlr_sspnd.changing))); + erts_smp_atomic_band(&schdlr_sspnd.changing, ~ERTS_SCHDLR_SSPND_CHNG_WAITER); } } @@ -2754,15 +3270,16 @@ ErtsSchedSuspendResult erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) { int ix, res, have_unlocked_plocks = 0; + long changing; ErtsProcList *plp; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - - if (schdlr_sspnd.changing) { + changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + if (changing) { res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */ } else if (on) { /* ------ BLOCK ------ */ - if (erts_is_multi_scheduling_blocked()) { + if (schdlr_sspnd.msb.procs) { plp = proclist_create(p); plp->next = schdlr_sspnd.msb.procs; schdlr_sspnd.msb.procs = plp; @@ -2772,19 +3289,22 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; } else { + int online = schdlr_sspnd.online; p->flags |= F_HAVE_BLCKD_MSCHED; if (plocks) { have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } + ASSERT(0 == erts_smp_atomic_read(&schdlr_sspnd.msb.ongoing)); erts_smp_atomic_set(&schdlr_sspnd.msb.ongoing, 1); - if (schdlr_sspnd.online == 1) { + if (online == 1) { res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1); ASSERT(p->scheduler_data->no == 1); } else { - schdlr_sspnd.changing = ERTS_SCHED_CHANGING_MULTI_SCHED; + ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); if (p->scheduler_data->no == 1) { res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; schdlr_sspnd.msb.wait_active = 1; @@ -2798,17 +3318,19 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) schdlr_sspnd.msb.wait_active = 2; } if (erts_common_run_queue) { - for (ix = 1; ix < schdlr_sspnd.online; ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, 1); + for (ix = 1; ix < online; ix++) + erts_smp_atomic_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ERTS_SSI_FLG_SUSPENDED); wake_all_schedulers(); } else { erts_smp_mtx_unlock(&schdlr_sspnd.mtx); erts_smp_mtx_lock(&balance_info.update_mtx); erts_smp_atomic_set(&balance_info.used_runqs, 1); - for (ix = 0; ix < schdlr_sspnd.online; ix++) { + for (ix = 0; ix < online; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); erts_smp_runq_lock(rq); + ASSERT(!(rq->flags & ERTS_RUNQ_FLG_SUSPENDED)); ERTS_RUNQ_RESET_MIGRATION_PATHS(rq, 0x7); erts_smp_runq_unlock(rq); } @@ -2833,6 +3355,13 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) susp_sched_prep_block, susp_sched_resume_block, NULL); + ASSERT(res != ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED + ? (ERTS_SCHDLR_SSPND_CHNG_WAITER + & erts_smp_atomic_read(&schdlr_sspnd.changing)) + : (ERTS_SCHDLR_SSPND_CHNG_WAITER + == erts_smp_atomic_read(&schdlr_sspnd.changing))); + erts_smp_atomic_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_WAITER); } plp = proclist_create(p); plp->next = schdlr_sspnd.msb.procs; @@ -2876,7 +3405,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) if (schdlr_sspnd.msb.procs) res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; else { - schdlr_sspnd.changing = ERTS_SCHED_CHANGING_MULTI_SCHED; + ERTS_SCHDLR_SSPND_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0); #ifdef DEBUG ERTS_FOREACH_RUNQ(rq, { @@ -2903,13 +3432,13 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) if (schdlr_sspnd.online == 1) { /* No schedulers to resume */ ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1); - schdlr_sspnd.changing = 0; + ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_MSB); } else if (erts_common_run_queue) { for (ix = 1; ix < schdlr_sspnd.online; ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, 0); + erts_smp_atomic_band(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ~ERTS_SSI_FLG_SUSPENDED); wake_all_schedulers(); - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); } else { int online = schdlr_sspnd.online; @@ -2926,6 +3455,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) erts_smp_runq_lock(rq); ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x4); erts_smp_runq_unlock(rq); + scheduler_ix_resume_wake(ix); } /* Spread evacuation paths among all online run queues */ @@ -2941,7 +3471,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) erts_smp_runq_unlock(ERTS_RUNQ_IX(0)); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); } res = ERTS_SCHDLR_SSPND_DONE; } @@ -2968,8 +3497,11 @@ erts_dbg_multi_scheduling_return_trap(Process *p, Eterm return_value) int erts_is_multi_scheduling_blocked(void) { - return (erts_smp_atomic_read(&schdlr_sspnd.msb.ongoing) - && erts_smp_atomic_read(&schdlr_sspnd.active) == 1); + int res; + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + res = schdlr_sspnd.msb.procs != NULL; + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + return res; } Eterm @@ -2978,7 +3510,7 @@ erts_multi_scheduling_blockers(Process *p) Eterm res = NIL; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - if (erts_is_multi_scheduling_blocked()) { + if (schdlr_sspnd.msb.procs) { Eterm *hp, *hp_end; ErtsProcList *plp1, *plp2; Uint max_size; @@ -3010,18 +3542,34 @@ erts_multi_scheduling_blockers(Process *p) static void * sched_thread_func(void *vesdp) { +#ifdef ERTS_SMP + Uint no = ((ErtsSchedulerData *) vesdp)->no; +#endif #ifdef ERTS_ENABLE_LOCK_CHECK { char buf[31]; - Uint no = ((ErtsSchedulerData *) vesdp)->no; erts_snprintf(&buf[0], 31, "scheduler %bpu", no); erts_lc_set_thread_name(&buf[0]); } #endif - erts_alloc_reg_scheduler_id(((ErtsSchedulerData *) vesdp)->no); + erts_alloc_reg_scheduler_id(no); erts_tsd_set(sched_data_key, vesdp); #ifdef ERTS_SMP + + if (no <= erts_max_main_threads) { + erts_thr_set_main_status(1, (int) no); + if (erts_reader_groups) { + int rg = (int) no; + if (rg > erts_reader_groups) + rg = (((int) no) - 1) % erts_reader_groups + 1; + erts_smp_rwmtx_set_reader_group(rg); + } + } + erts_proc_lock_prepare_proc_lock_waiter(); + ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch(); + + #endif erts_register_blockable_thread(); #ifdef HIPE @@ -3030,30 +3578,30 @@ sched_thread_func(void *vesdp) erts_thread_init_float(); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - ASSERT(schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE); + ASSERT(erts_smp_atomic_read(&schdlr_sspnd.changing) + & ERTS_SCHDLR_SSPND_CHNG_ONLN); - schdlr_sspnd.curr_online--; + if (--schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) { + erts_smp_atomic_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_ONLN); + if (((ErtsSchedulerData *) vesdp)->no != 1) + erts_smp_cnd_signal(&schdlr_sspnd.cnd); + } - if (((ErtsSchedulerData *) vesdp)->no != 1) { - if (schdlr_sspnd.online == schdlr_sspnd.curr_online) { - schdlr_sspnd.changing = 0; - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); + if (((ErtsSchedulerData *) vesdp)->no == 1) { + if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) { + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, + susp_sched_prep_block, + susp_sched_resume_block, + NULL); + while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, + susp_sched_prep_block, + susp_sched_resume_block, + NULL); } - } - else if (schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) - schdlr_sspnd.changing = 0; - else { - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - ASSERT(!schdlr_sspnd.changing); + ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); @@ -3089,11 +3637,7 @@ erts_start_schedulers(void) ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(actual); actual++; ASSERT(actual == esdp->no); -#ifdef ERTS_ENABLE_LOCK_COUNT - res = erts_lcnt_thr_create(&esdp->tid,sched_thread_func,(void*)esdp,&opts); -#else res = ethr_thr_create(&esdp->tid,sched_thread_func,(void*)esdp,&opts); -#endif if (res != 0) { actual--; break; @@ -3407,6 +3951,7 @@ processor_order_cmp(const void *vx, const void *vy) static void check_cpu_bind(ErtsSchedulerData *esdp) { + int rg = 0; int res; int cpu_id; erts_smp_runq_unlock(esdp->run_queue); @@ -3425,19 +3970,25 @@ check_cpu_bind(ErtsSchedulerData *esdp) goto unbind; } } - else if (cpu_id < 0 && scheduler2cpu_map[esdp->no].bound_id >= 0) { + else if (cpu_id < 0) { unbind: /* Get rid of old binding */ res = erts_unbind_from_cpu(erts_cpuinfo); if (res == 0) esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1; - else { + else if (res != -ENOTSUP) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Scheduler %d failed to unbind from cpu %d: %s\n", (int) esdp->no, cpu_id, erl_errno_id(-res)); erts_send_error_to_logger_nogl(dsbufp); } } + if (erts_reader_groups) { + if (esdp->cpu_id >= 0) + rg = reader_group_lookup(esdp->cpu_id); + else + rg = (((int) esdp->no) - 1) % erts_reader_groups + 1; + } erts_smp_runq_lock(esdp->run_queue); #ifdef ERTS_SMP if (erts_common_run_queue) @@ -3448,6 +3999,8 @@ check_cpu_bind(ErtsSchedulerData *esdp) #endif erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); + if (erts_reader_groups) + erts_smp_rwmtx_set_reader_group(rg); } static void @@ -3476,11 +4029,13 @@ signal_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size) wake_all_schedulers(); } else { - ERTS_FOREACH_RUNQ(rq, - { + for (s_ix = 0; s_ix < erts_no_run_queues; s_ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(s_ix); + erts_smp_runq_lock(rq); rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; - wake_scheduler(rq, 0); - }); + erts_smp_runq_unlock(rq); + wake_scheduler(rq, 0, 1); + }; } #else check_cpu_bind(erts_get_scheduler_data()); @@ -3496,14 +4051,15 @@ erts_init_scheduler_bind_type(char *how) if (!system_cpudata && !user_cpudata) return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY; - if (sys_strcmp(how, "s") == 0) + if (sys_strcmp(how, "db") == 0) + cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND; + else if (sys_strcmp(how, "s") == 0) cpu_bind_order = ERTS_CPU_BIND_SPREAD; else if (sys_strcmp(how, "ps") == 0) cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; else if (sys_strcmp(how, "ts") == 0) cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; - else if (sys_strcmp(how, "db") == 0 - || sys_strcmp(how, "tnnps") == 0) + else if (sys_strcmp(how, "tnnps") == 0) cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; else if (sys_strcmp(how, "nnps") == 0) cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; @@ -3519,6 +4075,490 @@ erts_init_scheduler_bind_type(char *how) return ERTS_INIT_SCHED_BIND_TYPE_SUCCESS; } +/* + * reader groups map + */ + +typedef struct { + int level[ERTS_TOPOLOGY_MAX_DEPTH+1]; +} erts_avail_cput; + +typedef struct { + int *map; + int size; + int groups; +} erts_reader_groups_map_test; + +typedef struct { + int id; + int sub_levels; + int reader_groups; +} erts_rg_count_t; + +typedef struct { + int logical; + int reader_group; +} erts_reader_groups_map_t; + +typedef struct { + erts_reader_groups_map_t *map; + int map_size; + int logical_processors; + int groups; +} erts_make_reader_groups_map_test; + +static int reader_groups_available_cpu_check; +static int reader_groups_logical_processors; +static int reader_groups_map_size; +static erts_reader_groups_map_t *reader_groups_map; + +#define ERTS_TOPOLOGY_RG ERTS_TOPOLOGY_MAX_DEPTH + +static void +make_reader_groups_map(erts_make_reader_groups_map_test *test); + +static Eterm +get_reader_groups_map(Process *c_p, + erts_reader_groups_map_t *map, + int map_size, + int logical_processors) +{ +#ifdef DEBUG + Eterm *endp; +#endif + Eterm res = NIL, tuple; + Eterm *hp; + int i; + + hp = HAlloc(c_p, logical_processors*(2+3)); +#ifdef DEBUG + endp = hp + logical_processors*(2+3); +#endif + for (i = map_size - 1; i >= 0; i--) { + if (map[i].logical >= 0) { + tuple = TUPLE2(hp, + make_small(map[i].logical), + make_small(map[i].reader_group)); + hp += 3; + res = CONS(hp, tuple, res); + hp += 2; + } + } + ASSERT(hp == endp); + return res; +} + +Eterm +erts_debug_reader_groups_map(Process *c_p, int groups) +{ + Eterm res; + erts_make_reader_groups_map_test test; + + test.groups = groups; + make_reader_groups_map(&test); + if (!test.map) + res = NIL; + else { + res = get_reader_groups_map(c_p, + test.map, + test.map_size, + test.logical_processors); + erts_free(ERTS_ALC_T_TMP, test.map); + } + return res; +} + + +Eterm +erts_get_reader_groups_map(Process *c_p) +{ + Eterm res; + erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); + res = get_reader_groups_map(c_p, + reader_groups_map, + reader_groups_map_size, + reader_groups_logical_processors); + erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); + return res; +} + +static void +make_available_cpu_topology(erts_avail_cput *no, + erts_avail_cput *avail, + erts_cpu_topology_t *cpudata, + int *size, + int test) +{ + int len = *size; + erts_cpu_topology_t last; + int a, i, j; + + no->level[ERTS_TOPOLOGY_NODE] = -1; + no->level[ERTS_TOPOLOGY_PROCESSOR] = -1; + no->level[ERTS_TOPOLOGY_PROCESSOR_NODE] = -1; + no->level[ERTS_TOPOLOGY_CORE] = -1; + no->level[ERTS_TOPOLOGY_THREAD] = -1; + no->level[ERTS_TOPOLOGY_LOGICAL] = -1; + + last.node = INT_MIN; + last.processor = INT_MIN; + last.processor_node = INT_MIN; + last.core = INT_MIN; + last.thread = INT_MIN; + last.logical = INT_MIN; + + a = 0; + + for (i = 0; i < len; i++) { + + if (!test && !erts_is_cpu_available(erts_cpuinfo, cpudata[i].logical)) + continue; + + if (last.node != cpudata[i].node) + goto node; + if (last.processor != cpudata[i].processor) + goto processor; + if (last.processor_node != cpudata[i].processor_node) + goto processor_node; + if (last.core != cpudata[i].core) + goto core; + ASSERT(last.thread != cpudata[i].thread); + goto thread; + + node: + no->level[ERTS_TOPOLOGY_NODE]++; + processor: + no->level[ERTS_TOPOLOGY_PROCESSOR]++; + processor_node: + no->level[ERTS_TOPOLOGY_PROCESSOR_NODE]++; + core: + no->level[ERTS_TOPOLOGY_CORE]++; + thread: + no->level[ERTS_TOPOLOGY_THREAD]++; + + no->level[ERTS_TOPOLOGY_LOGICAL]++; + + for (j = 0; j < ERTS_TOPOLOGY_LOGICAL; j++) + avail[a].level[j] = no->level[j]; + + avail[a].level[ERTS_TOPOLOGY_LOGICAL] = cpudata[i].logical; + avail[a].level[ERTS_TOPOLOGY_RG] = 0; + + ASSERT(last.logical != cpudata[a].logical); + + last = cpudata[i]; + a++; + } + + no->level[ERTS_TOPOLOGY_NODE]++; + no->level[ERTS_TOPOLOGY_PROCESSOR]++; + no->level[ERTS_TOPOLOGY_PROCESSOR_NODE]++; + no->level[ERTS_TOPOLOGY_CORE]++; + no->level[ERTS_TOPOLOGY_THREAD]++; + no->level[ERTS_TOPOLOGY_LOGICAL]++; + + *size = a; +} + +static int +reader_group_lookup(int logical) +{ + int start = logical % reader_groups_map_size; + int ix = start; + + do { + if (reader_groups_map[ix].logical == logical) { + ASSERT(reader_groups_map[ix].reader_group > 0); + return reader_groups_map[ix].reader_group; + } + ix++; + if (ix == reader_groups_map_size) + ix = 0; + } while (ix != start); + + erl_exit(ERTS_ABORT_EXIT, "Logical cpu id %d not found\n", logical); +} + +static void +reader_group_insert(erts_reader_groups_map_t *map, int map_size, + int logical, int reader_group) +{ + int start = logical % map_size; + int ix = start; + + do { + if (map[ix].logical < 0) { + map[ix].logical = logical; + map[ix].reader_group = reader_group; + return; + } + ix++; + if (ix == map_size) + ix = 0; + } while (ix != start); + + erl_exit(ERTS_ABORT_EXIT, "Reader groups map full\n"); +} + + +static int +sub_levels(erts_rg_count_t *rgc, int level, int aix, int avail_sz, erts_avail_cput *avail) +{ + int sub_level = level+1; + int last = -1; + rgc->sub_levels = 0; + + do { + if (last != avail[aix].level[sub_level]) { + rgc->sub_levels++; + last = avail[aix].level[sub_level]; + } + aix++; + } + while (aix < avail_sz && rgc->id == avail[aix].level[level]); + rgc->reader_groups = 0; + return aix; +} + +static int +write_reader_groups(int *rgp, erts_rg_count_t *rgcp, + int level, int a, + int avail_sz, erts_avail_cput *avail) +{ + int rg = *rgp; + int sub_level = level+1; + int sl_per_gr = rgcp->sub_levels / rgcp->reader_groups; + int xsl = rgcp->sub_levels % rgcp->reader_groups; + int sls = 0; + int last = -1; + int xsl_rg_lim = (rgcp->reader_groups - xsl) + rg + 1; + + ASSERT(level < 0 || avail[a].level[level] == rgcp->id) + + do { + if (last != avail[a].level[sub_level]) { + if (!sls) { + sls = sl_per_gr; + rg++; + if (rg >= xsl_rg_lim) + sls++; + } + last = avail[a].level[sub_level]; + sls--; + } + avail[a].level[ERTS_TOPOLOGY_RG] = rg; + a++; + } while (a < avail_sz && (level < 0 + || avail[a].level[level] == rgcp->id)); + + ASSERT(rgcp->reader_groups == rg - *rgp); + + *rgp = rg; + + return a; +} + +static int +rg_count_sub_levels_compare(const void *vx, const void *vy) +{ + erts_rg_count_t *x = (erts_rg_count_t *) vx; + erts_rg_count_t *y = (erts_rg_count_t *) vy; + if (x->sub_levels != y->sub_levels) + return y->sub_levels - x->sub_levels; + return x->id - y->id; +} + +static int +rg_count_id_compare(const void *vx, const void *vy) +{ + erts_rg_count_t *x = (erts_rg_count_t *) vx; + erts_rg_count_t *y = (erts_rg_count_t *) vy; + return x->id - y->id; +} + +static void +make_reader_groups_map(erts_make_reader_groups_map_test *test) +{ + int i, spread_level, avail_sz; + erts_avail_cput no, *avail; + erts_cpu_topology_t *cpudata; + erts_reader_groups_map_t *map; + int map_sz; + int groups = erts_reader_groups; + + if (test) { + test->map = NULL; + test->map_size = 0; + groups = test->groups; + } + + if (!groups) + return; + + if (!test) { + if (reader_groups_map) + erts_free(ERTS_ALC_T_RDR_GRPS_MAP, reader_groups_map); + + reader_groups_logical_processors = 0; + reader_groups_map_size = 0; + reader_groups_map = NULL; + } + + create_tmp_cpu_topology_copy(&cpudata, &avail_sz); + + if (!cpudata) + return; + + cpu_bind_order_sort(cpudata, + avail_sz, + ERTS_CPU_BIND_NO_SPREAD, + 1); + + avail = erts_alloc(ERTS_ALC_T_TMP, + sizeof(erts_avail_cput)*avail_sz); + + make_available_cpu_topology(&no, avail, cpudata, + &avail_sz, test != NULL); + + destroy_tmp_cpu_topology_copy(cpudata); + + map_sz = avail_sz*2+1; + + if (test) { + map = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_reader_groups_map_t) + * map_sz)); + test->map = map; + test->map_size = map_sz; + test->logical_processors = avail_sz; + } + else { + map = erts_alloc(ERTS_ALC_T_RDR_GRPS_MAP, + (sizeof(erts_reader_groups_map_t) + * map_sz)); + reader_groups_map = map; + reader_groups_logical_processors = avail_sz; + reader_groups_map_size = map_sz; + + } + + for (i = 0; i < map_sz; i++) { + map[i].logical = -1; + map[i].reader_group = 0; + } + + spread_level = ERTS_TOPOLOGY_CORE; + for (i = ERTS_TOPOLOGY_NODE; i < ERTS_TOPOLOGY_THREAD; i++) { + if (no.level[i] > groups) { + spread_level = i; + break; + } + } + + if (no.level[spread_level] <= groups) { + int a, rg, last = -1; + rg = 0; + ASSERT(spread_level == ERTS_TOPOLOGY_CORE); + for (a = 0; a < avail_sz; a++) { + if (last != avail[a].level[spread_level]) { + rg++; + last = avail[a].level[spread_level]; + } + reader_group_insert(map, + map_sz, + avail[a].level[ERTS_TOPOLOGY_LOGICAL], + rg); + } + } + else { /* groups < no.level[spread_level] */ + erts_rg_count_t *rg_count; + int a, rg, tl, toplevels; + + tl = spread_level-1; + + if (spread_level == ERTS_TOPOLOGY_NODE) + toplevels = 1; + else + toplevels = no.level[tl]; + + rg_count = erts_alloc(ERTS_ALC_T_TMP, + toplevels*sizeof(erts_rg_count_t)); + + if (toplevels == 1) { + rg_count[0].id = 0; + rg_count[0].sub_levels = no.level[spread_level]; + rg_count[0].reader_groups = groups; + } + else { + int rgs_per_tl, rgs; + rgs = groups; + rgs_per_tl = rgs / toplevels; + + a = 0; + for (i = 0; i < toplevels; i++) { + rg_count[i].id = avail[a].level[tl]; + a = sub_levels(&rg_count[i], tl, a, avail_sz, avail); + } + + qsort(rg_count, + toplevels, + sizeof(erts_rg_count_t), + rg_count_sub_levels_compare); + + for (i = 0; i < toplevels; i++) { + if (rg_count[i].sub_levels < rgs_per_tl) { + rg_count[i].reader_groups = rg_count[i].sub_levels; + rgs -= rg_count[i].sub_levels; + } + else { + rg_count[i].reader_groups = rgs_per_tl; + rgs -= rgs_per_tl; + } + } + + while (rgs > 0) { + for (i = 0; i < toplevels; i++) { + if (rg_count[i].sub_levels == rg_count[i].reader_groups) + break; + else { + rg_count[i].reader_groups++; + if (--rgs == 0) + break; + } + } + } + + qsort(rg_count, + toplevels, + sizeof(erts_rg_count_t), + rg_count_id_compare); + } + + a = i = rg = 0; + while (a < avail_sz) { + a = write_reader_groups(&rg, &rg_count[i], tl, + a, avail_sz, avail); + i++; + } + + ASSERT(groups == rg); + + for (a = 0; a < avail_sz; a++) + reader_group_insert(map, + map_sz, + avail[a].level[ERTS_TOPOLOGY_LOGICAL], + avail[a].level[ERTS_TOPOLOGY_RG]); + + erts_free(ERTS_ALC_T_TMP, rg_count); + } + + erts_free(ERTS_ALC_T_TMP, avail); +} + +/* + * CPU topology + */ + typedef struct { int *id; int used; @@ -3765,11 +4805,11 @@ verify_topology(erts_cpu_topology_t *cpudata, int size) /* Verify logical ids */ logical = erts_alloc(ERTS_ALC_T_TMP, sizeof(int)*size); - for (i = 0; i < user_cpudata_size; i++) - logical[i] = user_cpudata[i].logical; + for (i = 0; i < size; i++) + logical[i] = cpudata[i].logical; - qsort(logical, user_cpudata_size, sizeof(int), int_cmp); - for (i = 0; i < user_cpudata_size-1; i++) { + qsort(logical, size, sizeof(int), int_cmp); + for (i = 0; i < size-1; i++) { if (logical[i] == logical[i+1]) { erts_free(ERTS_ALC_T_TMP, logical); return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS; @@ -3782,13 +4822,13 @@ verify_topology(erts_cpu_topology_t *cpudata, int size) /* Verify unique entities */ - for (i = 1; i < user_cpudata_size; i++) { - if (user_cpudata[i-1].processor == user_cpudata[i].processor - && user_cpudata[i-1].node == user_cpudata[i].node - && (user_cpudata[i-1].processor_node - == user_cpudata[i].processor_node) - && user_cpudata[i-1].core == user_cpudata[i].core - && user_cpudata[i-1].thread == user_cpudata[i].thread) { + for (i = 1; i < size; i++) { + if (cpudata[i-1].processor == cpudata[i].processor + && cpudata[i-1].node == cpudata[i].node + && (cpudata[i-1].processor_node + == cpudata[i].processor_node) + && cpudata[i-1].core == cpudata[i].core + && cpudata[i-1].thread == cpudata[i].thread) { return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES; } } @@ -4032,6 +5072,8 @@ erts_set_cpu_topology(Process *c_p, Eterm term) sizeof(erts_cpu_topology_t)*cpudata_size); } + make_reader_groups_map(NULL); + signal_schedulers_bind_change(cpudata, cpudata_size); done: @@ -4146,14 +5188,15 @@ erts_bind_schedulers(Process *c_p, Eterm how) old_cpu_bind_order = cpu_bind_order; - if (ERTS_IS_ATOM_STR("spread", how)) + if (ERTS_IS_ATOM_STR("default_bind", how)) + cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND; + else if (ERTS_IS_ATOM_STR("spread", how)) cpu_bind_order = ERTS_CPU_BIND_SPREAD; else if (ERTS_IS_ATOM_STR("processor_spread", how)) cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; else if (ERTS_IS_ATOM_STR("thread_spread", how)) cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; - else if (ERTS_IS_ATOM_STR("default_bind", how) - || ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how)) + else if (ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how)) cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how)) cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; @@ -4199,14 +5242,15 @@ erts_fake_scheduler_bindings(Process *p, Eterm how) int cpudata_size; Eterm res; - if (ERTS_IS_ATOM_STR("spread", how)) + if (ERTS_IS_ATOM_STR("default_bind", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND; + else if (ERTS_IS_ATOM_STR("spread", how)) fake_cpu_bind_order = ERTS_CPU_BIND_SPREAD; else if (ERTS_IS_ATOM_STR("processor_spread", how)) fake_cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; else if (ERTS_IS_ATOM_STR("thread_spread", how)) fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; - else if (ERTS_IS_ATOM_STR("default_bind", how) - || ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how)) + else if (ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how)) fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how)) fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; @@ -4431,7 +5475,12 @@ early_cpu_bind_init(void) (sizeof(erts_cpu_topology_t) * system_cpudata_size)); - cpu_bind_order = ERTS_CPU_BIND_NONE; + cpu_bind_order = ERTS_CPU_BIND_UNDEFINED; + + reader_groups_available_cpu_check = 1; + reader_groups_logical_processors = 0; + reader_groups_map_size = 0; + reader_groups_map = NULL; if (!erts_get_cpu_topology(erts_cpuinfo, system_cpudata) || ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(system_cpudata, @@ -4457,6 +5506,19 @@ late_cpu_bind_init(void) scheduler2cpu_map[ix].bound_id = -1; } + if (cpu_bind_order == ERTS_CPU_BIND_UNDEFINED) { + int ncpus = erts_get_cpu_configured(erts_cpuinfo); + if (ncpus < 1 || erts_no_schedulers < ncpus) + cpu_bind_order = ERTS_CPU_BIND_NONE; + else + cpu_bind_order = ((system_cpudata || user_cpudata) + && (erts_bind_to_cpu(erts_cpuinfo, -1) != -ENOTSUP) + ? ERTS_CPU_BIND_DEFAULT_BIND + : ERTS_CPU_BIND_NONE); + } + + make_reader_groups_map(NULL); + if (cpu_bind_order != ERTS_CPU_BIND_NONE) { erts_cpu_topology_t *cpudata; int cpudata_size; @@ -4467,6 +5529,39 @@ late_cpu_bind_init(void) } } +int +erts_update_cpu_info(void) +{ + int changed; + erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx); + changed = erts_cpu_info_update(erts_cpuinfo); + if (changed) { + erts_cpu_topology_t *cpudata; + int cpudata_size; + erts_free(ERTS_ALC_T_CPUDATA, system_cpudata); + + system_cpudata_size = erts_get_cpu_topology_size(erts_cpuinfo); + system_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, + (sizeof(erts_cpu_topology_t) + * system_cpudata_size)); + + if (!erts_get_cpu_topology(erts_cpuinfo, system_cpudata) + || ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(system_cpudata, + system_cpudata_size)) { + erts_free(ERTS_ALC_T_CPUDATA, system_cpudata); + system_cpudata = NULL; + system_cpudata_size = 0; + } + + create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); + ASSERT(cpudata); + signal_schedulers_bind_change(cpudata, cpudata_size); + destroy_tmp_cpu_topology_copy(cpudata); + } + erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); + return changed; +} + #ifdef ERTS_SMP static void @@ -4481,7 +5576,7 @@ add_pend_suspend(Process *suspendee, sizeof(ErtsPendingSuspend)); psp->next = NULL; #ifdef DEBUG -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP psp->end = (ErtsPendingSuspend *) 0xdeaddeaddeaddead; #else psp->end = (ErtsPendingSuspend *) 0xdeaddead; @@ -5322,7 +6417,7 @@ dequeue_process(ErtsRunQueue *runq, Process *p) } /* schedule a process */ -static ERTS_INLINE void +static ERTS_INLINE ErtsRunQueue * internal_add_to_runq(ErtsRunQueue *runq, Process *p) { Uint32 prev_status = p->status; @@ -5333,12 +6428,12 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p) ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); if (p->status_flags & ERTS_PROC_SFLG_INRUNQ) - return; + return NULL; else if (p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) { ASSERT(p->status != P_SUSPENDED); ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p); p->status_flags |= ERTS_PROC_SFLG_PENDADD2SCHEDQ; - return; + return NULL; } ASSERT(!p->scheduler_data); #endif @@ -5377,20 +6472,23 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p) profile_runnable_proc(p, am_active); } - smp_notify_inc_runq(add_runq); - if (add_runq != runq) erts_smp_runq_unlock(add_runq); + + return add_runq; } void erts_add_to_runq(Process *p) { + ErtsRunQueue *notify_runq; ErtsRunQueue *runq = erts_get_runq_proc(p); erts_smp_runq_lock(runq); - internal_add_to_runq(runq, p); + notify_runq = internal_add_to_runq(runq, p); erts_smp_runq_unlock(runq); + smp_notify_inc_runq(notify_runq); + } /* Possibly remove a scheduled process we need to suspend */ @@ -5529,8 +6627,6 @@ erts_proc_migrate(Process *p, ErtsProcLocks *plcks, p->run_queue = to_rq; enqueue_process(to_rq, p); - smp_notify_inc_runq(to_rq); - return ERTS_MIGRATE_SUCCESS; } #endif /* ERTS_SMP */ @@ -5727,30 +6823,6 @@ erts_set_process_priority(Process *p, Eterm new_value) return old_value; } -#ifdef ERTS_SMP - -static ERTS_INLINE int -prepare_for_sys_schedule(void) -{ - while (!erts_port_task_have_outstanding_io_tasks() - && !erts_smp_atomic_xchg(&doing_sys_schedule, 1)) { - if (!erts_port_task_have_outstanding_io_tasks()) - return 1; - erts_smp_atomic_set(&doing_sys_schedule, 0); - } - return 0; -} - -#else - -static ERTS_INLINE int -prepare_for_sys_schedule(void) -{ - return !erts_port_task_have_outstanding_io_tasks(); -} - -#endif - /* note that P_RUNNING is only set so that we don't try to remove ** running processes from the schedule queue if they exit - a running ** process not being in the schedule queue!! @@ -5839,6 +6911,9 @@ Process *schedule(Process *p, int calls) } if (IS_TRACED(p)) { + if (IS_TRACED_FL(p, F_TRACE_CALLS) && p->status != P_FREE) { + erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_OUT); + } switch (p->status) { case P_EXITING: if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT)) @@ -5882,8 +6957,11 @@ Process *schedule(Process *p, int calls) p->status_flags &= ~ERTS_PROC_SFLG_RUNNING; if (p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) { + ErtsRunQueue *notify_runq; p->status_flags &= ~ERTS_PROC_SFLG_PENDADD2SCHEDQ; - internal_add_to_runq(rq, p); + notify_runq = internal_add_to_runq(rq, p); + if (notify_runq != rq) + smp_notify_inc_runq(notify_runq); } #endif @@ -5951,7 +7029,10 @@ Process *schedule(Process *p, int calls) | ERTS_RUNQ_FLG_CHK_CPU_BIND | ERTS_RUNQ_FLG_SUSPENDED)) { if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || erts_smp_atomic_read(&esdp->suspended)) { + || (erts_smp_atomic_read(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED)) { + ASSERT(erts_smp_atomic_read(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED); suspend_scheduler(esdp); } if ((rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) @@ -5960,12 +7041,21 @@ Process *schedule(Process *p, int calls) } } -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - if (esdp->check_children) { - esdp->check_children = 0; - erts_smp_runq_unlock(rq); - erts_check_children(); - erts_smp_runq_lock(rq); +#if defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) \ + || defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) + { + ErtsSchedulerSleepInfo *ssi = esdp->ssi; + long aux_work = erts_smp_atomic_read(&ssi->aux_work); + if (aux_work) { + erts_smp_runq_unlock(rq); +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = blockable_aux_work(esdp, ssi, aux_work); +#endif +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK + nonblockable_aux_work(esdp, ssi, aux_work); +#endif + erts_smp_runq_lock(rq); + } } #endif @@ -5997,7 +7087,10 @@ Process *schedule(Process *p, int calls) if (rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ | ERTS_RUNQ_FLG_SUSPENDED)) { if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || erts_smp_atomic_read(&esdp->suspended)) { + || (erts_smp_atomic_read(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED)) { + ASSERT(erts_smp_atomic_read(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED); non_empty_runq(rq); goto continue_check_activities_to_run; } @@ -6014,17 +7107,7 @@ Process *schedule(Process *p, int calls) } } - if (prepare_for_sys_schedule()) { - erts_smp_atomic_set(&function_calls, 0); - fcalls = 0; - sched_sys_wait(esdp->no, rq); - erts_smp_atomic_set(&doing_sys_schedule, 0); - } - else { - /* If all schedulers are waiting, one of them *should* - be waiting in erl_sys_schedule() */ - sched_cnd_wait(esdp->no, rq); - } + scheduler_wait(&fcalls, esdp, rq); non_empty_runq(rq); @@ -6086,7 +7169,7 @@ Process *schedule(Process *p, int calls) else { if (erts_common_run_queue) { if (erts_common_run_queue->waiting) - wake_one_scheduler(); + wake_scheduler(erts_common_run_queue, 0, 1); } else if (erts_smp_atomic_read(&no_empty_run_queues) != 0) { wake_scheduler_on_empty_runq(rq); @@ -6231,10 +7314,10 @@ Process *schedule(Process *p, int calls) erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); if (erts_sched_stat.enabled) { - Uint old = ERTS_PROC_SCHED_ID(p, + UWord old = ERTS_PROC_SCHED_ID(p, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_STATUS), - esdp->no); + (UWord) esdp->no); int migrated = old && old != esdp->no; erts_smp_spin_lock(&erts_sched_stat.lock); @@ -6275,7 +7358,11 @@ Process *schedule(Process *p, int calls) trace_virtual_sched(p, am_in); break; } + if (IS_TRACED_FL(p, F_TRACE_CALLS)) { + erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_IN); + } } + if (p->status != P_EXITING) p->status = P_RUNNING; @@ -6397,8 +7484,8 @@ erts_schedule_misc_op(void (*func)(void *), void *arg) else rq->misc.start = molp; rq->misc.end = molp; - smp_notify_inc_runq(rq); erts_smp_runq_unlock(rq); + smp_notify_inc_runq(rq); } static void @@ -6640,7 +7727,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). Eterm args, /* Arguments for function (must be well-formed list). */ ErlSpawnOpts* so) /* Options for spawn. */ { - ErtsRunQueue *rq; + ErtsRunQueue *rq, *notify_runq; Process *p; Sint arity; /* Number of arguments. */ #ifndef HYBRID @@ -6719,11 +7806,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). /* * Must initialize binary lists here before copying binaries to process. */ - p->off_heap.mso = NULL; -#ifndef HYBRID /* FIND ME! */ - p->off_heap.funs = NULL; -#endif - p->off_heap.externals = NULL; + p->off_heap.first = NULL; p->off_heap.overhead = 0; heap_need += @@ -6757,13 +7840,14 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->bin_vheap_sz = p->min_vheap_size; p->bin_old_vheap_sz = p->min_vheap_size; p->bin_old_vheap = 0; + p->bin_vheap_mature = 0; /* No need to initialize p->fcalls. */ p->current = p->initial+INITIAL_MOD; - p->i = (Eterm *) beam_apply; - p->cp = (Eterm *) beam_apply+1; + p->i = (BeamInstr *) beam_apply; + p->cp = (BeamInstr *) beam_apply+1; p->arg_reg = p->def_arg_reg; p->max_arg_reg = sizeof(p->def_arg_reg)/sizeof(p->def_arg_reg[0]); @@ -6813,7 +7897,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->group_leader = IS_CONST(parent->group_leader) ? parent->group_leader - : STORE_NC(&p->htop, &p->off_heap.externals, parent->group_leader); + : STORE_NC(&p->htop, &p->off_heap, parent->group_leader); } erts_get_default_tracing(&p->trace_flags, &p->tracer_proc); @@ -6957,10 +8041,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #endif p->status = P_WAITING; - internal_add_to_runq(rq, p); + notify_runq = internal_add_to_runq(rq, p); erts_smp_runq_unlock(rq); + smp_notify_inc_runq(notify_runq); + res = p->id; erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); @@ -7007,6 +8093,7 @@ void erts_init_empty_process(Process *p) p->bin_vheap_sz = BIN_VH_MIN_SIZE; p->bin_old_vheap_sz = BIN_VH_MIN_SIZE; p->bin_old_vheap = 0; + p->bin_vheap_mature = 0; #ifdef ERTS_SMP p->u.ptimer = NULL; p->bound_runq = NULL; @@ -7014,11 +8101,7 @@ void erts_init_empty_process(Process *p) memset(&(p->u.tm), 0, sizeof(ErlTimer)); #endif p->next = NULL; - p->off_heap.mso = NULL; -#ifndef HYBRID /* FIND ME! */ - p->off_heap.funs = NULL; -#endif - p->off_heap.externals = NULL; + p->off_heap.first = NULL; p->off_heap.overhead = 0; p->reg = NULL; p->heap_sz = 0; @@ -7165,11 +8248,7 @@ erts_debug_verify_clean_empty_process(Process* p) /* Thing that erts_cleanup_empty_process() cleans up */ - ASSERT(p->off_heap.mso == NULL); -#ifndef HYBRID /* FIND ME! */ - ASSERT(p->off_heap.funs == NULL); -#endif - ASSERT(p->off_heap.externals == NULL); + ASSERT(p->off_heap.first == NULL); ASSERT(p->off_heap.overhead == 0); ASSERT(p->mbuf == NULL); @@ -7180,25 +8259,16 @@ erts_debug_verify_clean_empty_process(Process* p) void erts_cleanup_empty_process(Process* p) { - ErlHeapFragment* mbufp; - /* We only check fields that are known to be used... */ erts_cleanup_offheap(&p->off_heap); - p->off_heap.mso = NULL; -#ifndef HYBRID /* FIND ME! */ - p->off_heap.funs = NULL; -#endif - p->off_heap.externals = NULL; + p->off_heap.first = NULL; p->off_heap.overhead = 0; - mbufp = p->mbuf; - while (mbufp) { - ErlHeapFragment *next = mbufp->next; - free_message_buffer(mbufp); - mbufp = next; + if (p->mbuf != NULL) { + free_message_buffer(p->mbuf); + p->mbuf = NULL; } - p->mbuf = NULL; #if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) erts_lcnt_proc_lock_destroy(p); #endif @@ -7214,7 +8284,6 @@ static void delete_process(Process* p) { ErlMessage* mp; - ErlHeapFragment* bp; VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->id)); @@ -7230,7 +8299,7 @@ delete_process(Process* p) * The mso list should not be used anymore, but if it is, make sure that * we'll notice. */ - p->off_heap.mso = (void *) 0x8DEFFACD; + p->off_heap.first = (void *) 0x8DEFFACD; if (p->arg_reg != p->def_arg_reg) { erts_free(ERTS_ALC_T_ARG_REG, p->arg_reg); @@ -7264,11 +8333,8 @@ delete_process(Process* p) /* * Free all pending message buffers. */ - bp = p->mbuf; - while (bp != NULL) { - ErlHeapFragment* next_bp = bp->next; - free_message_buffer(bp); - bp = next_bp; + if (p->mbuf != NULL) { + free_message_buffer(p->mbuf); } erts_erase_dicts(p); @@ -7348,7 +8414,7 @@ set_proc_exiting(Process *p, Eterm reason, ErlHeapFragment *bp) p->freason = EXTAG_EXIT; KILL_CATCHES(p); cancel_timer(p); - p->i = (Eterm *) beam_exit; + p->i = (BeamInstr *) beam_exit; } @@ -7778,9 +8844,10 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) erts_port_release(prt); } else if (is_internal_pid(mon->pid)) {/* local by name or pid */ Eterm watched; - Eterm lhp[3]; + DeclareTmpHeapNoproc(lhp,3); ErtsProcLocks rp_locks = (ERTS_PROC_LOCK_LINK | ERTS_PROC_LOCKS_MSG_SEND); + UseTmpHeapNoproc(3); rp = erts_pid2proc(NULL, 0, mon->pid, rp_locks); if (rp == NULL) { goto done; @@ -7795,6 +8862,7 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) erts_queue_monitor_message(rp, &rp_locks, mon->ref, am_process, watched, pcontext->reason); } + UnUseTmpHeapNoproc(3); /* else: demonitor while we exited, i.e. do nothing... */ erts_smp_proc_unlock(rp, rp_locks); } else { /* external by pid or name */ @@ -8025,8 +9093,13 @@ erts_do_exit_process(Process* p, Eterm reason) ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); #endif - if (IS_TRACED_FL(p,F_TRACE_PROCS)) - trace_proc(p, p, am_exit, reason); + if (IS_TRACED(p)) { + if (IS_TRACED_FL(p, F_TRACE_CALLS)) + erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_EXITING); + + if (IS_TRACED_FL(p,F_TRACE_PROCS)) + trace_proc(p, p, am_exit, reason); + } erts_trace_check_exiting(p->id); @@ -8075,6 +9148,8 @@ continue_exit_process(Process *p Eterm reason = p->fvalue; DistEntry *dep; struct saved_calls *scb; + process_breakpoint_time_t *pbt; + #ifdef DEBUG int yield_allowed = 1; #endif @@ -8214,6 +9289,7 @@ continue_exit_process(Process *p ? ERTS_PROC_SET_DIST_ENTRY(p, ERTS_PROC_LOCKS_ALL, NULL) : NULL); scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, ERTS_PROC_LOCKS_ALL, NULL); + pbt = ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCKS_ALL, NULL); erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); processes_busy--; @@ -8228,11 +9304,12 @@ continue_exit_process(Process *p * Pre-build the EXIT tuple if there are any links. */ if (lnk) { - Eterm tmp_heap[4]; + DeclareTmpHeap(tmp_heap,4,p); Eterm exit_tuple; Uint exit_tuple_sz; Eterm* hp; + UseTmpHeap(4,p); hp = &tmp_heap[0]; exit_tuple = TUPLE3(hp, am_EXIT, p->id, reason); @@ -8243,16 +9320,21 @@ continue_exit_process(Process *p ExitLinkContext context = {p, reason, exit_tuple, exit_tuple_sz}; erts_sweep_links(lnk, &doit_exit_link, &context); } + UnUseTmpHeap(4,p); } { ExitMonitorContext context = {reason, p}; - erts_sweep_monitors(mon,&doit_exit_monitor,&context); + erts_sweep_monitors(mon,&doit_exit_monitor,&context); /* Allocates TmpHeap, but we + have none here */ } if (scb) erts_free(ERTS_ALC_T_CALLS_BUF, (void *) scb); + if (pbt) + erts_free(ERTS_ALC_T_BPD, (void *) pbt); + delete_process(p); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); @@ -8271,7 +9353,7 @@ continue_exit_process(Process *p ASSERT(p->status == P_EXITING); - p->i = (Eterm *) beam_continue_exit; + p->i = (BeamInstr *) beam_continue_exit; if (!(curr_locks & ERTS_PROC_LOCK_STATUS)) { erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); @@ -8291,7 +9373,7 @@ continue_exit_process(Process *p static void timeout_proc(Process* p) { - p->i = (Eterm *) p->def_arg_reg[0]; + p->i = *((BeamInstr **) (UWord) p->def_arg_reg); p->flags |= F_TIMO; p->flags &= ~F_INSLPQUEUE; @@ -8390,9 +9472,9 @@ erts_program_counter_info(int to, void *to_arg, Process *p) } static void -print_function_from_pc(int to, void *to_arg, Eterm* x) +print_function_from_pc(int to, void *to_arg, BeamInstr* x) { - Eterm* addr = find_function_from_pc(x); + BeamInstr* addr = find_function_from_pc(x); if (addr == NULL) { if (x == beam_exit) { erts_print(to, to_arg, "<terminate process>"); @@ -8426,7 +9508,7 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) } if (is_CP(x)) { - erts_print(to, to_arg, "Return addr %p (", (Eterm *) x); + erts_print(to, to_arg, "Return addr %p (", (Eterm *) EXPAND_POINTER(x)); print_function_from_pc(to, to_arg, cp_val(x)); erts_print(to, to_arg, ")\n"); yreg = 0; @@ -9255,8 +10337,8 @@ init_processes_bif(void) processes_trap_export.code[0] = am_erlang; processes_trap_export.code[1] = am_processes_trap; processes_trap_export.code[2] = 2; - processes_trap_export.code[3] = (Eterm) em_apply_bif; - processes_trap_export.code[4] = (Eterm) &processes_trap; + processes_trap_export.code[3] = (BeamInstr) em_apply_bif; + processes_trap_export.code[4] = (BeamInstr) &processes_trap; #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST erts_get_emu_time(&debug_tv_start); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index f58b6932b3..e49710a7ed 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -89,6 +89,7 @@ extern int erts_sched_thread_suggested_stack_size; #define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192 /* Kilo words */ #ifdef ERTS_SMP +extern Uint erts_max_main_threads; #include "erl_bits.h" #endif @@ -179,7 +180,7 @@ extern int erts_sched_thread_suggested_stack_size; #ifdef DEBUG -# ifdef ARCH_64 +# if defined(ARCH_64) && !HALFWORD_HEAP # define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \ (*((char **) &(RQP)) = (char *) (0xdeadbeefdead0003 | ((N) << 4))) # define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ @@ -194,8 +195,8 @@ do { \ # define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ do { \ ASSERT((RQP) != NULL); \ - ASSERT(((((Uint) (RQP)) & ((Uint) 1))) == ((Uint) 0)); \ - ASSERT((((Uint) (RQP)) & ~((Uint) 0xffff)) != ((Uint) 0xdead0000)); \ + ASSERT(((((UWord) (RQP)) & ((UWord) 1))) == ((UWord) 0)); \ + ASSERT((((UWord) (RQP)) & ~((UWord) 0xffff)) != ((UWord) 0xdead0000)); \ } while (0) # endif #else @@ -219,6 +220,51 @@ typedef enum { ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED } ErtsMigrateResult; +#define ERTS_SSI_FLG_SLEEPING (((long) 1) << 0) +#define ERTS_SSI_FLG_POLL_SLEEPING (((long) 1) << 1) +#define ERTS_SSI_FLG_TSE_SLEEPING (((long) 1) << 2) +#define ERTS_SSI_FLG_WAITING (((long) 1) << 3) +#define ERTS_SSI_FLG_SUSPENDED (((long) 1) << 4) + +#define ERTS_SSI_FLGS_SLEEP_TYPE \ + (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING) + +#define ERTS_SSI_FLGS_SLEEP \ + (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLGS_SLEEP_TYPE) + +#define ERTS_SSI_FLGS_ALL \ + (ERTS_SSI_FLGS_SLEEP \ + | ERTS_SSI_FLG_WAITING \ + | ERTS_SSI_FLG_SUSPENDED) + + +#if !defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) \ + && defined(ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN) +#define ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK +#endif + +#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((long) 1) << 0) + +#define ERTS_SSI_BLOCKABLE_AUX_WORK_MASK \ + (ERTS_SSI_AUX_WORK_CHECK_CHILDREN) +#define ERTS_SSI_NONBLOCKABLE_AUX_WORK_MASK \ + (0) + +typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo; + +typedef struct { + erts_smp_spinlock_t lock; + ErtsSchedulerSleepInfo *list; +} ErtsSchedulerSleepList; + +struct ErtsSchedulerSleepInfo_ { + ErtsSchedulerSleepInfo *next; + ErtsSchedulerSleepInfo *prev; + erts_smp_atomic_t flags; + erts_tse_t *event; + erts_smp_atomic_t aux_work; +}; + /* times to reschedule low prio process before running */ #define RESCHEDULE_LOW 8 @@ -271,8 +317,9 @@ struct ErtsRunQueue_ { erts_smp_mtx_t mtx; erts_smp_cnd_t cnd; - erts_smp_atomic_t spin_waiter; - erts_smp_atomic_t spin_wake; +#ifdef ERTS_SMP + ErtsSchedulerSleepList sleepers; +#endif ErtsSchedulerData *scheduler; int waiting; /* < 0 in sys schedule; > 0 on cnd variable */ @@ -344,13 +391,25 @@ struct ErtsSchedulerData_ { * numbered registers as possible in the same cache * line). */ +#if !HALFWORD_HEAP Eterm save_reg[ERTS_X_REGS_ALLOCATED]; /* X registers */ +#else + Eterm *save_reg; +#endif FloatDef freg[MAX_REG]; /* Floating point registers. */ ethr_tid tid; /* Thread id */ struct erl_bits_state erl_bits_state; /* erl_bits.c state */ void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */ + ErtsSchedulerSleepInfo *ssi; Process *free_process; #endif +#if !HEAP_ON_C_STACK + Eterm tmp_heap[TMP_HEAP_SIZE]; + int num_tmp_heap_used; + Eterm beam_emu_tmp_heap[BEAM_EMU_TMP_HEAP_SIZE]; + Eterm cmp_tmp_heap[CMP_TMP_HEAP_SIZE]; + Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE]; +#endif Process *current_process; Uint no; /* Scheduler number */ @@ -363,11 +422,6 @@ struct ErtsSchedulerData_ { #ifdef ERTS_SMP /* NOTE: These fields are modified under held mutexes by other threads */ -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - int check_children; /* run queue mutex */ - int blocked_check_children; /* schdlr_sspnd mutex */ -#endif - erts_smp_atomic_t suspended; /* Only used when common run queue */ erts_smp_atomic_t chk_cpu_bind; /* Only used when common run queue */ #endif }; @@ -386,8 +440,9 @@ extern ErtsSchedulerData *erts_scheduler_data; #define ERTS_PSD_SAVED_CALLS_BUF 1 #define ERTS_PSD_SCHED_ID 2 #define ERTS_PSD_DIST_ENTRY 3 +#define ERTS_PSD_CALL_TIME_BP 4 -#define ERTS_PSD_SIZE 4 +#define ERTS_PSD_SIZE 5 typedef struct { void *data[ERTS_PSD_SIZE]; @@ -408,6 +463,9 @@ typedef struct { #define ERTS_PSD_DIST_ENTRY_GET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_DIST_ENTRY_SET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN + typedef struct { ErtsProcLocks get_locks; ErtsProcLocks set_locks; @@ -479,6 +537,7 @@ struct ErtsPendingSuspend_ { # define MIN_VHEAP_SIZE(p) (p)->min_vheap_size # define BIN_VHEAP_SZ(p) (p)->bin_vheap_sz +# define BIN_VHEAP_MATURE(p) (p)->bin_vheap_mature # define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz # define BIN_OLD_VHEAP(p) (p)->bin_old_vheap @@ -518,8 +577,8 @@ struct process { unsigned max_arg_reg; /* Maximum number of argument registers available. */ Eterm def_arg_reg[6]; /* Default array for argument registers. */ - Eterm* cp; /* Continuation pointer (for threaded code). */ - Eterm* i; /* Program counter for threaded code. */ + BeamInstr* cp; /* (untagged) Continuation pointer (for threaded code). */ + BeamInstr* i; /* Program counter for threaded code. */ Sint catches; /* Number of catches on stack */ Sint fcalls; /* * Number of reductions left to execute. @@ -566,11 +625,12 @@ struct process { Uint seq_trace_lastcnt; Eterm seq_trace_token; /* Sequential trace token (tuple size 5 see below) */ - Eterm initial[3]; /* Initial module(0), function(1), arity(2) */ - Eterm* current; /* Current Erlang function: + BeamInstr initial[3]; /* Initial module(0), function(1), arity(2), often used instead + of pointer to funcinfo instruction, hence the BeamInstr datatype */ + BeamInstr* current; /* Current Erlang function, part of the funcinfo: * module(0), function(1), arity(2) * (module and functions are tagged atoms; - * arity an untagged integer). + * arity an untagged integer). BeamInstr * because it references code */ /* @@ -595,9 +655,10 @@ struct process { Uint mbuf_sz; /* Size of all message buffers */ ErtsPSD *psd; /* Rarely used process specific data */ - Uint bin_vheap_sz; /* Virtual heap block size for binaries */ - Uint bin_old_vheap_sz; /* Virtual old heap block size for binaries */ - Uint bin_old_vheap; /* Virtual old heap size for binaries */ + Uint64 bin_vheap_sz; /* Virtual heap block size for binaries */ + Uint64 bin_vheap_mature; /* Virtual heap block size for binaries */ + Uint64 bin_old_vheap_sz; /* Virtual old heap block size for binaries */ + Uint64 bin_old_vheap; /* Virtual old heap size for binaries */ union { #ifdef ERTS_SMP @@ -753,7 +814,7 @@ ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp) { ErlHeapFragment* hf = MBUF(p); - ASSERT(hf!=NULL && (hp - hf->mem < (unsigned long)hf->size)); + ASSERT(hf!=NULL && (hp - hf->mem < (unsigned long)hf->alloc_size)); hf->used_size = hp - hf->mem; } @@ -812,7 +873,7 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; #define F_INSLPQUEUE (1 << 1) /* Set if in timer queue */ #define F_TIMO (1 << 2) /* Set if timeout */ #define F_HEAP_GROW (1 << 3) -#define F_NEED_FULLSWEEP (1 << 4) /* If process has old binaries & funs. */ +#define F_NEED_FULLSWEEP (1 << 4) #define F_USING_DB (1 << 5) /* If have created tables */ #define F_DISTRIBUTION (1 << 6) /* Process used in distribution */ #define F_USING_DDLL (1 << 7) /* Process has used the DDLL interface */ @@ -966,6 +1027,7 @@ int erts_init_scheduler_bind_type(char *how); #define ERTS_INIT_CPU_TOPOLOGY_MISSING 9 int erts_init_cpu_topology(char *topology_str); +int erts_update_cpu_info(void); void erts_pre_init_process(void); void erts_late_init_process(void); @@ -1069,6 +1131,9 @@ void erts_handle_pending_exit(Process *, ErtsProcLocks); void erts_deep_process_dump(int, void *); +Eterm erts_get_reader_groups_map(Process *c_p); +Eterm erts_debug_reader_groups_map(Process *c_p, int groups); + Sint erts_test_next_pid(int, Uint); Eterm erts_debug_processes(Process *c_p); Eterm erts_debug_processes_bif_info(Process *c_p); @@ -1184,7 +1249,7 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data) #endif #define ERTS_PROC_SCHED_ID(P, L, ID) \ - ((Uint) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID))) + ((UWord) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID))) #define ERTS_PROC_GET_DIST_ENTRY(P) \ ((DistEntry *) erts_psd_get((P), ERTS_PSD_DIST_ENTRY)) @@ -1196,6 +1261,12 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data) #define ERTS_PROC_SET_SAVED_CALLS_BUF(P, L, SCB) \ ((struct saved_calls *) erts_psd_set((P), (L), ERTS_PSD_SAVED_CALLS_BUF, (void *) (SCB))) +#define ERTS_PROC_GET_CALL_TIME(P) \ + ((process_breakpoint_time_t *) erts_psd_get((P), ERTS_PSD_CALL_TIME_BP)) +#define ERTS_PROC_SET_CALL_TIME(P, L, PBT) \ + ((process_breakpoint_time_t *) erts_psd_set((P), (L), ERTS_PSD_CALL_TIME_BP, (void *) (PBT))) + + ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p); ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, @@ -1209,8 +1280,8 @@ erts_proc_get_error_handler(Process *p) if (!val) return am_error_handler; else { - ASSERT(is_atom(((Eterm) val))); - return (Eterm) val; + ASSERT(is_atom(((Eterm) (UWord) val))); + return (Eterm) (UWord) val; } } @@ -1220,13 +1291,13 @@ erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, Eterm handler) void *old_val; void *new_val; ASSERT(is_atom(handler)); - new_val = handler == am_error_handler ? NULL : (void *) handler; + new_val = (handler == am_error_handler) ? NULL : (void *) (UWord) handler; old_val = erts_psd_set(p, plocks, ERTS_PSD_ERROR_HANDLER, new_val); if (!old_val) return am_error_handler; else { - ASSERT(is_atom(((Eterm) old_val))); - return (Eterm) old_val; + ASSERT(is_atom(((Eterm) (UWord) old_val))); + return (Eterm) (UWord) old_val; } } @@ -1487,29 +1558,30 @@ extern int erts_disable_proc_not_running_opt; #define ERTS_MIN_PROCESSES 16 #endif -#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS -ERTS_GLB_INLINE void erts_smp_notify_inc_runq(ErtsRunQueue *runq); -void erts_smp_notify_inc_runq__(ErtsRunQueue *runq); -#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */ +void erts_smp_notify_inc_runq(ErtsRunQueue *runq); -#if ERTS_GLB_INLINE_INCL_FUNC_DEF +#ifdef ERTS_SMP +void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, long); +ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi); -#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS +#if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void -erts_smp_notify_inc_runq(ErtsRunQueue *runq) +erts_sched_poke(ErtsSchedulerSleepInfo *ssi) { -#ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); - if (runq->waiting) - erts_smp_notify_inc_runq__(runq); -#endif + long flags = erts_smp_atomic_read(&ssi->flags); + ASSERT(!(flags & ERTS_SSI_FLG_SLEEPING) + || (flags & ERTS_SSI_FLG_WAITING)); + if (flags & ERTS_SSI_FLG_SLEEPING) { + flags = erts_smp_atomic_band(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP); + erts_sched_finish_poke(ssi, flags); + } } -#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */ - #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ +#endif /* #ifdef ERTS_SMP */ + #include "erl_process_lock.h" #undef ERTS_INCLUDE_SCHEDULER_INTERNALS diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 1666509c72..7a7042abe4 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2003-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2003-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -45,16 +45,16 @@ static void dump_dist_ext(int to, void *to_arg, ErtsDistExternal *edep); static void dump_element_nl(int to, void *to_arg, Eterm x); static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg); -static void print_function_from_pc(int to, void *to_arg, Eterm* x); +static void print_function_from_pc(int to, void *to_arg, BeamInstr* x); static void heap_dump(int to, void *to_arg, Eterm x); static void dump_binaries(int to, void *to_arg, Binary* root); static void dump_externally(int to, void *to_arg, Eterm term); static Binary* all_binaries; -extern Eterm beam_apply[]; -extern Eterm beam_exit[]; -extern Eterm beam_continue_exit[]; +extern BeamInstr beam_apply[]; +extern BeamInstr beam_exit[]; +extern BeamInstr beam_continue_exit[]; void @@ -223,7 +223,7 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) } if (is_CP(x)) { - erts_print(to, to_arg, "SReturn addr 0x%X (", (Eterm *) x); + erts_print(to, to_arg, "SReturn addr 0x%X (", cp_val(x)); print_function_from_pc(to, to_arg, cp_val(x)); erts_print(to, to_arg, ")\n"); yreg = 0; @@ -239,9 +239,9 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) } static void -print_function_from_pc(int to, void *to_arg, Eterm* x) +print_function_from_pc(int to, void *to_arg, BeamInstr* x) { - Eterm* addr = find_function_from_pc(x); + BeamInstr* addr = find_function_from_pc(x); if (addr == NULL) { if (x == beam_exit) { erts_print(to, to_arg, "<terminate process>"); @@ -273,7 +273,7 @@ heap_dump(int to, void *to_arg, Eterm x) if (x == OUR_NIL) { /* We are done. */ return; } if (is_CP(x)) { - next = (Eterm *) x; + next = (Eterm *) EXPAND_POINTER(x); } else if (is_list(x)) { ptr = list_val(x); if (ptr[0] != OUR_NIL) { @@ -286,7 +286,7 @@ heap_dump(int to, void *to_arg, Eterm x) ptr[1] = make_small(0); } x = ptr[0]; - ptr[0] = (Eterm) next; + ptr[0] = (Eterm) COMPRESS_POINTER(next); next = ptr + 1; goto again; } @@ -316,7 +316,7 @@ heap_dump(int to, void *to_arg, Eterm x) ptr[0] = OUR_NIL; } else { x = ptr[arity]; - ptr[0] = (Eterm) next; + ptr[0] = (Eterm) COMPRESS_POINTER(next); next = ptr + arity - 1; goto again; } @@ -351,7 +351,7 @@ heap_dump(int to, void *to_arg, Eterm x) Binary* val = pb->val; if (erts_smp_atomic_xchg(&val->refc, 0) != 0) { - val->flags = (Uint) all_binaries; + val->flags = (UWord) all_binaries; all_binaries = val; } erts_print(to, to_arg, "Yc%X:%X:%X", val, diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index 52440fb635..a4d12139e9 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2007-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2007-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -71,9 +71,12 @@ const Process erts_proc_lock_busy; #ifdef ERTS_SMP -/*#define ERTS_PROC_LOCK_SPIN_ON_GATE*/ -#define ERTS_PROC_LOCK_SPIN_COUNT_MAX 16000 +#define ERTS_PROC_LOCK_SPIN_COUNT_MAX 2000 +#define ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC 32 #define ERTS_PROC_LOCK_SPIN_COUNT_BASE 1000 +#define ERTS_PROC_LOCK_AUX_SPIN_COUNT 50 + +#define ERTS_PROC_LOCK_SPIN_UNTIL_YIELD 25 #ifdef ERTS_PROC_LOCK_DEBUG #define ERTS_PROC_LOCK_HARD_DEBUG @@ -83,32 +86,19 @@ const Process erts_proc_lock_busy; static void check_queue(erts_proc_lock_t *lck); #endif - -typedef struct erts_proc_lock_waiter_t_ erts_proc_lock_waiter_t; -struct erts_proc_lock_waiter_t_ { - erts_proc_lock_waiter_t *next; - erts_proc_lock_waiter_t *prev; - ErtsProcLocks wait_locks; - erts_smp_gate_t gate; - erts_proc_lock_queues_t *queues; -}; +#if SIZEOF_INT < 4 +#error "The size of the 'uflgs' field of the erts_tse_t type is too small" +#endif struct erts_proc_lock_queues_t_ { erts_proc_lock_queues_t *next; - erts_proc_lock_waiter_t *queue[ERTS_PROC_LOCK_MAX_BIT+1]; -}; - -struct erts_proc_lock_thr_spec_data_t_ { - erts_proc_lock_queues_t *qs; - erts_proc_lock_waiter_t *wtr; + erts_tse_t *queue[ERTS_PROC_LOCK_MAX_BIT+1]; }; static erts_proc_lock_queues_t zeroqs = {0}; -static erts_smp_spinlock_t wtr_lock; -static erts_proc_lock_waiter_t *waiter_free_list; +static erts_smp_spinlock_t qs_lock; static erts_proc_lock_queues_t *queue_free_list; -static erts_tsd_key_t waiter_key; #ifdef ERTS_ENABLE_LOCK_CHECK static struct { @@ -122,35 +112,26 @@ static struct { erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS]; static int proc_lock_spin_count; -static int proc_lock_trans_spin_cost; +static int aux_thr_proc_lock_spin_count; -static void cleanup_waiter(void); +static void cleanup_tse(void); void erts_init_proc_lock(void) { int i; int cpus; - erts_smp_spinlock_init(&wtr_lock, "proc_lck_wtr_alloc"); + erts_smp_spinlock_init(&qs_lock, "proc_lck_qs_alloc"); for (i = 0; i < ERTS_NO_OF_PIX_LOCKS; i++) { -#if ERTS_PROC_LOCK_MUTEX_IMPL -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_smp_mtx_init_x(&erts_pix_locks[i].u.mtx, "pix_lock", make_small(i)); -#else - erts_smp_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock"); -#endif -#else #ifdef ERTS_ENABLE_LOCK_COUNT - erts_smp_spinlock_init_x(&erts_pix_locks[i].u.spnlck, "pix_lock", make_small(i)); + erts_smp_spinlock_init_x(&erts_pix_locks[i].u.spnlck, + "pix_lock", make_small(i)); #else erts_smp_spinlock_init(&erts_pix_locks[i].u.spnlck, "pix_lock"); #endif -#endif } - waiter_free_list = NULL; queue_free_list = NULL; - erts_tsd_key_create(&waiter_key); - erts_thr_install_exit_handler(cleanup_waiter); + erts_thr_install_exit_handler(cleanup_tse); #ifdef ERTS_ENABLE_LOCK_CHECK lc_id.proc_lock_main = erts_lc_get_lock_order_id("proc_main"); lc_id.proc_lock_link = erts_lc_get_lock_order_id("proc_link"); @@ -158,86 +139,106 @@ erts_init_proc_lock(void) lc_id.proc_lock_status = erts_lc_get_lock_order_id("proc_status"); #endif cpus = erts_get_cpu_configured(erts_cpuinfo); - if (cpus > 1) - proc_lock_spin_count = (ERTS_PROC_LOCK_SPIN_COUNT_BASE - * ((int) erts_no_schedulers)); - else if (cpus == 1) - proc_lock_spin_count = 0; - else /* No of cpus unknown. Assume multi proc, but be conservative. */ + if (cpus > 1) { proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_BASE; - if (proc_lock_spin_count > ERTS_PROC_LOCK_SPIN_COUNT_MAX) - proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_MAX; - proc_lock_trans_spin_cost = proc_lock_spin_count/20; -} - -static ERTS_INLINE erts_proc_lock_waiter_t * -alloc_wtr(void) -{ - erts_proc_lock_waiter_t *wtr; - erts_smp_spin_lock(&wtr_lock); - wtr = waiter_free_list; - if (wtr) { - waiter_free_list = wtr->next; - ERTS_LC_ASSERT(queue_free_list); - wtr->queues = queue_free_list; - queue_free_list = wtr->queues->next; - erts_smp_spin_unlock(&wtr_lock); + proc_lock_spin_count += (ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC + * ((int) erts_no_schedulers)); + aux_thr_proc_lock_spin_count = ERTS_PROC_LOCK_AUX_SPIN_COUNT; } - else { - erts_smp_spin_unlock(&wtr_lock); - wtr = erts_alloc(ERTS_ALC_T_PROC_LCK_WTR, - sizeof(erts_proc_lock_waiter_t)); - erts_smp_gate_init(&wtr->gate); - wtr->wait_locks = (ErtsProcLocks) 0; - wtr->queues = erts_alloc(ERTS_ALC_T_PROC_LCK_QS, - sizeof(erts_proc_lock_queues_t)); - sys_memcpy((void *) wtr->queues, - (void *) &zeroqs, - sizeof(erts_proc_lock_queues_t)); + else if (cpus == 1) { + proc_lock_spin_count = 0; + aux_thr_proc_lock_spin_count = 0; } - return wtr; + else { /* No of cpus unknown. Assume multi proc, but be conservative. */ + proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_BASE/2; + aux_thr_proc_lock_spin_count = ERTS_PROC_LOCK_AUX_SPIN_COUNT/2; + } + if (proc_lock_spin_count > ERTS_PROC_LOCK_SPIN_COUNT_MAX) + proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_MAX; } #ifdef ERTS_ENABLE_LOCK_CHECK static void -check_unused_waiter(erts_proc_lock_waiter_t *wtr) +check_unused_tse(erts_tse_t *wtr) { int i; - ERTS_LC_ASSERT(wtr->wait_locks == 0); + erts_proc_lock_queues_t *queues = wtr->udata; + ERTS_LC_ASSERT(wtr->uflgs == 0); for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) - ERTS_LC_ASSERT(!wtr->queues->queue[i]); + ERTS_LC_ASSERT(!queues->queue[i]); } -#define CHECK_UNUSED_WAITER(W) check_unused_waiter((W)) +#define CHECK_UNUSED_TSE(W) check_unused_tse((W)) #else -#define CHECK_UNUSED_WAITER(W) +#define CHECK_UNUSED_TSE(W) #endif +static ERTS_INLINE erts_tse_t * +tse_fetch(erts_pix_lock_t *pix_lock) +{ + erts_tse_t *tse = erts_tse_fetch(); + if (!tse->udata) { + erts_proc_lock_queues_t *qs; +#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL + if (pix_lock) + erts_pix_unlock(pix_lock); +#endif + erts_smp_spin_lock(&qs_lock); + qs = queue_free_list; + if (qs) { + queue_free_list = queue_free_list->next; + erts_smp_spin_unlock(&qs_lock); + } + else { + erts_smp_spin_unlock(&qs_lock); + qs = erts_alloc(ERTS_ALC_T_PROC_LCK_QS, + sizeof(erts_proc_lock_queues_t)); + sys_memcpy((void *) qs, + (void *) &zeroqs, + sizeof(erts_proc_lock_queues_t)); + } + tse->udata = qs; +#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL + if (pix_lock) + erts_pix_lock(pix_lock); +#endif + } + tse->uflgs = 0; + return tse; +} static ERTS_INLINE void -free_wtr(erts_proc_lock_waiter_t *wtr) +tse_return(erts_tse_t *tse, int force_free_q) { - CHECK_UNUSED_WAITER(wtr); - erts_smp_spin_lock(&wtr_lock); - wtr->next = waiter_free_list; - waiter_free_list = wtr; - wtr->queues->next = queue_free_list; - queue_free_list = wtr->queues; - erts_smp_spin_unlock(&wtr_lock); + CHECK_UNUSED_TSE(tse); + if (force_free_q || erts_tse_is_tmp(tse)) { + erts_proc_lock_queues_t *qs = tse->udata; + ASSERT(qs); + erts_smp_spin_lock(&qs_lock); + qs->next = queue_free_list; + queue_free_list = qs; + erts_smp_spin_unlock(&qs_lock); + tse->udata = NULL; + } + erts_tse_return(tse); } void erts_proc_lock_prepare_proc_lock_waiter(void) { - erts_tsd_set(waiter_key, (void *) alloc_wtr()); + tse_return(tse_fetch(NULL), 0); } static void -cleanup_waiter(void) +cleanup_tse(void) { - erts_proc_lock_waiter_t *wtr = erts_tsd_get(waiter_key); - if (wtr) - free_wtr(wtr); + erts_tse_t *tse = erts_tse_fetch(); + if (tse) { + if (tse->udata) + tse_return(tse, 1); + else + erts_tse_return(tse); + } } @@ -250,7 +251,7 @@ cleanup_waiter(void) static ERTS_INLINE void enqueue_waiter(erts_proc_lock_queues_t *qs, int ix, - erts_proc_lock_waiter_t *wtr) + erts_tse_t *wtr) { if (!qs->queue[ix]) { qs->queue[ix] = wtr; @@ -266,10 +267,10 @@ enqueue_waiter(erts_proc_lock_queues_t *qs, } } -static erts_proc_lock_waiter_t * +static erts_tse_t * dequeue_waiter(erts_proc_lock_queues_t *qs, int ix) { - erts_proc_lock_waiter_t *wtr = qs->queue[ix]; + erts_tse_t *wtr = qs->queue[ix]; ERTS_LC_ASSERT(qs->queue[ix]); if (wtr->next == wtr) { ERTS_LC_ASSERT(qs->queue[ix]->prev == wtr); @@ -295,10 +296,10 @@ dequeue_waiter(erts_proc_lock_queues_t *qs, int ix) * lock. */ static ERTS_INLINE void -try_aquire(erts_proc_lock_t *lck, erts_proc_lock_waiter_t *wtr) +try_aquire(erts_proc_lock_t *lck, erts_tse_t *wtr) { ErtsProcLocks got_locks = (ErtsProcLocks) 0; - ErtsProcLocks locks = wtr->wait_locks; + ErtsProcLocks locks = wtr->uflgs; int lock_no; ERTS_LC_ASSERT(lck->queues); @@ -334,7 +335,7 @@ try_aquire(erts_proc_lock_t *lck, erts_proc_lock_waiter_t *wtr) } } - wtr->wait_locks &= ~got_locks; + wtr->uflgs &= ~got_locks; } /* @@ -350,8 +351,8 @@ transfer_locks(Process *p, int unlock) { int transferred = 0; - erts_proc_lock_waiter_t *wake = NULL; - erts_proc_lock_waiter_t *wtr; + erts_tse_t *wake = NULL; + erts_tse_t *wtr; ErtsProcLocks unset_waiter = 0; ErtsProcLocks tlocks = trnsfr_lcks; int lock_no; @@ -377,11 +378,11 @@ transfer_locks(Process *p, ERTS_LC_ASSERT(wtr); if (!qs->queue[lock_no]) unset_waiter |= lock; - ERTS_LC_ASSERT(wtr->wait_locks & lock); - wtr->wait_locks &= ~lock; - if (wtr->wait_locks) + ERTS_LC_ASSERT(wtr->uflgs & lock); + wtr->uflgs &= ~lock; + if (wtr->uflgs) try_aquire(&p->lock, wtr); - if (!wtr->wait_locks) { + if (!wtr->uflgs) { /* * The other thread got all locks it needs; * need to wake it up. @@ -412,9 +413,10 @@ transfer_locks(Process *p, erts_pix_unlock(pix_lock); do { - erts_proc_lock_waiter_t *tmp = wake; + erts_tse_t *tmp = wake; wake = wake->next; - erts_smp_gate_let_through(&tmp->gate, 1); + erts_atomic_set(&tmp->uaflgs, 0); + erts_tse_set(tmp); } while (wake); if (!unlock) @@ -462,26 +464,16 @@ wait_for_locks(Process *p, ErtsProcLocks olflgs) { erts_pix_lock_t *pix_lock = pixlck ? pixlck : ERTS_PID2PIXLOCK(p->id); - int tsd; - erts_proc_lock_waiter_t *wtr; + erts_tse_t *wtr; + erts_proc_lock_queues_t *qs; /* Acquire a waiter object on which this thread can wait. */ - wtr = erts_tsd_get(waiter_key); - if (wtr) - tsd = 1; - else { -#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL - erts_pix_unlock(pix_lock); -#endif - wtr = alloc_wtr(); - tsd = 0; -#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL - erts_pix_lock(pix_lock); -#endif - } + wtr = tse_fetch(pix_lock); /* Record which locks this waiter needs. */ - wtr->wait_locks = need_locks; + wtr->uflgs = need_locks; + + ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); #if ERTS_PROC_LOCK_ATOMIC_IMPL erts_pix_lock(pix_lock); @@ -489,14 +481,16 @@ wait_for_locks(Process *p, ERTS_LC_ASSERT(erts_lc_pix_lock_is_locked(pix_lock)); + qs = wtr->udata; + ASSERT(qs); /* Provide the process with waiter queues, if it doesn't have one. */ if (!p->lock.queues) { - wtr->queues->next = NULL; - p->lock.queues = wtr->queues; + qs->next = NULL; + p->lock.queues = qs; } else { - wtr->queues->next = p->lock.queues->next; - p->lock.queues->next = wtr->queues; + qs->next = p->lock.queues->next; + p->lock.queues->next = qs; } #ifdef ERTS_PROC_LOCK_HARD_DEBUG @@ -506,46 +500,59 @@ wait_for_locks(Process *p, /* Try to aquire locks one at a time in lock order and set wait flag */ try_aquire(&p->lock, wtr); + ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); + #ifdef ERTS_PROC_LOCK_HARD_DEBUG check_queue(&p->lock); #endif - if (wtr->wait_locks) { /* We didn't get them all; need to wait... */ - /* Got to wait for locks... */ + if (wtr->uflgs) { + /* We didn't get them all; need to wait... */ + + ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); + + erts_atomic_set(&wtr->uaflgs, 1); erts_pix_unlock(pix_lock); - /* - * Wait for needed locks. When we return all needed locks have - * have been acquired by other threads and transfered to us. - */ -#ifdef ERTS_PROC_LOCK_SPIN_ON_GATE - erts_smp_gate_swait(&wtr->gate, proc_lock_spin_count); -#else - erts_smp_gate_wait(&wtr->gate); -#endif + while (1) { + int res; + erts_tse_reset(wtr); + + if (erts_atomic_read(&wtr->uaflgs) == 0) + break; + + /* + * Wait for needed locks. When we are woken all needed locks have + * have been acquired by other threads and transfered to us. + * However, we need to be prepared for spurious wakeups. + */ + do { + res = erts_tse_wait(wtr); /* might return EINTR */ + } while (res != 0); + } erts_pix_lock(pix_lock); + + ASSERT(wtr->uflgs == 0); } /* Recover some queues to store in the waiter. */ ERTS_LC_ASSERT(p->lock.queues); if (p->lock.queues->next) { - wtr->queues = p->lock.queues->next; - p->lock.queues->next = wtr->queues->next; + qs = p->lock.queues->next; + p->lock.queues->next = qs->next; } else { - wtr->queues = p->lock.queues; + qs = p->lock.queues; p->lock.queues = NULL; } + wtr->udata = qs; erts_pix_unlock(pix_lock); ERTS_LC_ASSERT(locks == (ERTS_PROC_LOCK_FLGS_READ_(&p->lock) & locks)); - if (tsd) - CHECK_UNUSED_WAITER(wtr); - else - free_wtr(wtr); + tse_return(wtr, 0); } /* @@ -563,52 +570,57 @@ erts_proc_lock_failed(Process *p, ErtsProcLocks locks, ErtsProcLocks old_lflgs) { -#ifdef ERTS_PROC_LOCK_SPIN_ON_GATE - int spin_count = 0; -#else - int spin_count = proc_lock_spin_count; -#endif - + int until_yield = ERTS_PROC_LOCK_SPIN_UNTIL_YIELD; + int thr_spin_count; + int spin_count; ErtsProcLocks need_locks = locks; ErtsProcLocks olflgs = old_lflgs; - while (need_locks != 0) - { - ErtsProcLocks can_grab = in_order_locks(olflgs, need_locks); + if (erts_thr_get_main_status()) + thr_spin_count = proc_lock_spin_count; + else + thr_spin_count = aux_thr_proc_lock_spin_count; + + spin_count = thr_spin_count; + + while (need_locks != 0) { + ErtsProcLocks can_grab; + + can_grab = in_order_locks(olflgs, need_locks); - if (can_grab == 0) - { + if (can_grab == 0) { /* Someone already has the lowest-numbered lock we want. */ - if (spin_count-- <= 0) - { + if (spin_count-- <= 0) { /* Too many retries, give up and sleep for the lock. */ wait_for_locks(p, pixlck, locks, need_locks, olflgs); return; } + ERTS_SPIN_BODY; + + if (--until_yield == 0) { + until_yield = ERTS_PROC_LOCK_SPIN_UNTIL_YIELD; + erts_thr_yield(); + } + olflgs = ERTS_PROC_LOCK_FLGS_READ_(&p->lock); } - else - { + else { /* Try to grab all of the grabbable locks at once with cmpxchg. */ ErtsProcLocks grabbed = olflgs | can_grab; ErtsProcLocks nflgs = - ERTS_PROC_LOCK_FLGS_CMPXCHG_(&p->lock, grabbed, olflgs); + ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(&p->lock, grabbed, olflgs); - if (nflgs == olflgs) - { + if (nflgs == olflgs) { /* Success! We grabbed the 'can_grab' locks. */ olflgs = grabbed; need_locks &= ~can_grab; -#ifndef ERTS_PROC_LOCK_SPIN_ON_GATE /* Since we made progress, reset the spin count. */ - spin_count = proc_lock_spin_count; -#endif + spin_count = thr_spin_count; } - else - { + else { /* Compare-and-exchange failed, try again. */ olflgs = nflgs; } @@ -1407,7 +1419,7 @@ check_queue(erts_proc_lock_t *lck) wtr = (((ErtsProcLocks) 1) << lock_no) << ERTS_PROC_LOCK_WAITER_SHIFT; if (lflgs & wtr) { int n; - erts_proc_lock_waiter_t *wtr; + erts_tse_t *wtr; ERTS_LC_ASSERT(lck->queues && lck->queues->queue[lock_no]); wtr = lck->queues->queue[lock_no]; n = 0; diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index d71e5a0a6e..7cfc9893fa 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -255,11 +255,7 @@ void erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks); typedef struct { union { -#if ERTS_PROC_LOCK_MUTEX_IMPL - erts_smp_mtx_t mtx; -#else erts_smp_spinlock_t spnlck; -#endif char buf[64]; /* Try to get locks in different cache lines */ } u; } erts_pix_lock_t; @@ -277,9 +273,12 @@ typedef struct { ((ErtsProcLocks) erts_smp_atomic_band(&(L)->flags, (long) (MSK))) #define ERTS_PROC_LOCK_FLGS_BOR_(L, MSK) \ ((ErtsProcLocks) erts_smp_atomic_bor(&(L)->flags, (long) (MSK))) -#define ERTS_PROC_LOCK_FLGS_CMPXCHG_(L, NEW, EXPECTED) \ - ((ErtsProcLocks) erts_smp_atomic_cmpxchg(&(L)->flags, \ - (long) (NEW), (long) (EXPECTED))) +#define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \ + ((ErtsProcLocks) erts_smp_atomic_cmpxchg_acqb(&(L)->flags, \ + (long) (NEW), (long) (EXPECTED))) +#define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \ + ((ErtsProcLocks) erts_smp_atomic_cmpxchg_relb(&(L)->flags, \ + (long) (NEW), (long) (EXPECTED))) #define ERTS_PROC_LOCK_FLGS_READ_(L) \ ((ErtsProcLocks) erts_smp_atomic_read(&(L)->flags)) @@ -289,6 +288,9 @@ ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_band(erts_proc_lock_t *, ErtsProcLocks); ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_bor(erts_proc_lock_t *, ErtsProcLocks); +ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *, + ErtsProcLocks, + ErtsProcLocks); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -322,7 +324,9 @@ erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *lck, ErtsProcLocks new, #define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) erts_proc_lock_flags_band((L), (MSK)) #define ERTS_PROC_LOCK_FLGS_BOR_(L, MSK) erts_proc_lock_flags_bor((L), (MSK)) -#define ERTS_PROC_LOCK_FLGS_CMPXCHG_(L, NEW, EXPECTED) \ +#define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \ + erts_proc_lock_flags_cmpxchg((L), (NEW), (EXPECTED)) +#define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \ erts_proc_lock_flags_cmpxchg((L), (NEW), (EXPECTED)) #define ERTS_PROC_LOCK_FLGS_READ_(L) ((L)->flags) @@ -348,9 +352,9 @@ ERTS_GLB_INLINE ErtsProcLocks erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks); #ifdef ERTS_ENABLE_LOCK_COUNT ERTS_GLB_INLINE void erts_smp_proc_lock_x__(Process *, - erts_pix_lock_t *, - ErtsProcLocks, - char *file, unsigned int line); + erts_pix_lock_t *, + ErtsProcLocks, + char *file, unsigned int line); #else ERTS_GLB_INLINE void erts_smp_proc_lock__(Process *, erts_pix_lock_t *, @@ -372,30 +376,18 @@ ERTS_GLB_INLINE void erts_proc_lock_op_debug(Process *, ErtsProcLocks, int); ERTS_GLB_INLINE void erts_pix_lock(erts_pix_lock_t *pixlck) { ERTS_LC_ASSERT(pixlck); -#if ERTS_PROC_LOCK_MUTEX_IMPL - erts_smp_mtx_lock(&pixlck->u.mtx); -#else erts_smp_spin_lock(&pixlck->u.spnlck); -#endif } ERTS_GLB_INLINE void erts_pix_unlock(erts_pix_lock_t *pixlck) { ERTS_LC_ASSERT(pixlck); -#if ERTS_PROC_LOCK_MUTEX_IMPL - erts_smp_mtx_unlock(&pixlck->u.mtx); -#else erts_smp_spin_unlock(&pixlck->u.spnlck); -#endif } ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *pixlck) { -#if ERTS_PROC_LOCK_MUTEX_IMPL - return erts_smp_lc_mtx_is_locked(&pixlck->u.mtx); -#else return erts_smp_lc_spinlock_is_locked(&pixlck->u.spnlck); -#endif } /* @@ -417,9 +409,9 @@ erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks) ErtsProcLocks expct_lflgs = 0; while (1) { - ErtsProcLocks lflgs = ERTS_PROC_LOCK_FLGS_CMPXCHG_(&p->lock, - expct_lflgs | locks, - expct_lflgs); + ErtsProcLocks lflgs = ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(&p->lock, + expct_lflgs | locks, + expct_lflgs); if (ERTS_LIKELY(lflgs == expct_lflgs)) { /* We successfully grabbed all locks. */ return 0; @@ -535,7 +527,7 @@ erts_smp_proc_unlock__(Process *p, if (want_lflgs != old_lflgs) { ErtsProcLocks new_lflgs = - ERTS_PROC_LOCK_FLGS_CMPXCHG_(&p->lock, want_lflgs, old_lflgs); + ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(&p->lock, want_lflgs, old_lflgs); if (new_lflgs != old_lflgs) { /* cmpxchg failed, try again. */ diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index 03d2a586e3..b41fa70476 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ /* @@ -43,9 +43,17 @@ typedef erts_thr_init_data_t erts_smp_thr_init_data_t; typedef erts_tid_t erts_smp_tid_t; typedef erts_mtx_t erts_smp_mtx_t; typedef erts_cnd_t erts_smp_cnd_t; +#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER ERTS_RWMTX_OPT_DEFAULT_INITER +#define ERTS_SMP_RWMTX_TYPE_NORMAL ERTS_RWMTX_TYPE_NORMAL +#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ ERTS_RWMTX_TYPE_FREQUENT_READ +#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \ + ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ +#define ERTS_SMP_RWMTX_LONG_LIVED ERTS_RWMTX_LONG_LIVED +#define ERTS_SMP_RWMTX_SHORT_LIVED ERTS_RWMTX_SHORT_LIVED +#define ERTS_SMP_RWMTX_UNKNOWN_LIVED ERTS_RWMTX_UNKNOWN_LIVED +typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t; typedef erts_rwmtx_t erts_smp_rwmtx_t; typedef erts_tsd_key_t erts_smp_tsd_key_t; -typedef erts_gate_t erts_smp_gate_t; typedef ethr_atomic_t erts_smp_atomic_t; typedef erts_spinlock_t erts_smp_spinlock_t; typedef erts_rwlock_t erts_smp_rwlock_t; @@ -54,15 +62,27 @@ void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ #else /* #ifdef ERTS_SMP */ -#define ERTS_SMP_THR_OPTS_DEFAULT_INITER 0 +#define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0} typedef int erts_smp_thr_opts_t; typedef int erts_smp_thr_init_data_t; typedef int erts_smp_tid_t; typedef int erts_smp_mtx_t; typedef int erts_smp_cnd_t; +#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER {0} +#define ERTS_SMP_RWMTX_TYPE_NORMAL 0 +#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ 0 +#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0 +#define ERTS_SMP_RWMTX_LONG_LIVED 0 +#define ERTS_SMP_RWMTX_SHORT_LIVED 0 +#define ERTS_SMP_RWMTX_UNKNOWN_LIVED 0 +typedef struct { + char type; + char lived; + int main_spincount; + int aux_spincount; +} erts_smp_rwmtx_opt_t; typedef int erts_smp_rwmtx_t; typedef int erts_smp_tsd_key_t; -typedef int erts_smp_gate_t; typedef long erts_smp_atomic_t; #if __GNUC__ > 2 typedef struct { } erts_smp_spinlock_t; @@ -103,8 +123,6 @@ ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, ERTS_GLB_INLINE void erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_smp_mtx_destroy(erts_smp_mtx_t *mtx); -ERTS_GLB_INLINE void erts_smp_mtx_set_forksafe(erts_smp_mtx_t *mtx); -ERTS_GLB_INLINE void erts_smp_mtx_unset_forksafe(erts_smp_mtx_t *mtx); ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx); #ifdef ERTS_ENABLE_LOCK_COUNT ERTS_GLB_INLINE void erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, int line); @@ -119,9 +137,17 @@ ERTS_GLB_INLINE void erts_smp_cnd_wait(erts_smp_cnd_t *cnd, erts_smp_mtx_t *mtx); ERTS_GLB_INLINE void erts_smp_cnd_signal(erts_smp_cnd_t *cnd); ERTS_GLB_INLINE void erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd); +ERTS_GLB_INLINE void erts_smp_rwmtx_set_reader_group(int no); +ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx, + erts_smp_rwmtx_opt_t *opt, + char *name, + Eterm extra); ERTS_GLB_INLINE void erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra); +ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx, + erts_smp_rwmtx_opt_t *opt, + char *name); ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name); ERTS_GLB_INLINE void erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx); @@ -155,6 +181,16 @@ ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp, long expected); ERTS_GLB_INLINE long erts_smp_atomic_bor(erts_smp_atomic_t *var, long mask); ERTS_GLB_INLINE long erts_smp_atomic_band(erts_smp_atomic_t *var, long mask); +ERTS_GLB_INLINE long erts_smp_atomic_read_acqb(erts_smp_atomic_t *var); +ERTS_GLB_INLINE void erts_smp_atomic_set_relb(erts_smp_atomic_t *var, long i); +ERTS_GLB_INLINE void erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp); +ERTS_GLB_INLINE long erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp); +ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp, + long new, + long exp); +ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp, + long new, + long exp); ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra); @@ -190,12 +226,6 @@ ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp); ERTS_GLB_INLINE void erts_smp_tsd_key_delete(erts_smp_tsd_key_t key); ERTS_GLB_INLINE void erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value); ERTS_GLB_INLINE void * erts_smp_tsd_get(erts_smp_tsd_key_t key); -ERTS_GLB_INLINE void erts_smp_gate_init(erts_smp_gate_t *gp); -ERTS_GLB_INLINE void erts_smp_gate_destroy(erts_smp_gate_t *gp); -ERTS_GLB_INLINE void erts_smp_gate_close(erts_smp_gate_t *gp); -ERTS_GLB_INLINE void erts_smp_gate_let_through(erts_smp_gate_t *gp, unsigned no); -ERTS_GLB_INLINE void erts_smp_gate_wait(erts_smp_gate_t *gp); -ERTS_GLB_INLINE void erts_smp_gate_swait(erts_smp_gate_t *gp, int spincount); #ifdef ERTS_THR_HAVE_SIG_FUNCS #define ERTS_SMP_THR_HAVE_SIG_FUNCS 1 @@ -331,22 +361,6 @@ erts_smp_mtx_destroy(erts_smp_mtx_t *mtx) #endif } -ERTS_GLB_INLINE void -erts_smp_mtx_set_forksafe(erts_smp_mtx_t *mtx) -{ -#ifdef ERTS_SMP - erts_mtx_set_forksafe(mtx); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_mtx_unset_forksafe(erts_smp_mtx_t *mtx) -{ -#ifdef ERTS_SMP - erts_mtx_unset_forksafe(mtx); -#endif -} - ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx) { @@ -433,6 +447,25 @@ erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd) } ERTS_GLB_INLINE void +erts_smp_rwmtx_set_reader_group(int no) +{ +#ifdef ERTS_SMP + erts_rwmtx_set_reader_group(no); +#endif +} + +ERTS_GLB_INLINE void +erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx, + erts_smp_rwmtx_opt_t *opt, + char *name, + Eterm extra) +{ +#ifdef ERTS_SMP + erts_rwmtx_init_opt_x(rwmtx, opt, name, extra); +#endif +} + +ERTS_GLB_INLINE void erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra) { #ifdef ERTS_SMP @@ -441,6 +474,16 @@ erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra) } ERTS_GLB_INLINE void +erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx, + erts_smp_rwmtx_opt_t *opt, + char *name) +{ +#ifdef ERTS_SMP + erts_rwmtx_init_opt(rwmtx, opt, name); +#endif +} + +ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name) { #ifdef ERTS_SMP @@ -709,6 +752,73 @@ erts_smp_atomic_band(erts_smp_atomic_t *var, long mask) #endif } +ERTS_GLB_INLINE long +erts_smp_atomic_read_acqb(erts_smp_atomic_t *var) +{ +#ifdef ERTS_SMP + return erts_atomic_read_acqb(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic_set_relb(erts_smp_atomic_t *var, long i) +{ +#ifdef ERTS_SMP + erts_atomic_set_relb(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp) +{ +#ifdef ERTS_SMP + erts_atomic_dec_relb(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE long +erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp) +{ +#ifdef ERTS_SMP + return erts_atomic_dectest_relb(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp, + long new, + long exp) +{ +#ifdef ERTS_SMP + return erts_atomic_cmpxchg_acqb(xchgp, new, exp); +#else + long old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp, + long new, + long exp) +{ +#ifdef ERTS_SMP + return erts_atomic_cmpxchg_relb(xchgp, new, exp); +#else + long old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra) { @@ -919,54 +1029,6 @@ erts_smp_tsd_get(erts_smp_tsd_key_t key) #endif } -ERTS_GLB_INLINE void -erts_smp_gate_init(erts_smp_gate_t *gp) -{ -#ifdef ERTS_SMP - erts_gate_init((erts_gate_t *) gp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_destroy(erts_smp_gate_t *gp) -{ -#ifdef ERTS_SMP - erts_gate_destroy((erts_gate_t *) gp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_close(erts_smp_gate_t *gp) -{ -#ifdef ERTS_SMP - erts_gate_close((erts_gate_t *) gp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_let_through(erts_smp_gate_t *gp, unsigned no) -{ -#ifdef ERTS_SMP - erts_gate_let_through((erts_gate_t *) gp, no); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_wait(erts_smp_gate_t *gp) -{ -#ifdef ERTS_SMP - erts_gate_wait((erts_gate_t *) gp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_swait(erts_smp_gate_t *gp, int spincount) -{ -#ifdef ERTS_SMP - erts_gate_swait((erts_gate_t *) gp, spincount); -#endif -} - #ifdef ERTS_THR_HAVE_SIG_FUNCS #define ERTS_SMP_THR_HAVE_SIG_FUNCS 1 diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c index 2924abbd51..c6458a0e45 100644 --- a/erts/emulator/beam/erl_term.c +++ b/erts/emulator/beam/erl_term.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2000-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -68,7 +68,9 @@ unsigned tag_val_def(Eterm x) static char msg[32]; switch (x & _TAG_PRIMARY_MASK) { - case TAG_PRIMARY_LIST: return LIST_DEF; + case TAG_PRIMARY_LIST: + ET_ASSERT(_list_precond(x),file,line); + return LIST_DEF; case TAG_PRIMARY_BOXED: { Eterm hdr = *boxed_val(x); ET_ASSERT(is_header(hdr),file,line); @@ -103,7 +105,7 @@ unsigned tag_val_def(Eterm x) break; } } - sprintf(msg, "tag_val_def: %#lx", x); + sprintf(msg, "tag_val_def: %#lx", (unsigned long) x); et_abort(msg, file, line); #undef file #undef line @@ -121,12 +123,12 @@ FUNTY checked_##FUN(ARGTY x, const char *file, unsigned line) \ return _unchecked_##FUN(x); \ } -ET_DEFINE_CHECKED(Eterm,make_boxed,Eterm*,_is_aligned); +ET_DEFINE_CHECKED(Eterm,make_boxed,Eterm*,_is_taggable_pointer); ET_DEFINE_CHECKED(int,is_boxed,Eterm,!is_header); -ET_DEFINE_CHECKED(Eterm*,boxed_val,Eterm,is_boxed); -ET_DEFINE_CHECKED(Eterm,make_list,Eterm*,_is_aligned); +ET_DEFINE_CHECKED(Eterm*,boxed_val,Eterm,_boxed_precond); +ET_DEFINE_CHECKED(Eterm,make_list,Eterm*,_is_taggable_pointer); ET_DEFINE_CHECKED(int,is_not_list,Eterm,!is_header); -ET_DEFINE_CHECKED(Eterm*,list_val,Eterm,is_list); +ET_DEFINE_CHECKED(Eterm*,list_val,Eterm,_list_precond); ET_DEFINE_CHECKED(Uint,unsigned_val,Eterm,is_small); ET_DEFINE_CHECKED(Sint,signed_val,Eterm,is_small); ET_DEFINE_CHECKED(Uint,atom_val,Eterm,is_atom); @@ -163,8 +165,8 @@ ET_DEFINE_CHECKED(Uint32*,external_ref_data,Eterm,is_external_ref); ET_DEFINE_CHECKED(struct erl_node_*,external_ref_node,Eterm,is_external_ref); ET_DEFINE_CHECKED(Eterm*,export_val,Eterm,is_export); -ET_DEFINE_CHECKED(Eterm,make_cp,Uint*,_is_aligned); -ET_DEFINE_CHECKED(Uint*,cp_val,Eterm,is_CP); +ET_DEFINE_CHECKED(Eterm,make_cp,UWord *,_is_taggable_pointer); +ET_DEFINE_CHECKED(UWord *,cp_val,Eterm,is_CP); ET_DEFINE_CHECKED(Uint,catch_val,Eterm,is_catch); ET_DEFINE_CHECKED(Uint,x_reg_offset,Uint,_is_xreg); ET_DEFINE_CHECKED(Uint,y_reg_offset,Uint,_is_yreg); diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index a6596558fa..b8e4473141 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -20,6 +20,32 @@ #ifndef __ERL_TERM_H #define __ERL_TERM_H +#include "sys.h" /* defines HALFWORD_HEAP */ + +#if HALFWORD_HEAP +# define HEAP_ON_C_STACK 0 +# if HALFWORD_ASSERT +# ifdef ET_DEBUG +# undef ET_DEBUG +# endif +# define ET_DEBUG 1 +# endif +# if 1 +# define CHECK_POINTER_MASK 0xFFFFFFFF00000000UL +# define COMPRESS_POINTER(APointer) ((Eterm) (UWord) (APointer)) +# define EXPAND_POINTER(AnEterm) ((UWord) (AnEterm)) +# else +# define CHECK_POINTER_MASK 0x0UL +# define COMPRESS_POINTER(AnUint) (AnUint) +# define EXPAND_POINTER(APointer) (APointer) +# endif +#else +# define HEAP_ON_C_STACK 1 +# define CHECK_POINTER_MASK 0x0UL +# define COMPRESS_POINTER(AnUint) (AnUint) +# define EXPAND_POINTER(APointer) (APointer) +#endif + struct erl_node_; /* Declared in erl_node_tables.h */ /* @@ -158,9 +184,16 @@ struct erl_node_; /* Declared in erl_node_tables.h */ /* boxed object access methods */ +#if HALFWORD_HEAP +#define _is_taggable_pointer(x) (((UWord)(x) & (CHECK_POINTER_MASK | 0x3)) == 0) +#define _boxed_precond(x) (is_boxed(x)) +#else +#define _is_taggable_pointer(x) (((Uint)(x) & 0x3) == 0) +#define _boxed_precond(x) (is_boxed(x)) +#endif #define _is_aligned(x) (((Uint)(x) & 0x3) == 0) -#define _unchecked_make_boxed(x) ((Uint)(x) + TAG_PRIMARY_BOXED) -_ET_DECLARE_CHECKED(Eterm,make_boxed,Eterm*) +#define _unchecked_make_boxed(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_BOXED) +_ET_DECLARE_CHECKED(Eterm,make_boxed,Eterm*); #define make_boxed(x) _ET_APPLY(make_boxed,(x)) #if 1 #define _is_not_boxed(x) ((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_BOXED)) @@ -170,13 +203,13 @@ _ET_DECLARE_CHECKED(int,is_boxed,Eterm) #else #define is_boxed(x) (((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_BOXED) #endif -#define _unchecked_boxed_val(x) ((Eterm*)((x) - TAG_PRIMARY_BOXED)) -_ET_DECLARE_CHECKED(Eterm*,boxed_val,Eterm) +#define _unchecked_boxed_val(x) ((Eterm*) EXPAND_POINTER(((x) - TAG_PRIMARY_BOXED))) +_ET_DECLARE_CHECKED(Eterm*,boxed_val,Eterm); #define boxed_val(x) _ET_APPLY(boxed_val,(x)) /* cons cell ("list") access methods */ -#define _unchecked_make_list(x) ((Uint)(x) + TAG_PRIMARY_LIST) -_ET_DECLARE_CHECKED(Eterm,make_list,Eterm*) +#define _unchecked_make_list(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_LIST) +_ET_DECLARE_CHECKED(Eterm,make_list,Eterm*); #define make_list(x) _ET_APPLY(make_list,(x)) #if 1 #define _unchecked_is_not_list(x) ((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_LIST)) @@ -187,8 +220,13 @@ _ET_DECLARE_CHECKED(int,is_not_list,Eterm) #define is_list(x) (((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_LIST) #define is_not_list(x) (!is_list((x))) #endif -#define _unchecked_list_val(x) ((Eterm*)((x) - TAG_PRIMARY_LIST)) -_ET_DECLARE_CHECKED(Eterm*,list_val,Eterm) +#if HALFWORD_HEAP +#define _list_precond(x) (is_list(x)) +#else +#define _list_precond(x) (is_list(x)) +#endif +#define _unchecked_list_val(x) ((Eterm*) EXPAND_POINTER((x) - TAG_PRIMARY_LIST)) +_ET_DECLARE_CHECKED(Eterm*,list_val,Eterm); #define list_val(x) _ET_APPLY(list_val,(x)) #define CONS(hp, car, cdr) \ @@ -198,13 +236,13 @@ _ET_DECLARE_CHECKED(Eterm*,list_val,Eterm) #define CDR(x) ((x)[1]) /* generic tagged pointer (boxed or list) access methods */ -#define _unchecked_ptr_val(x) ((Eterm*)((x) & ~((Uint) 0x3))) +#define _unchecked_ptr_val(x) ((Eterm*) EXPAND_POINTER((x) & ~((Uint) 0x3))) #define ptr_val(x) _unchecked_ptr_val((x)) /*XXX*/ #define _unchecked_offset_ptr(x,offs) ((x)+((offs)*sizeof(Eterm))) #define offset_ptr(x,offs) _unchecked_offset_ptr(x,offs) /*XXX*/ /* fixnum ("small") access methods */ -#if defined(ARCH_64) +#if defined(ARCH_64) && !HALFWORD_HEAP #define SMALL_BITS (64-4) #define SMALL_DIGITS (17) #else @@ -329,7 +367,11 @@ _ET_DECLARE_CHECKED(Eterm*,fun_val,Eterm) _ET_DECLARE_CHECKED(Eterm*,export_val,Eterm) #define export_val(x) _ET_APPLY(export_val,(x)) #define is_export_header(x) ((x) == HEADER_EXPORT) +#if HALFWORD_HEAP +#define HEADER_EXPORT _make_header(2,_TAG_HEADER_EXPORT) +#else #define HEADER_EXPORT _make_header(1,_TAG_HEADER_EXPORT) +#endif /* bignum access methods */ #define make_pos_bignum_header(sz) _make_header((sz),_TAG_HEADER_POS_BIG) @@ -353,7 +395,7 @@ _ET_DECLARE_CHECKED(Eterm*,big_val,Eterm) #define big_val(x) _ET_APPLY(big_val,(x)) /* flonum ("float") access methods */ -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP #define HEADER_FLONUM _make_header(1,_TAG_HEADER_FLOAT) #else #define HEADER_FLONUM _make_header(2,_TAG_HEADER_FLOAT) @@ -374,12 +416,12 @@ typedef union float_def byte fb[sizeof(ieee754_8)]; Uint16 fs[sizeof(ieee754_8) / sizeof(Uint16)]; Uint32 fw[sizeof(ieee754_8) / sizeof(Uint32)]; -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP Uint fdw; #endif } FloatDef; -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP #define GET_DOUBLE(x, f) (f).fdw = *(float_val(x)+1) #define PUT_DOUBLE(f, x) *(x) = HEADER_FLONUM, \ @@ -679,7 +721,7 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_port_node,Eterm) #define ERTS_MAX_REF_NUMBERS 3 #define ERTS_REF_NUMBERS ERTS_MAX_REF_NUMBERS -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP # define ERTS_REF_WORDS (ERTS_REF_NUMBERS/2 + 1) # define ERTS_REF_32BIT_WORDS (ERTS_REF_NUMBERS+1) #else @@ -701,7 +743,7 @@ typedef struct { #define make_ref_thing_header(DW) \ _make_header((DW)+REF_THING_HEAD_SIZE-1,_TAG_HEADER_REF) -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP /* * Ref layout on a 64-bit little endian machine: @@ -779,10 +821,10 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_ref_node,Eterm) * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * |A A A A A A A A A A A A A A A A A A A A A A A A A A|t t t t|0 0| Thing * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - * |N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N| Next - * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * |E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E| ErlNode * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N| Next + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * |X X X X X X X X X X X X X X X X X X X X X X X X X X X X X X X X| Data 0 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * . . . @@ -793,7 +835,7 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_ref_node,Eterm) * t : External pid thing tag (1100) * t : External port thing tag (1101) * t : External ref thing tag (1110) - * N : Next (external thing) pointer + * N : Next (off_heap) pointer * E : ErlNode pointer * X : Type specific data * @@ -807,11 +849,19 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_ref_node,Eterm) * */ +/* XXX:PaN - this structure is not perfect for halfword heap, it takes + a lot of memory due to padding, and the array will not begin at the end of the + structure, as otherwise expected. Be sure to access data.ui32 array and not try + to do pointer manipulation on an Eterm * to reach the actual data... + + XXX:Sverk - Problem made worse by "one off-heap list" when 'next' pointer + must align with 'next' in ProcBin, erl_fun_thing and erl_off_heap_header. +*/ typedef struct external_thing_ { /* ----+ */ Eterm header; /* | */ - struct external_thing_ *next; /* > External thing head */ - struct erl_node_ *node; /* | */ + struct erl_node_* node; /* > External thing head */ + struct erl_off_heap_header* next; /* | */ /* ----+ */ union { Uint32 ui32[1]; @@ -944,15 +994,15 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm) #error "fix yer arch, like" #endif -#define _unchecked_make_cp(x) ((Eterm)(x)) -_ET_DECLARE_CHECKED(Eterm,make_cp,Uint*) +#define _unchecked_make_cp(x) ((Eterm) COMPRESS_POINTER(x)) +_ET_DECLARE_CHECKED(Eterm,make_cp,BeamInstr*); #define make_cp(x) _ET_APPLY(make_cp,(x)) #define is_not_CP(x) ((x) & _CPMASK) #define is_CP(x) (!is_not_CP(x)) -#define _unchecked_cp_val(x) ((Uint*)(x)) -_ET_DECLARE_CHECKED(Uint*,cp_val,Eterm) +#define _unchecked_cp_val(x) ((BeamInstr*) EXPAND_POINTER(x)) +_ET_DECLARE_CHECKED(BeamInstr*,cp_val,Eterm); #define cp_val(x) _ET_APPLY(cp_val,(x)) #define make_catch(x) (((x) << _TAG_IMMED2_SIZE) | _TAG_IMMED2_CATCH) diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index d635916dd8..0b7269262e 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2001-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -25,6 +25,11 @@ #ifndef ERL_THREAD_H__ #define ERL_THREAD_H__ +#define ERTS_SPIN_BODY ETHR_SPIN_BODY + +#define ERTS_MAX_READER_GROUPS 8 +extern int erts_reader_groups; + #include "sys.h" #ifdef USE_THREADS @@ -34,6 +39,8 @@ #include "erl_lock_count.h" #include "erl_term.h" +#define ERTS_THR_MEMORY_BARRIER ETHR_MEMORY_BARRIER + #ifdef ERTS_ENABLE_LOCK_COUNT #define erts_mtx_lock(L) erts_mtx_lock_x(L, __FILE__, __LINE__) #define erts_spin_lock(L) erts_spin_lock_x(L, __FILE__, __LINE__) @@ -46,6 +53,7 @@ #define ERTS_THR_OPTS_DEFAULT_INITER ETHR_THR_OPTS_DEFAULT_INITER typedef ethr_thr_opts erts_thr_opts_t; typedef ethr_init_data erts_thr_init_data_t; +typedef ethr_late_init_data erts_thr_late_init_data_t; typedef ethr_tid erts_tid_t; /* mutex */ @@ -71,8 +79,19 @@ typedef struct { erts_lcnt_lock_t lcnt; #endif } erts_rwmtx_t; + +#define ERTS_RWMTX_OPT_DEFAULT_INITER ETHR_RWMUTEX_OPT_DEFAULT_INITER +#define ERTS_RWMTX_TYPE_NORMAL ETHR_RWMUTEX_TYPE_NORMAL +#define ERTS_RWMTX_TYPE_FREQUENT_READ ETHR_RWMUTEX_TYPE_FREQUENT_READ +#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \ + ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ +#define ERTS_RWMTX_LONG_LIVED ETHR_RWMUTEX_LONG_LIVED +#define ERTS_RWMTX_SHORT_LIVED ETHR_RWMUTEX_SHORT_LIVED +#define ERTS_RWMTX_UNKNOWN_LIVED ETHR_RWMUTEX_UNKNOWN_LIVED +typedef ethr_rwmutex_opt erts_rwmtx_opt_t; + typedef ethr_tsd_key erts_tsd_key_t; -typedef ethr_gate erts_gate_t; +typedef ethr_ts_event erts_tse_t; typedef ethr_atomic_t erts_atomic_t; /* spinlock */ @@ -101,36 +120,41 @@ typedef ethr_timeval erts_thr_timeval_t; __decl_noreturn void __noreturn erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ -#ifdef ERTS_ENABLE_LOCK_CHECK -#define ERTS_REC_MTX_INITER \ - {ETHR_REC_MUTEX_INITER, \ - ERTS_LC_LOCK_INIT(-1,THE_NON_VALUE,ERTS_LC_FLG_LT_MUTEX)} -#define ERTS_MTX_INITER \ - {ETHR_MUTEX_INITER, \ - ERTS_LC_LOCK_INIT(-1, THE_NON_VALUE, ERTS_LC_FLG_LT_MUTEX)} -#else -#define ERTS_REC_MTX_INITER {ETHR_REC_MUTEX_INITER} -#define ERTS_MTX_INITER {ETHR_MUTEX_INITER} -#endif -#define ERTS_CND_INITER ETHR_COND_INITER #define ERTS_THR_INIT_DATA_DEF_INITER ETHR_INIT_DATA_DEFAULT_INITER +#define ERTS_THR_LATE_INIT_DATA_DEF_INITER \ + ETHR_LATE_INIT_DATA_DEFAULT_INITER #ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT # define ERTS_HAVE_REC_MTX_INIT ETHR_HAVE_ETHR_REC_MUTEX_INIT #endif - #else /* #ifdef USE_THREADS */ +#define ERTS_THR_MEMORY_BARRIER + #define ERTS_THR_OPTS_DEFAULT_INITER 0 typedef int erts_thr_opts_t; typedef int erts_thr_init_data_t; +typedef int erts_thr_late_init_data_t; typedef int erts_tid_t; typedef int erts_mtx_t; typedef int erts_cnd_t; +#define ERTS_RWMTX_OPT_DEFAULT_INITER {0} +#define ERTS_RWMTX_TYPE_NORMAL 0 +#define ERTS_RWMTX_TYPE_FREQUENT_READ 0 +#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0 +#define ERTS_RWMTX_LONG_LIVED 0 +#define ERTS_RWMTX_SHORT_LIVED 0 +#define ERTS_RWMTX_UNKNOWN_LIVED 0 +typedef struct { + char type; + char lived; + int main_spincount; + int aux_spincount; +} erts_rwmtx_opt_t; typedef int erts_rwmtx_t; typedef int erts_tsd_key_t; -typedef int erts_gate_t; +typedef int erts_tse_t; typedef long erts_atomic_t; #if __GNUC__ > 2 typedef struct { } erts_spinlock_t; @@ -144,7 +168,6 @@ typedef struct { long tv_nsec; } erts_thr_timeval_t; -#define ERTS_REC_MTX_INITER 0 #define ERTS_MTX_INITER 0 #define ERTS_CND_INITER 0 #define ERTS_THR_INIT_DATA_DEF_INITER 0 @@ -154,6 +177,7 @@ typedef struct { #endif /* #ifdef USE_THREADS */ ERTS_GLB_INLINE void erts_thr_init(erts_thr_init_data_t *id); +ERTS_GLB_INLINE void erts_thr_late_init(erts_thr_late_init_data_t *id); ERTS_GLB_INLINE void erts_thr_create(erts_tid_t *tid, void * (*func)(void *), void *arg, erts_thr_opts_t *opts); ERTS_GLB_INLINE void erts_thr_join(erts_tid_t tid, void **thr_res); @@ -162,9 +186,6 @@ ERTS_GLB_INLINE void erts_thr_exit(void *res); ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void)); ERTS_GLB_INLINE erts_tid_t erts_thr_self(void); ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y); -#ifdef ERTS_HAVE_REC_MTX_INIT -ERTS_GLB_INLINE void erts_rec_mtx_init(erts_mtx_t *mtx); -#endif ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra); ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt); ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx, @@ -173,8 +194,6 @@ ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx, ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx); -ERTS_GLB_INLINE void erts_mtx_set_forksafe(erts_mtx_t *mtx); -ERTS_GLB_INLINE void erts_mtx_unset_forksafe(erts_mtx_t *mtx); ERTS_GLB_INLINE int erts_mtx_trylock(erts_mtx_t *mtx); #ifdef ERTS_ENABLE_LOCK_COUNT ERTS_GLB_INLINE void erts_mtx_lock_x(erts_mtx_t *mtx, char *file, unsigned int line); @@ -188,9 +207,17 @@ ERTS_GLB_INLINE void erts_cnd_destroy(erts_cnd_t *cnd); ERTS_GLB_INLINE void erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx); ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd); ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd); +ERTS_GLB_INLINE void erts_rwmtx_set_reader_group(int no); +ERTS_GLB_INLINE void erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx, + erts_rwmtx_opt_t *opt, + char *name, + Eterm extra); ERTS_GLB_INLINE void erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, char *name, Eterm extra); +ERTS_GLB_INLINE void erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, + erts_rwmtx_opt_t *opt, + char *name); ERTS_GLB_INLINE void erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name); ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx); @@ -224,6 +251,20 @@ ERTS_GLB_INLINE long erts_atomic_cmpxchg(erts_atomic_t *xchgp, long expected); ERTS_GLB_INLINE long erts_atomic_bor(erts_atomic_t *var, long mask); ERTS_GLB_INLINE long erts_atomic_band(erts_atomic_t *var, long mask); +ERTS_GLB_INLINE long erts_atomic_read_acqb(erts_atomic_t *var); +ERTS_GLB_INLINE void erts_atomic_set_relb(erts_atomic_t *var, long i); +ERTS_GLB_INLINE void erts_atomic_dec_relb(erts_atomic_t *decp); +ERTS_GLB_INLINE long erts_atomic_dectest_relb(erts_atomic_t *decp); +ERTS_GLB_INLINE long erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp, + long new, + long exp); +ERTS_GLB_INLINE long erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp, + long new, + long exp); +ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock, + char *name, + Eterm extra, + Uint16 opt); ERTS_GLB_INLINE void erts_spinlock_init_x(erts_spinlock_t *lock, char *name, Eterm extra); @@ -259,12 +300,16 @@ ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp); ERTS_GLB_INLINE void erts_tsd_key_delete(erts_tsd_key_t key); ERTS_GLB_INLINE void erts_tsd_set(erts_tsd_key_t key, void *value); ERTS_GLB_INLINE void * erts_tsd_get(erts_tsd_key_t key); -ERTS_GLB_INLINE void erts_gate_init(erts_gate_t *gp); -ERTS_GLB_INLINE void erts_gate_destroy(erts_gate_t *gp); -ERTS_GLB_INLINE void erts_gate_close(erts_gate_t *gp); -ERTS_GLB_INLINE void erts_gate_let_through(erts_gate_t *gp, unsigned no); -ERTS_GLB_INLINE void erts_gate_wait(erts_gate_t *gp); -ERTS_GLB_INLINE void erts_gate_swait(erts_gate_t *gp, int spincount); +ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void); +ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep); +ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep); +ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep); +ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep); +ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount); +ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep); +ERTS_GLB_INLINE void erts_thr_set_main_status(int, int); +ERTS_GLB_INLINE int erts_thr_get_main_status(void); +ERTS_GLB_INLINE void erts_thr_yield(void); #ifdef ETHR_HAVE_ETHR_SIG_FUNCS #define ERTS_THR_HAVE_SIG_FUNCS 1 @@ -286,15 +331,21 @@ erts_thr_init(erts_thr_init_data_t *id) } ERTS_GLB_INLINE void +erts_thr_late_init(erts_thr_late_init_data_t *id) +{ +#ifdef USE_THREADS + int res = ethr_late_init(id); + if (res) + erts_thr_fatal_error(res, "complete initialization of thread library"); +#endif +} + +ERTS_GLB_INLINE void erts_thr_create(erts_tid_t *tid, void * (*func)(void *), void *arg, erts_thr_opts_t *opts) { #ifdef USE_THREADS -#ifdef ERTS_ENABLE_LOCK_COUNT - int res = erts_lcnt_thr_create(tid, func, arg, opts); -#else int res = ethr_thr_create(tid, func, arg, opts); -#endif if (res) erts_thr_fatal_error(res, "create thread"); #endif @@ -362,20 +413,6 @@ erts_equal_tids(erts_tid_t x, erts_tid_t y) #endif } - -#ifdef ERTS_HAVE_REC_MTX_INIT -ERTS_GLB_INLINE void -erts_rec_mtx_init(erts_mtx_t *mtx) -{ -#ifdef USE_THREADS - int res = ethr_rec_mutex_init(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "initialize recursive mutex"); -#endif -} -#endif - - ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra) { @@ -422,9 +459,7 @@ erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); #endif - res = ethr_mutex_lock(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "lock mutex"); + ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &mtx->lc); #endif @@ -463,9 +498,7 @@ erts_mtx_init_locked(erts_mtx_t *mtx, char *name) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_init_lock(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX); #endif - res = ethr_mutex_lock(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "lock mutex"); + ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &mtx->lc); #endif @@ -492,26 +525,6 @@ erts_mtx_destroy(erts_mtx_t *mtx) #endif } -ERTS_GLB_INLINE void -erts_mtx_set_forksafe(erts_mtx_t *mtx) -{ -#ifdef USE_THREADS - int res = ethr_mutex_set_forksafe(&mtx->mtx); - if (res != 0 && res != ENOTSUP) - erts_thr_fatal_error(res, "set mutex forksafe"); -#endif -} - -ERTS_GLB_INLINE void -erts_mtx_unset_forksafe(erts_mtx_t *mtx) -{ -#ifdef USE_THREADS - int res = ethr_mutex_unset_forksafe(&mtx->mtx); - if (res != 0 && res != ENOTSUP) - erts_thr_fatal_error(res, "unset mutex forksafe"); -#endif -} - ERTS_GLB_INLINE int erts_mtx_trylock(erts_mtx_t *mtx) { @@ -531,11 +544,7 @@ erts_mtx_trylock(erts_mtx_t *mtx) #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock(&mtx->lcnt, res); -#endif - - if (res != 0 && res != EBUSY) - erts_thr_fatal_error(res, "try lock mutex"); - +#endif return res; #else return 0; @@ -551,19 +560,16 @@ erts_mtx_lock(erts_mtx_t *mtx) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock(&mtx->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock(&mtx->lcnt); #endif - res = ethr_mutex_lock(&mtx->mtx); + ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&mtx->lcnt, file, line); #endif - if (res) - erts_thr_fatal_error(res, "lock mutex"); #endif } @@ -571,16 +577,13 @@ ERTS_GLB_INLINE void erts_mtx_unlock(erts_mtx_t *mtx) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock(&mtx->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock(&mtx->lcnt); #endif - res = ethr_mutex_unlock(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "unlock mutex"); + ethr_mutex_unlock(&mtx->mtx); #endif } @@ -648,9 +651,7 @@ ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd) { #ifdef USE_THREADS - int res = ethr_cond_signal(cnd); - if (res) - erts_thr_fatal_error(res, "signal on condition variable"); + ethr_cond_signal(cnd); #endif } @@ -659,19 +660,34 @@ ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd) { #ifdef USE_THREADS - int res = ethr_cond_broadcast(cnd); - if (res) - erts_thr_fatal_error(res, "broadcast on condition variable"); + ethr_cond_broadcast(cnd); #endif } /* rwmutex */ ERTS_GLB_INLINE void -erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, char *name, Eterm extra) +erts_rwmtx_set_reader_group(int no) { #ifdef USE_THREADS - int res = ethr_rwmutex_init(&rwmtx->rwmtx); + int res; +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_check_no_locked_of_type(ERTS_LC_FLG_LT_RWMUTEX); +#endif + res = ethr_rwmutex_set_reader_group(no); + if (res != 0) + erts_thr_fatal_error(res, "set reader group"); +#endif +} + +ERTS_GLB_INLINE void +erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx, + erts_rwmtx_opt_t *opt, + char *name, + Eterm extra) +{ +#ifdef USE_THREADS + int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt); if (res != 0) erts_thr_fatal_error(res, "initialize rwmutex"); #ifdef ERTS_ENABLE_LOCK_CHECK @@ -684,10 +700,20 @@ erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, char *name, Eterm extra) } ERTS_GLB_INLINE void -erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name) +erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, + char *name, + Eterm extra) +{ + erts_rwmtx_init_opt_x(rwmtx, NULL, name, extra); +} + +ERTS_GLB_INLINE void +erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, + erts_rwmtx_opt_t *opt, + char *name) { #ifdef USE_THREADS - int res = ethr_rwmutex_init(&rwmtx->rwmtx); + int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt); if (res != 0) erts_thr_fatal_error(res, "initialize rwmutex"); #ifdef ERTS_ENABLE_LOCK_CHECK @@ -700,6 +726,12 @@ erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name) } ERTS_GLB_INLINE void +erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name) +{ + erts_rwmtx_init_opt(rwmtx, NULL, name); +} + +ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx) { #ifdef USE_THREADS @@ -736,9 +768,6 @@ erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ); #endif - - if (res != 0 && res != EBUSY) - erts_thr_fatal_error(res, "try read lock rwmutex"); return res; #else @@ -754,19 +783,16 @@ erts_rwmtx_rlock(erts_rwmtx_t *rwmtx) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ); #endif - res = ethr_rwmutex_rlock(&rwmtx->rwmtx); + ethr_rwmutex_rlock(&rwmtx->rwmtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line); #endif - if (res != 0) - erts_thr_fatal_error(res, "read lock rwmutex"); #endif } @@ -774,16 +800,13 @@ ERTS_GLB_INLINE void erts_rwmtx_runlock(erts_rwmtx_t *rwmtx) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ); #endif - res = ethr_rwmutex_runlock(&rwmtx->rwmtx); - if (res != 0) - erts_thr_fatal_error(res, "read unlock rwmutex"); + ethr_rwmutex_runlock(&rwmtx->rwmtx); #endif } @@ -808,9 +831,6 @@ erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ_WRITE); #endif - - if (res != 0 && res != EBUSY) - erts_thr_fatal_error(res, "try write lock rwmutex"); return res; #else @@ -826,19 +846,16 @@ erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif - res = ethr_rwmutex_rwlock(&rwmtx->rwmtx); + ethr_rwmutex_rwlock(&rwmtx->rwmtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line); #endif - if (res != 0) - erts_thr_fatal_error(res, "write lock rwmutex"); #endif } @@ -846,16 +863,13 @@ ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif - res = ethr_rwmutex_rwunlock(&rwmtx->rwmtx); - if (res != 0) - erts_thr_fatal_error(res, "write unlock rwmutex"); + ethr_rwmutex_rwunlock(&rwmtx->rwmtx); #endif } @@ -917,9 +931,7 @@ ERTS_GLB_INLINE void erts_atomic_init(erts_atomic_t *var, long i) { #ifdef USE_THREADS - int res = ethr_atomic_init(var, i); - if (res) - erts_thr_fatal_error(res, "perform atomic init"); + ethr_atomic_init(var, i); #else *var = i; #endif @@ -929,9 +941,7 @@ ERTS_GLB_INLINE void erts_atomic_set(erts_atomic_t *var, long i) { #ifdef USE_THREADS - int res = ethr_atomic_set(var, i); - if (res) - erts_thr_fatal_error(res, "perform atomic set"); + ethr_atomic_set(var, i); #else *var = i; #endif @@ -941,11 +951,7 @@ ERTS_GLB_INLINE long erts_atomic_read(erts_atomic_t *var) { #ifdef USE_THREADS - long i; - int res = ethr_atomic_read(var, &i); - if (res) - erts_thr_fatal_error(res, "perform atomic read"); - return i; + return ethr_atomic_read(var); #else return *var; #endif @@ -955,11 +961,7 @@ ERTS_GLB_INLINE long erts_atomic_inctest(erts_atomic_t *incp) { #ifdef USE_THREADS - long test; - int res = ethr_atomic_inctest(incp, &test); - if (res) - erts_thr_fatal_error(res, "perform atomic increment and test"); - return test; + return ethr_atomic_inc_read(incp); #else return ++(*incp); #endif @@ -969,11 +971,7 @@ ERTS_GLB_INLINE long erts_atomic_dectest(erts_atomic_t *decp) { #ifdef USE_THREADS - long test; - int res = ethr_atomic_dectest(decp, &test); - if (res) - erts_thr_fatal_error(res, "perform atomic decrement and test"); - return test; + return ethr_atomic_dec_read(decp); #else return --(*decp); #endif @@ -983,9 +981,7 @@ ERTS_GLB_INLINE void erts_atomic_inc(erts_atomic_t *incp) { #ifdef USE_THREADS - int res = ethr_atomic_inc(incp); - if (res) - erts_thr_fatal_error(res, "perform atomic increment"); + ethr_atomic_inc(incp); #else ++(*incp); #endif @@ -995,9 +991,7 @@ ERTS_GLB_INLINE void erts_atomic_dec(erts_atomic_t *decp) { #ifdef USE_THREADS - int res = ethr_atomic_dec(decp); - if (res) - erts_thr_fatal_error(res, "perform atomic decrement"); + ethr_atomic_dec(decp); #else --(*decp); #endif @@ -1007,11 +1001,7 @@ ERTS_GLB_INLINE long erts_atomic_addtest(erts_atomic_t *addp, long i) { #ifdef USE_THREADS - long test; - int res = ethr_atomic_addtest(addp, i, &test); - if (res) - erts_thr_fatal_error(res, "perform atomic addition and test"); - return test; + return ethr_atomic_add_read(addp, i); #else return *addp += i; #endif @@ -1021,9 +1011,7 @@ ERTS_GLB_INLINE void erts_atomic_add(erts_atomic_t *addp, long i) { #ifdef USE_THREADS - int res = ethr_atomic_add(addp, i); - if (res) - erts_thr_fatal_error(res, "perform atomic addition"); + ethr_atomic_add(addp, i); #else *addp += i; #endif @@ -1034,9 +1022,7 @@ erts_atomic_xchg(erts_atomic_t *xchgp, long new) { long old; #ifdef USE_THREADS - int res = ethr_atomic_xchg(xchgp, new, &old); - if (res) - erts_thr_fatal_error(res, "perform atomic exchange"); + return ethr_atomic_xchg(xchgp, new); #else old = *xchgp; *xchgp = new; @@ -1048,11 +1034,7 @@ ERTS_GLB_INLINE long erts_atomic_cmpxchg(erts_atomic_t *xchgp, long new, long expected) { #ifdef USE_THREADS - long old; - int res = ethr_atomic_cmpxchg(xchgp, new, expected, &old); - if (ERTS_UNLIKELY(res != 0)) - erts_thr_fatal_error(res, "perform atomic exchange"); - return old; + return ethr_atomic_cmpxchg(xchgp, new, expected); #else long old = *xchgp; if (old == expected) @@ -1064,31 +1046,95 @@ erts_atomic_cmpxchg(erts_atomic_t *xchgp, long new, long expected) ERTS_GLB_INLINE long erts_atomic_bor(erts_atomic_t *var, long mask) { - long old; #ifdef USE_THREADS - int res = ethr_atomic_or_old(var, mask, &old); - if (res != 0) - erts_thr_fatal_error(res, "perform atomic bitwise or"); + return ethr_atomic_read_bor(var, mask); #else + long old; old = *var; *var |= mask; -#endif return old; +#endif } ERTS_GLB_INLINE long erts_atomic_band(erts_atomic_t *var, long mask) { - long old; #ifdef USE_THREADS - int res = ethr_atomic_and_old(var, mask, &old); - if (res != 0) - erts_thr_fatal_error(res, "perform atomic bitwise and"); + return ethr_atomic_read_band(var, mask); #else + long old; old = *var; *var &= mask; + return old; #endif +} + +ERTS_GLB_INLINE long +erts_atomic_read_acqb(erts_atomic_t *var) +{ +#ifdef USE_THREADS + return ethr_atomic_read_acqb(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic_set_relb(erts_atomic_t *var, long i) +{ +#ifdef USE_THREADS + ethr_atomic_set_relb(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic_dec_relb(erts_atomic_t *decp) +{ +#ifdef USE_THREADS + ethr_atomic_dec_relb(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE long +erts_atomic_dectest_relb(erts_atomic_t *decp) +{ +#ifdef USE_THREADS + return ethr_atomic_dec_read_relb(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE long erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp, + long new, + long exp) +{ +#ifdef USE_THREADS + return ethr_atomic_cmpxchg_acqb(xchgp, new, exp); +#else + long old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE long erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp, + long new, + long exp) +{ +#ifdef USE_THREADS + return ethr_atomic_cmpxchg_relb(xchgp, new, exp); +#else + long old = *xchgp; + if (old == exp) + *xchgp = new; return old; +#endif } /* spinlock */ @@ -1112,6 +1158,26 @@ erts_spinlock_init_x(erts_spinlock_t *lock, char *name, Eterm extra) } ERTS_GLB_INLINE void +erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, Eterm extra, + Uint16 opt) +{ +#ifdef USE_THREADS + int res = ethr_spinlock_init(&lock->slck); + if (res) + erts_thr_fatal_error(res, "init spinlock"); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra); +#endif +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK|opt, extra); +#endif +#else + (void)lock; +#endif +} + + +ERTS_GLB_INLINE void erts_spinlock_init(erts_spinlock_t *lock, char *name) { #ifdef USE_THREADS @@ -1152,16 +1218,13 @@ ERTS_GLB_INLINE void erts_spin_unlock(erts_spinlock_t *lock) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock(&lock->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock(&lock->lcnt); #endif - res = ethr_spin_unlock(&lock->slck); - if (res) - erts_thr_fatal_error(res, "release spin lock"); + ethr_spin_unlock(&lock->slck); #else (void)lock; #endif @@ -1175,19 +1238,16 @@ erts_spin_lock(erts_spinlock_t *lock) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock(&lock->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock(&lock->lcnt); #endif - res = ethr_spin_lock(&lock->slck); + ethr_spin_lock(&lock->slck); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&lock->lcnt, file, line); #endif - if (res) - erts_thr_fatal_error(res, "take spin lock"); #else (void)lock; #endif @@ -1268,16 +1328,13 @@ ERTS_GLB_INLINE void erts_read_unlock(erts_rwlock_t *lock) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ); #endif - res = ethr_read_unlock(&lock->rwlck); - if (res) - erts_thr_fatal_error(res, "release read lock"); + ethr_read_unlock(&lock->rwlck); #else (void)lock; #endif @@ -1291,19 +1348,16 @@ erts_read_lock(erts_rwlock_t *lock) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ); #endif - res = ethr_read_lock(&lock->rwlck); + ethr_read_lock(&lock->rwlck); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&lock->lcnt, file, line); #endif - if (res) - erts_thr_fatal_error(res, "take read lock"); #else (void)lock; #endif @@ -1313,16 +1367,13 @@ ERTS_GLB_INLINE void erts_write_unlock(erts_rwlock_t *lock) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif - res = ethr_write_unlock(&lock->rwlck); - if (res) - erts_thr_fatal_error(res, "release write lock"); + ethr_write_unlock(&lock->rwlck); #else (void)lock; #endif @@ -1336,19 +1387,16 @@ erts_write_lock(erts_rwlock_t *lock) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif - res = ethr_write_lock(&lock->rwlck); + ethr_write_lock(&lock->rwlck); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&lock->lcnt, file, line); #endif - if (res) - erts_thr_fatal_error(res, "take write lock"); #else (void)lock; #endif @@ -1432,66 +1480,95 @@ erts_tsd_get(erts_tsd_key_t key) #endif } -ERTS_GLB_INLINE void -erts_gate_init(erts_gate_t *gp) +ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void) { #ifdef USE_THREADS - int res = ethr_gate_init((ethr_gate *) gp); - if (res != 0) - erts_thr_fatal_error(res, "initialize gate"); + return (erts_tse_t *) ethr_get_ts_event(); +#else + return (erts_tse_t *) NULL; #endif } -ERTS_GLB_INLINE void -erts_gate_destroy(erts_gate_t *gp) +ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep) { #ifdef USE_THREADS - int res = ethr_gate_destroy((ethr_gate *) gp); - if (res != 0) - erts_thr_fatal_error(res, "destroy gate"); + ethr_leave_ts_event(ep); #endif } -ERTS_GLB_INLINE void -erts_gate_close(erts_gate_t *gp) +ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep) { #ifdef USE_THREADS - int res = ethr_gate_close((ethr_gate *) gp); - if (res != 0) - erts_thr_fatal_error(res, "close gate"); + ethr_event_set(&((ethr_ts_event *) ep)->event); #endif } -ERTS_GLB_INLINE void -erts_gate_let_through(erts_gate_t *gp, unsigned no) +ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep) { #ifdef USE_THREADS - int res = ethr_gate_let_through((ethr_gate *) gp, no); - if (res != 0) - erts_thr_fatal_error(res, "let through gate"); + ethr_event_reset(&((ethr_ts_event *) ep)->event); #endif } -ERTS_GLB_INLINE void -erts_gate_wait(erts_gate_t *gp) +ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep) { #ifdef USE_THREADS - int res = ethr_gate_wait((ethr_gate *) gp); + return ethr_event_wait(&((ethr_ts_event *) ep)->event); +#else + return ENOTSUP; +#endif +} + +ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount) +{ +#ifdef USE_THREADS + return ethr_event_swait(&((ethr_ts_event *) ep)->event, spincount); +#else + return ENOTSUP; +#endif +} + +ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep) +{ +#ifdef USE_THREADS + return (ep->iflgs & ETHR_TS_EV_TMP) == ETHR_TS_EV_TMP; +#else + return 0; +#endif +} + +ERTS_GLB_INLINE void erts_thr_set_main_status(int on, int no) +{ +#ifdef USE_THREADS + int res = ethr_set_main_thr_status(on, no); if (res != 0) - erts_thr_fatal_error(res, "wait on gate"); + erts_thr_fatal_error(res, "set thread main status"); #endif } -ERTS_GLB_INLINE void -erts_gate_swait(erts_gate_t *gp, int spincount) +ERTS_GLB_INLINE int erts_thr_get_main_status(void) { #ifdef USE_THREADS - int res = ethr_gate_swait((ethr_gate *) gp, spincount); + int main_status; + int res = ethr_get_main_thr_status(&main_status); if (res != 0) - erts_thr_fatal_error(res, "swait on gate"); + erts_thr_fatal_error(res, "get thread main status"); + return main_status; +#else + return 1; #endif } +ERTS_GLB_INLINE void erts_thr_yield(void) +{ +#ifdef USE_THREADS + int res = ETHR_YIELD(); + if (res != 0) + erts_thr_fatal_error(res, "yield"); +#endif +} + + #ifdef ETHR_HAVE_ETHR_SIG_FUNCS ERTS_GLB_INLINE void diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index c15f85f8f1..7b8706ea13 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -650,6 +650,22 @@ local_to_univ(Sint *year, Sint *month, Sint *day, t.tm_sec = *second; t.tm_isdst = isdst; the_clock = mktime(&t); + if (the_clock == -1) { + if (isdst) { + /* If this is a timezone without DST and the OS (correctly) + refuses to give us a DST time, we simulate the Linux/Solaris + behaviour of giving the same data as if is_dst was not set. */ + t.tm_isdst = 0; + the_clock = mktime(&t); + if (the_clock == -1) { + /* Failed anyway, something else is bad - will be a badarg */ + return 0; + } + } else { + /* Something else is the matter, badarg. */ + return 0; + } + } #ifdef HAVE_GMTIME_R gmtime_r(&the_clock, (tm = &tmbuf)); #else @@ -663,6 +679,10 @@ local_to_univ(Sint *year, Sint *month, Sint *day, *second = tm->tm_sec; return 1; } +#if defined(HAVE_POSIX2TIME) && defined(HAVE_DECL_POSIX2TIME) && \ + !HAVE_DECL_POSIX2TIME +extern time_t posix2time(time_t); +#endif int univ_to_local(Sint *year, Sint *month, Sint *day, diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 2842c2361a..3043bb1e8c 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -43,8 +43,9 @@ #undef DEBUG_PRINTOUTS #endif -extern Eterm beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ -extern Eterm beam_return_trace[1]; /* OpCode(i_return_trace) */ +extern Eterm beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ +extern Eterm beam_return_trace[1]; /* OpCode(i_return_trace) */ +extern Eterm beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ /* Pseudo export entries. Never filled in with data, only used to yield unique pointers of the correct type. */ @@ -397,11 +398,13 @@ WRITE_SYS_MSG_TO_PORT(Eterm unused_to, */ static void do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) { - Eterm local_heap[4+5+5]; +#define LOCAL_HEAP_SIZE (4+5+5) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); Eterm message; Eterm *hp; Eterm mfarity; + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); ASSERT(is_pid(pid)); ASSERT(is_tuple(timestamp)); ASSERT(*tuple_val(timestamp) == make_arityval(3)); @@ -426,6 +429,8 @@ do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) { pid, SYS_MSG_TYPE_UNDEFINED, message); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE } #endif @@ -442,7 +447,9 @@ send_to_port(Process *c_p, Eterm message, Eterm *tracer_pid, Uint *tracee_flags) { Port* trace_port; #ifndef ERTS_SMP - Eterm ts, local_heap[4], *hp; +#define LOCAL_HEAP_SIZE (4) + Eterm ts, *hp; + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); #endif ASSERT(is_internal_port(*tracer_pid)); @@ -486,6 +493,8 @@ send_to_port(Process *c_p, Eterm message, * (e.g. getting_linked) need not be the current process. That other * process might not have timestamps enabled. */ + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + if (*tracee_flags & F_TIMESTAMP) { ASSERT(is_tuple(message)); hp = tuple_val(message); @@ -522,6 +531,8 @@ send_to_port(Process *c_p, Eterm message, */ do_send_schedfix_to_port(trace_port, c_p->id, ts); } + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE #endif } @@ -589,7 +600,10 @@ seq_trace_send_to_port(Process *c_p, { Port* trace_port; #ifndef ERTS_SMP - Eterm ts, local_heap[4], *hp; + Eterm ts, *hp; +#define LOCAL_HEAP_SIZE (4) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); #endif ASSERT(is_internal_port(seq_tracer)); @@ -607,6 +621,9 @@ seq_trace_send_to_port(Process *c_p, if (INVALID_TRACER_PORT(trace_port, seq_tracer)) { invalid_tracer_port: system_seq_tracer = am_false; +#ifndef ERTS_SMP + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#endif return; } @@ -620,6 +637,7 @@ seq_trace_send_to_port(Process *c_p, message); #ifndef ERTS_SMP + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); return; } /* Make a fake schedule only if the current process is traced @@ -660,6 +678,8 @@ seq_trace_send_to_port(Process *c_p, */ do_send_schedfix_to_port(trace_port, c_p->id, ts); } + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE #endif } @@ -719,7 +739,8 @@ send_to_tracer(Process *tracee, static void trace_sched_aux(Process *p, Eterm what, int never_fake_sched) { - Eterm local_heap[5+4+1+TS_HEAP_WORDS]; +#define LOCAL_HEAP_SIZE (5+4+1+TS_HEAP_WORDS) + DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p); Eterm tmp, mess, *hp; ErlHeapFragment *bp = NULL; ErlOffHeap *off_heap; @@ -768,8 +789,10 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched) curr_func = p->current != NULL; } + UseTmpHeap(LOCAL_HEAP_SIZE,p); + if (to_port) - hp = &local_heap[0]; + hp = local_heap; else { Uint size = 5; if (curr_func) @@ -802,6 +825,8 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched) } send_to_tracer(p, tracer_ref, mess, &hp, bp, no_fake_sched); + UnUseTmpHeap(LOCAL_HEAP_SIZE,p); +#undef LOCAL_HEAP_SIZE } /* Send {trace_ts, Pid, What, {Mod, Func, Arity}, Timestamp} @@ -848,7 +873,10 @@ trace_send(Process *p, Eterm to, Eterm msg) } if (is_internal_port(p->tracer_proc)) { - Eterm local_heap[11]; +#define LOCAL_HEAP_SIZE (11) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; mess = TUPLE5(hp, am_trace, p->id, operation, msg, to); hp += 6; @@ -857,6 +885,8 @@ trace_send(Process *p, Eterm to, Eterm msg) hp = patch_ts(mess, hp); } send_to_port(p, mess, &p->tracer_proc, &p->trace_flags); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } else { Uint need; @@ -908,7 +938,10 @@ trace_receive(Process *rp, Eterm msg) Eterm* hp; if (is_internal_port(rp->tracer_proc)) { - Eterm local_heap[10]; +#define LOCAL_HEAP_SIZE (10) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; mess = TUPLE4(hp, am_trace, rp->id, am_receive, msg); hp += 5; @@ -917,6 +950,8 @@ trace_receive(Process *rp, Eterm msg) hp = patch_ts(mess, hp); } send_to_port(rp, mess, &rp->tracer_proc, &rp->trace_flags); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } else { Uint hsz; @@ -1018,7 +1053,10 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, } if (is_internal_port(seq_tracer)) { - Eterm local_heap[64]; +#define LOCAL_HEAP_SIZE (64) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; label = SEQ_TRACE_T_LABEL(token); lastcnt_serial = TUPLE2(hp, SEQ_TRACE_T_LASTCNT(token), @@ -1043,6 +1081,8 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, mess = TUPLE4(hp, am_seq_trace, label, mess, ts); seq_trace_send_to_port(process, seq_tracer, mess, ts); } + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } else { #ifndef ERTS_SMP @@ -1143,14 +1183,18 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, * or {trace, Pid, return_to, {Mod, Func, Arity}} */ void -erts_trace_return_to(Process *p, Uint *pc) +erts_trace_return_to(Process *p, BeamInstr *pc) { +#define LOCAL_HEAP_SIZE (4+5+5) Eterm* hp; Eterm mfa; Eterm mess; - Eterm local_heap[4+5+5]; + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + + BeamInstr *code_ptr = find_function_from_pc(pc); + - Eterm *code_ptr = find_function_from_pc(pc); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); hp = local_heap; @@ -1196,6 +1240,8 @@ erts_trace_return_to(Process *p, Uint *pc) mess = copy_struct(mess, size, &hp, off_heap); ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, mess, bp); } + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } @@ -1204,7 +1250,7 @@ erts_trace_return_to(Process *p, Uint *pc) * or {trace, Pid, return_from, {Mod, Name, Arity}, Retval} */ void -erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid) +erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) { Eterm* hp; Eterm mfa; @@ -1262,7 +1308,9 @@ erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid) arity = fi[2]; if (is_internal_port(*tracer_pid)) { - Eterm local_heap[4+6+5]; +#define LOCAL_HEAP_SIZE (4+6+5) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); hp = local_heap; mfa = TUPLE3(hp, mod, name, make_small(arity)); hp += 4; @@ -1273,6 +1321,8 @@ erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid) hp = patch_ts(mess, hp); } send_to_port(p, mess, tracer_pid, tracee_flags); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } else { ErlHeapFragment *bp; @@ -1331,7 +1381,7 @@ erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid) * Where Class is atomic but Value is any term. */ void -erts_trace_exception(Process* p, Eterm mfa[3], Eterm class, Eterm value, +erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, Eterm *tracer_pid) { Eterm* hp; @@ -1385,21 +1435,26 @@ erts_trace_exception(Process* p, Eterm mfa[3], Eterm class, Eterm value, } if (is_internal_port(*tracer_pid)) { - Eterm local_heap[4+3+6+5]; +#define LOCAL_HEAP_SIZE (4+3+6+5) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; - mfa_tuple = TUPLE3(hp, mfa[0], mfa[1], make_small(mfa[2])); + mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], make_small((Eterm)mfa[2])); hp += 4; cv = TUPLE2(hp, class, value); hp += 3; mess = TUPLE5(hp, am_trace, p->id, am_exception_from, mfa_tuple, cv); hp += 6; - ASSERT((hp - local_heap)*sizeof(*hp) <= sizeof(local_heap)); + ASSERT((hp - local_heap) <= LOCAL_HEAP_SIZE); erts_smp_mtx_lock(&smq_mtx); if (*tracee_flags & F_TIMESTAMP) { hp = patch_ts(mess, hp); /* hp += 5 */ - ASSERT((hp - local_heap)*sizeof(*hp) == sizeof(local_heap)); + ASSERT((hp - local_heap) == LOCAL_HEAP_SIZE); } send_to_port(p, mess, tracer_pid, tracee_flags); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } else { ErlHeapFragment *bp; @@ -1431,7 +1486,7 @@ erts_trace_exception(Process* p, Eterm mfa[3], Eterm class, Eterm value, * Build the trace tuple and put it into receive queue of the tracer process. */ - mfa_tuple = TUPLE3(hp, mfa[0], mfa[1], make_small(mfa[2])); + mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], make_small((Eterm) mfa[2])); hp += 4; value = copy_struct(value, value_size, &hp, off_heap); cv = TUPLE2(hp, class, value); @@ -1468,7 +1523,7 @@ erts_trace_exception(Process* p, Eterm mfa[3], Eterm class, Eterm value, * if it is a pid or port we do a meta trace. */ Uint32 -erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, +erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, Eterm* args, int local, Eterm *tracer_pid) { Eterm* hp; @@ -1483,7 +1538,8 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, Eterm tracee; #endif Eterm transformed_args[MAX_ARG]; - ErlSubBin sub_bin_heap; + DeclareTmpHeap(sub_bin_heap_et,ERL_SUB_BIN_SIZE,p); + ErlSubBin *sub_bin_heap = (ErlSubBin *) sub_bin_heap_et; ASSERT(tracer_pid); if (*tracer_pid == am_true) { @@ -1534,19 +1590,20 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, * such as size_object() and copy_struct(), we must make sure that we * temporarily convert any match contexts to sub binaries. */ - arity = mfa[2]; + arity = (Eterm) mfa[2]; + UseTmpHeap(ERL_SUB_BIN_SIZE,p); #ifdef DEBUG - sub_bin_heap.thing_word = 0; + sub_bin_heap->thing_word = 0; #endif for (i = 0; i < arity; i++) { Eterm arg = args[i]; if (is_boxed(arg) && header_is_bin_matchstate(*boxed_val(arg))) { ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(arg); ErlBinMatchBuffer* mb = &ms->mb; - ErlSubBin* sb = &sub_bin_heap; + ErlSubBin* sb = sub_bin_heap; Uint bit_size; - ASSERT(sub_bin_heap.thing_word == 0); /* At most one of match context */ + ASSERT(sub_bin_heap->thing_word == 0); /* At most one of match context */ bit_size = mb->size - mb->offset; sb->thing_word = HEADER_SUB_BIN; @@ -1564,7 +1621,12 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, args = transformed_args; if (is_internal_port(*tracer_pid)) { +#if HEAP_ON_C_STACK Eterm local_heap[64+MAX_ARG]; +#else + Eterm *local_heap = erts_alloc(ERTS_ALC_T_TEMP_TERM, + sizeof(Eterm)*(64+MAX_ARG)); +#endif hp = local_heap; if (!erts_is_valid_tracer_port(*tracer_pid)) { @@ -1579,6 +1641,10 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, if (is_not_nil(tracee)) erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); #endif +#if !HEAP_ON_C_STACK + erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); +#endif + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return 0; } @@ -1605,6 +1671,10 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, &return_flags); if (is_non_value(pam_result)) { erts_match_set_release_result(p); +#if !HEAP_ON_C_STACK + erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); +#endif + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return 0; } } @@ -1612,16 +1682,28 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, /* Meta trace */ if (pam_result == am_false) { erts_match_set_release_result(p); +#if !HEAP_ON_C_STACK + erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); +#endif + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return return_flags; } } else { /* Non-meta trace */ if (*tracee_flags & F_TRACE_SILENT) { erts_match_set_release_result(p); +#if !HEAP_ON_C_STACK + erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); +#endif + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return 0; } if (pam_result == am_false) { erts_match_set_release_result(p); +#if !HEAP_ON_C_STACK + erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); +#endif + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return return_flags; } if (local && (*tracee_flags & F_TRACE_RETURN_TO)) { @@ -1644,7 +1726,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, hp += 2; } } - mfa_tuple = TUPLE3(hp, mfa[0], mfa[1], mfa_tuple); + mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], mfa_tuple); hp += 4; /* @@ -1664,6 +1746,10 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, send_to_port(p, mess, tracer_pid, tracee_flags); erts_smp_mtx_unlock(&smq_mtx); erts_match_set_release_result(p); +#if !HEAP_ON_C_STACK + erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); +#endif + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return *tracer_pid == NIL ? 0 : return_flags; } else { @@ -1706,6 +1792,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, if (is_not_nil(tracee)) erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); #endif + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return 0; } @@ -1731,6 +1818,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, &return_flags); if (is_non_value(pam_result)) { erts_match_set_release_result(p); + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return 0; } } @@ -1738,16 +1826,19 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, /* Meta trace */ if (pam_result == am_false) { erts_match_set_release_result(p); + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return return_flags; } } else { /* Non-meta trace */ if (*tracee_flags & F_TRACE_SILENT) { erts_match_set_release_result(p); + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return 0; } if (pam_result == am_false) { erts_match_set_release_result(p); + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return return_flags; } if (local && (*tracee_flags & F_TRACE_RETURN_TO)) { @@ -1798,7 +1889,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, hp += 2; } } - mfa_tuple = TUPLE3(hp, mfa[0], mfa[1], mfa_tuple); + mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], mfa_tuple); hp += 4; /* @@ -1831,6 +1922,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, ASSERT(hp == limit); ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return return_flags; } } @@ -1850,8 +1942,13 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) Eterm* hp; int need; + ERTS_SMP_LC_ASSERT((erts_proc_lc_my_proc_locks(t_p) != 0) || erts_is_system_blocked(0)); if (is_internal_port(t_p->tracer_proc)) { - Eterm local_heap[5+5]; +#define LOCAL_HEAP_SIZE (5+5) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + + hp = local_heap; mess = TUPLE4(hp, am_trace, t_p->id, what, data); hp += 5; @@ -1868,6 +1965,8 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) c_p, #endif mess, &t_p->tracer_proc, &t_p->trace_flags); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } else { Eterm tmp; @@ -1919,7 +2018,10 @@ trace_proc_spawn(Process *p, Eterm pid, Eterm* hp; if (is_internal_port(p->tracer_proc)) { - Eterm local_heap[4+6+5]; +#define LOCAL_HEAP_SIZE (4+6+5) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; mfa = TUPLE3(hp, mod, func, args); hp += 4; @@ -1930,6 +2032,8 @@ trace_proc_spawn(Process *p, Eterm pid, hp = patch_ts(mess, hp); } send_to_port(p, mess, &p->tracer_proc, &p->trace_flags); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } else { Eterm tmp; @@ -1991,7 +2095,7 @@ void save_calls(Process *p, Export *e) */ Eterm erts_bif_trace(int bif_index, Process* p, - Eterm arg1, Eterm arg2, Eterm arg3, Uint *I) + Eterm arg1, Eterm arg2, Eterm arg3, BeamInstr *I) { Eterm result; int meta = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_META); @@ -2005,28 +2109,22 @@ erts_bif_trace(int bif_index, Process* p, * no tracing will occur. Doing the whole else branch will * also do nothing, only slower. */ - Eterm (*func)(Process*, Eterm, Eterm, Eterm, Uint*) = bif_table[bif_index].f; + Eterm (*func)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = bif_table[bif_index].f; result = func(p, arg1, arg2, arg3, I); } else { - Eterm (*func)(Process*, Eterm, Eterm, Eterm, Uint*); + Eterm (*func)(Process*, Eterm, Eterm, Eterm, BeamInstr*); Export* ep = bif_export[bif_index]; Uint32 flags = 0, flags_meta = 0; int global = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_GLOBAL); int local = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_LOCAL); + int time = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_CALL_TIME); Eterm meta_tracer_pid = NIL; int applying = (I == &(ep->code[3])); /* Yup, the apply code for a bif * is actually in the * export entry */ - Eterm *cp = p->cp; + BeamInstr *cp = p->cp; -#ifndef _OSE_ Eterm args[3] = {arg1, arg2, arg3}; -#else - Eterm args[3]; - args[0] = arg1; - args[1] = arg2; - args[2] = arg3; -#endif /* * Make continuation pointer OK, it is not during direct BIF calls, @@ -2043,6 +2141,17 @@ erts_bif_trace(int bif_index, Process* p, flags_meta = erts_bif_mtrace(p, ep->code+3, args, local, &meta_tracer_pid); } + if (time) { + BpDataTime *bdt = NULL; + BeamInstr *pc = (BeamInstr *)ep->code+3; + + bdt = (BpDataTime *) erts_get_time_break(p, pc); + ASSERT(bdt); + + if (!bdt->pause) { + erts_trace_time_break(p, pc, bdt, ERTS_BP_CALL_TIME_CALL); + } + } /* Restore original continuation pointer (if changed). */ p->cp = cp; @@ -2051,17 +2160,21 @@ erts_bif_trace(int bif_index, Process* p, result = func(p, arg1, arg2, arg3, I); if (applying && (flags & MATCH_SET_RETURN_TO_TRACE)) { - Uint i_return_trace = beam_return_trace[0]; - Uint i_return_to_trace = beam_return_to_trace[0]; + BeamInstr i_return_trace = beam_return_trace[0]; + BeamInstr i_return_to_trace = beam_return_to_trace[0]; + BeamInstr i_return_time_trace = beam_return_time_trace[0]; Eterm *cpp; /* Maybe advance cp to skip trace stack frames */ for (cpp = p->stop; ; cp = cp_val(*cpp++)) { - ASSERT(is_CP((Eterm) cp)); - if (*cp_val((Eterm) cp) == i_return_trace) { + if (*cp == i_return_trace) { /* Skip stack frame variables */ while (is_not_CP(*cpp)) cpp++; cpp += 2; /* Skip return_trace parameters */ - } else if (*cp_val((Eterm) cp) == i_return_to_trace) { + } else if (*cp == i_return_time_trace) { + /* Skip stack frame variables */ + while (is_not_CP(*cpp)) cpp++; + cpp += 1; /* Skip return_time_trace parameters */ + } else if (*cp == i_return_to_trace) { /* A return_to trace message is going to be generated * by normal means, so we do not have to. */ @@ -2078,7 +2191,8 @@ erts_bif_trace(int bif_index, Process* p, if (reason != TRAP) { Eterm class; Eterm value = p->fvalue; - Eterm nocatch[3]; + DeclareTmpHeapNoproc(nocatch,3); + UseTmpHeapNoproc(3); /* Expand error value like in handle_error() */ if (reason & EXF_ARGLIST) { Eterm *tp; @@ -2126,6 +2240,7 @@ erts_bif_trace(int bif_index, Process* p, } } } + UnUseTmpHeapNoproc(3); if ((flags_meta|flags) & MATCH_SET_EXCEPTION_TRACE) { erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); p->trace_flags |= F_EXCEPTION_TRACE; @@ -2213,15 +2328,19 @@ trace_gc(Process *p, Eterm what) BIN_OLD_VHEAP(p), BIN_OLD_VHEAP_SZ(p) }; - Eterm local_heap[(sizeof(values)/sizeof(Uint)) - *(2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE) - + 5/*4-tuple */ + TS_HEAP_WORDS]; +#define LOCAL_HEAP_SIZE \ + (sizeof(values)/sizeof(Eterm)) * \ + (2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE) + \ + 5/*4-tuple */ + TS_HEAP_WORDS + DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p); #ifdef DEBUG Eterm* limit; #endif ASSERT(sizeof(values)/sizeof(Uint) == sizeof(tags)/sizeof(Eterm)); + UseTmpHeap(LOCAL_HEAP_SIZE,p); + if (is_internal_port(p->tracer_proc)) { hp = local_heap; #ifdef DEBUG @@ -2252,7 +2371,7 @@ trace_gc(Process *p, Eterm what) #ifdef DEBUG limit = hp + size; - ASSERT(size <= sizeof(local_heap)/sizeof(Eterm)); + ASSERT(size <= LOCAL_HEAP_SIZE); #endif msg = erts_bld_atom_uint_2tup_list(&hp, @@ -2275,6 +2394,8 @@ trace_gc(Process *p, Eterm what) else ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, msg, bp); erts_smp_mtx_unlock(&smq_mtx); + UnUseTmpHeap(LOCAL_HEAP_SIZE,p); +#undef LOCAL_HEAP_SIZE } @@ -2465,7 +2586,9 @@ profile_scheduler(Eterm scheduler_id, Eterm state) { Uint Ms, s, us; #ifndef ERTS_SMP - Eterm local_heap[4 + 7]; +#define LOCAL_HEAP_SIZE (4 + 7) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); hp = local_heap; #else ErlHeapFragment *bp; @@ -2498,6 +2621,8 @@ profile_scheduler(Eterm scheduler_id, Eterm state) { #ifndef ERTS_SMP profile_send(msg); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE #else enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp); #endif @@ -2510,7 +2635,10 @@ profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint M Eterm *hp, msg, timestamp; #ifndef ERTS_SMP - Eterm local_heap[4 + 7]; +#define LOCAL_HEAP_SIZE (4 + 7) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; #else ErlHeapFragment *bp; @@ -2528,6 +2656,8 @@ profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint M msg = TUPLE6(hp, am_profile, am_scheduler, scheduler_id, state, no_schedulers, timestamp); hp += 7; #ifndef ERTS_SMP profile_send(msg); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE #else enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp); #endif @@ -2558,7 +2688,10 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { Eterm* hp; if (is_internal_port(p->tracer_proc)) { - Eterm local_heap[5+6]; +#define LOCAL_HEAP_SIZE (5+6) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; mess = TUPLE5(hp, am_trace, calling_pid, am_open, p->id, drv_name); @@ -2569,6 +2702,8 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { } /* No fake schedule */ send_to_port(NULL, mess, &p->tracer_proc, &p->trace_flags); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } else { ErlHeapFragment *bp; @@ -2612,8 +2747,13 @@ trace_port(Port *t_p, Eterm what, Eterm data) { Eterm mess; Eterm* hp; + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) || erts_is_system_blocked(0)); + if (is_internal_port(t_p->tracer_proc)) { - Eterm local_heap[5+5]; +#define LOCAL_HEAP_SIZE (5+5) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; mess = TUPLE4(hp, am_trace, t_p->id, what, data); hp += 5; @@ -2623,6 +2763,8 @@ trace_port(Port *t_p, Eterm what, Eterm data) { } /* No fake schedule */ send_to_port(NULL, mess, &t_p->tracer_proc, &t_p->trace_flags); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } else { ErlHeapFragment *bp; @@ -2674,7 +2816,10 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { Eterm sched_id = am_undefined; if (is_internal_port(p->tracer_proc)) { - Eterm local_heap[5+6]; +#define LOCAL_HEAP_SIZE (5+6) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; if (IS_TRACED_FL(p, F_TRACE_SCHED_NO)) { @@ -2700,6 +2845,8 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { /* No fake scheduling */ send_to_port(NULL, mess, &p->tracer_proc, &p->trace_flags); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } else { ErlHeapFragment *bp; @@ -2750,7 +2897,11 @@ profile_runnable_port(Port *p, Eterm status) { Eterm count = make_small(0); #ifndef ERTS_SMP - Eterm local_heap[4 + 6]; +#define LOCAL_HEAP_SIZE (4 + 6) + + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; #else @@ -2771,6 +2922,8 @@ profile_runnable_port(Port *p, Eterm status) { #ifndef ERTS_SMP profile_send(msg); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE #else enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp); #endif @@ -2785,7 +2938,11 @@ profile_runnable_proc(Process *p, Eterm status){ Eterm where = am_undefined; #ifndef ERTS_SMP - Eterm local_heap[4 + 6 + 4]; +#define LOCAL_HEAP_SIZE (4 + 6 + 4) + + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; #else ErlHeapFragment *bp; @@ -2818,6 +2975,8 @@ profile_runnable_proc(Process *p, Eterm status){ msg = TUPLE5(hp, am_profile, p->id, status, where, timestamp); hp += 6; #ifndef ERTS_SMP profile_send(msg); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE #else enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp); #endif diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index ab5811c70f..d01a3661f9 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2008-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2008-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -90,9 +90,9 @@ void erts_init_unicode(void) am_atom_put("characters_to_utf8_trap",23); characters_to_utf8_trap_exp.code[2] = 3; characters_to_utf8_trap_exp.code[3] = - (Eterm) em_apply_bif; + (BeamInstr) em_apply_bif; characters_to_utf8_trap_exp.code[4] = - (Eterm) &characters_to_utf8_trap; + (BeamInstr) &characters_to_utf8_trap; memset(&characters_to_list_trap_1_exp, 0, sizeof(Export)); characters_to_list_trap_1_exp.address = @@ -102,9 +102,9 @@ void erts_init_unicode(void) am_atom_put("characters_to_list_trap_1",25); characters_to_list_trap_1_exp.code[2] = 3; characters_to_list_trap_1_exp.code[3] = - (Eterm) em_apply_bif; + (BeamInstr) em_apply_bif; characters_to_list_trap_1_exp.code[4] = - (Eterm) &characters_to_list_trap_1; + (BeamInstr) &characters_to_list_trap_1; memset(&characters_to_list_trap_2_exp, 0, sizeof(Export)); characters_to_list_trap_2_exp.address = @@ -114,9 +114,9 @@ void erts_init_unicode(void) am_atom_put("characters_to_list_trap_2",25); characters_to_list_trap_2_exp.code[2] = 3; characters_to_list_trap_2_exp.code[3] = - (Eterm) em_apply_bif; + (BeamInstr) em_apply_bif; characters_to_list_trap_2_exp.code[4] = - (Eterm) &characters_to_list_trap_2; + (BeamInstr) &characters_to_list_trap_2; memset(&characters_to_list_trap_3_exp, 0, sizeof(Export)); @@ -127,9 +127,9 @@ void erts_init_unicode(void) am_atom_put("characters_to_list_trap_3",25); characters_to_list_trap_3_exp.code[2] = 3; characters_to_list_trap_3_exp.code[3] = - (Eterm) em_apply_bif; + (BeamInstr) em_apply_bif; characters_to_list_trap_3_exp.code[4] = - (Eterm) &characters_to_list_trap_3; + (BeamInstr) &characters_to_list_trap_3; memset(&characters_to_list_trap_4_exp, 0, sizeof(Export)); characters_to_list_trap_4_exp.address = @@ -139,9 +139,9 @@ void erts_init_unicode(void) am_atom_put("characters_to_list_trap_4",25); characters_to_list_trap_4_exp.code[2] = 1; characters_to_list_trap_4_exp.code[3] = - (Eterm) em_apply_bif; + (BeamInstr) em_apply_bif; characters_to_list_trap_4_exp.code[4] = - (Eterm) &characters_to_list_trap_4; + (BeamInstr) &characters_to_list_trap_4; c_to_b_int_trap_exportp = erts_export_put(am_unicode,am_characters_to_binary_int,2); c_to_l_int_trap_exportp = erts_export_put(am_unicode,am_characters_to_list_int,2); diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index 50b3e5b61c..cd63401581 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -50,10 +50,20 @@ #define MAX_ARG 256 /* Max number of arguments allowed */ #define MAX_REG 1024 /* Max number of x(N) registers used */ +/* Scheduler stores data for temporary heaps if + !HEAP_ON_C_STACK. Macros (*TmpHeap*) in global.h selects if we put temporary + heap data on the C stack or if we use the buffers in the scheduler data. */ +#define TMP_HEAP_SIZE 128 /* Number of Eterm in the schedulers + small heap for transient heap data */ +#define CMP_TMP_HEAP_SIZE 2 /* cmp wants its own tmp-heap... */ +#define ERL_ARITH_TMP_HEAP_SIZE 4 /* as does erl_arith... */ +#define BEAM_EMU_TMP_HEAP_SIZE 2 /* and beam_emu... */ + /* * The new arithmetic operations need some extra X registers in the register array. + * so does the gc_bif's (i_gc_bif3 need 3 extra). */ -#define ERTS_X_REGS_ALLOCATED (MAX_REG+2) +#define ERTS_X_REGS_ALLOCATED (MAX_REG+3) #define INPUT_REDUCTIONS (2 * CONTEXT_REDS) @@ -74,6 +84,7 @@ #define ErtsHAllocLockCheck(P) \ ERTS_SMP_LC_ASSERT((ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks((P))) \ + || ((P)->id == ERTS_INVALID_PID) \ || ((P)->scheduler_data \ && (P) == (P)->scheduler_data->match_pseudo_process) \ || erts_is_system_blocked(0)) @@ -130,8 +141,12 @@ #define HeapWordsLeft(p) (HEAP_LIMIT(p) - HEAP_TOP(p)) #if defined(DEBUG) || defined(CHECK_FOR_HOLES) +#if HALFWORD_HEAP +# define ERTS_HOLE_MARKER (0xaf5e78ccU) +#else # define ERTS_HOLE_MARKER (((0xaf5e78ccUL << 24) << 8) | 0xaf5e78ccUL) #endif +#endif /* * Allocate heap memory on the ordinary heap, NEVER in a heap diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h index 4930def4ed..ddc2c1396d 100644 --- a/erts/emulator/beam/error.h +++ b/erts/emulator/beam/error.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -187,10 +187,10 @@ extern Eterm exception_tag[NUMBER_EXC_TAGS]; struct StackTrace { Eterm header; /* bignum header - must be first in struct */ Eterm freason; /* original exception reason is saved in the struct */ - Eterm* pc; - Eterm* current; + BeamInstr* pc; + BeamInstr* current; int depth; /* number of saved pointers in trace[] */ - Eterm *trace[1]; /* varying size - must be last in struct */ + BeamInstr *trace[1]; /* varying size - must be last in struct */ }; #endif /* __ERROR_H__ */ diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 271b40cf0f..5e81a2d624 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -43,11 +43,9 @@ static erts_smp_rwmtx_t export_table_lock; /* Locks the secondary export table. #define export_read_unlock() erts_smp_rwmtx_runlock(&export_table_lock) #define export_write_lock() erts_smp_rwmtx_rwlock(&export_table_lock) #define export_write_unlock() erts_smp_rwmtx_rwunlock(&export_table_lock) -#define export_init_lock() erts_smp_rwmtx_init(&export_table_lock, \ - "export_tab") -extern Eterm* em_call_error_handler; -extern Uint* em_call_traced_function; +extern BeamInstr* em_call_error_handler; +extern BeamInstr* em_call_traced_function; void export_info(int to, void *to_arg) @@ -93,7 +91,7 @@ export_alloc(Export* tmpl) obj->code[2] = tmpl->code[2]; obj->slot.index = -1; obj->address = obj->code+3; - obj->code[3] = (Eterm) em_call_error_handler; + obj->code[3] = (BeamInstr) em_call_error_handler; obj->code[4] = 0; obj->match_prog_set = NULL; return obj; @@ -111,8 +109,12 @@ void init_export_table(void) { HashFunctions f; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + erts_smp_rwmtx_init_opt(&export_table_lock, &rwmtx_opt, "export_tab"); - export_init_lock(); f.hash = (H_FUN) export_hash; f.cmp = (HCMP_FUN) export_cmp; f.alloc = (HALLOC_FUN) export_alloc; @@ -140,7 +142,7 @@ init_export_table(void) Export* erts_find_export_entry(Eterm m, Eterm f, unsigned int a) { - HashValue hval = EXPORT_HASH(m, f, a); + HashValue hval = EXPORT_HASH((BeamInstr) m, (BeamInstr) f, (BeamInstr) a); int ix; HashBucket* b; @@ -185,7 +187,7 @@ erts_find_function(Eterm m, Eterm f, unsigned int a) ep = hash_get(&export_table.htable, (void*) &e); if (ep != NULL && ep->address == ep->code+3 && - ep->code[3] != (Uint) em_call_traced_function) { + ep->code[3] != (BeamInstr) em_call_traced_function) { ep = NULL; } return ep; diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h index cd6af6dd85..c604fdf7c3 100644 --- a/erts/emulator/beam/export.h +++ b/erts/emulator/beam/export.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -37,7 +37,7 @@ typedef struct export void* address; /* Pointer to code for function. */ struct binary* match_prog_set; /* Match program for tracing. */ - Eterm fake_op_func_info_for_hipe[2]; /* MUST be just before code[] */ + BeamInstr fake_op_func_info_for_hipe[2]; /* MUST be just before code[] */ /* * code[0]: Tagged atom for module. * code[1]: Tagged atom for function. @@ -52,7 +52,7 @@ typedef struct export * on_load function that has not been run yet. * Otherwise: 0. */ - Eterm code[5]; + BeamInstr code[5]; } Export; @@ -74,6 +74,6 @@ Export *export_get(Export*); #include "beam_load.h" /* For em_* extern declarations */ #define ExportIsBuiltIn(EntryPtr) \ (((EntryPtr)->address == (EntryPtr)->code + 3) && \ - ((EntryPtr)->code[3] == (Uint) em_apply_bif)) + ((EntryPtr)->code[3] == (BeamInstr) em_apply_bif)) #endif diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 099eddd195..d7c8aa84e9 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -65,11 +65,9 @@ # endif #endif -/* - * For backward compatibility reasons, only encode integers that - * fit in 28 bits (signed) using INTEGER_EXT. +/* Does Sint fit in Sint32? */ -#define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2) +#define IS_SSMALL32(x) (((Uint) (((x) >> (32-1)) + 1)) < 2) /* * Valid creations for nodes are 1, 2, or 3. 0 can also be sent @@ -504,7 +502,7 @@ erts_make_dist_ext_copy(ErtsDistExternal *edep, Uint xsize) ASSERT(edep->ext_endp >= edep->extp); ext_sz = edep->ext_endp - edep->extp; - align_sz = ERTS_WORD_ALIGN_PAD_SZ(dist_ext_sz + ext_sz); + align_sz = ERTS_EXTRA_DATA_ALIGN_SZ(dist_ext_sz + ext_sz); new_edep = erts_alloc(ERTS_ALC_T_EXT_TERM_DATA, dist_ext_sz + ext_sz + align_sz + xsize); @@ -1470,11 +1468,11 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete *hpp += EXTERNAL_THING_HEAD_SIZE + 1; etp->header = make_external_pid_header(1); - etp->next = off_heap->externals; + etp->next = off_heap->first; etp->node = node; etp->data.ui[0] = data; - off_heap->externals = etp; + off_heap->first = (struct erl_off_heap_header*) etp; *objp = make_external_pid(etp); } return ep; @@ -1489,20 +1487,28 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete static byte* enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) { - DECLARE_ESTACK(s); + DECLARE_WSTACK(s); Uint n; Uint i; Uint j; Uint* ptr; Eterm val; FloatDef f; +#if HALFWORD_HEAP + UWord wobj; +#endif + goto L_jump_start; outer_loop: - while (!ESTACK_ISEMPTY(s)) { - obj = ESTACK_POP(s); - switch (val = ESTACK_POP(s)) { + while (!WSTACK_ISEMPTY(s)) { +#if HALFWORD_HEAP + obj = (Eterm) (wobj = WSTACK_POP(s)); +#else + obj = WSTACK_POP(s); +#endif + switch (val = WSTACK_POP(s)) { case ENC_TERM: break; case ENC_ONE_CONS: @@ -1513,29 +1519,40 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) obj = CAR(cons); tl = CDR(cons); - ESTACK_PUSH(s, is_list(tl) ? ENC_ONE_CONS : ENC_TERM); - ESTACK_PUSH(s, tl); + WSTACK_PUSH(s, is_list(tl) ? ENC_ONE_CONS : ENC_TERM); + WSTACK_PUSH(s, tl); } break; case ENC_PATCH_FUN_SIZE: { +#if HALFWORD_HEAP + byte* size_p = (byte *) wobj; +#else byte* size_p = (byte *) obj; - +#endif put_int32(ep - size_p, size_p); } goto outer_loop; case ENC_LAST_ARRAY_ELEMENT: { +#if HALFWORD_HEAP + Eterm* ptr = (Eterm *) wobj; +#else Eterm* ptr = (Eterm *) obj; +#endif obj = *ptr; } break; default: /* ENC_LAST_ARRAY_ELEMENT+1 and upwards */ { +#if HALFWORD_HEAP + Eterm* ptr = (Eterm *) wobj; +#else Eterm* ptr = (Eterm *) obj; +#endif obj = *ptr++; - ESTACK_PUSH(s, val-1); - ESTACK_PUSH(s, (Eterm) ptr); + WSTACK_PUSH(s, val-1); + WSTACK_PUSH(s, (UWord) ptr); } break; } @@ -1552,19 +1569,23 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) case SMALL_DEF: { + /* From R14B we no longer restrict INTEGER_EXT to 28 bits, + * as done earlier for backward compatibility reasons. */ Sint val = signed_val(obj); if ((Uint)val < 256) { *ep++ = SMALL_INTEGER_EXT; put_int8(val, ep); ep++; - } else if (sizeof(Sint) == 4 || IS_SSMALL28(val)) { + } else if (sizeof(Sint) == 4 || IS_SSMALL32(val)) { *ep++ = INTEGER_EXT; put_int32(val, ep); ep += 4; } else { - Eterm tmp_big[2]; - Eterm big = small_to_big(val, tmp_big); + DeclareTmpHeapNoproc(tmp_big,2); + Eterm big; + UseTmpHeapNoproc(2); + big = small_to_big(val, tmp_big); *ep++ = SMALL_BIG_EXT; n = big_bytes(big); ASSERT(n < 256); @@ -1572,23 +1593,38 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) ep += 1; *ep++ = big_sign(big); ep = big_to_bytes(big, ep); + UnUseTmpHeapNoproc(2); } } break; case BIG_DEF: - if ((n = big_bytes(obj)) < 256) { - *ep++ = SMALL_BIG_EXT; - put_int8(n, ep); - ep += 1; - } - else { - *ep++ = LARGE_BIG_EXT; - put_int32(n, ep); - ep += 4; + { + int sign = big_sign(obj); + n = big_bytes(obj); + if (sizeof(Sint)==4 && n<=4) { + Uint dig = big_digit(obj,0); + Sint val = sign ? -dig : dig; + if ((val<0) == sign) { + *ep++ = INTEGER_EXT; + put_int32(val, ep); + ep += 4; + break; + } + } + if (n < 256) { + *ep++ = SMALL_BIG_EXT; + put_int8(n, ep); + ep += 1; + } + else { + *ep++ = LARGE_BIG_EXT; + put_int32(n, ep); + ep += 4; + } + *ep++ = sign; + ep = big_to_bytes(obj, ep); } - *ep++ = big_sign(obj); - ep = big_to_bytes(obj, ep); break; case PID_DEF: @@ -1662,8 +1698,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) ep += 4; } if (i > 0) { - ESTACK_PUSH(s, ENC_LAST_ARRAY_ELEMENT+i-1); - ESTACK_PUSH(s, (Eterm) ptr); + WSTACK_PUSH(s, ENC_LAST_ARRAY_ELEMENT+i-1); + WSTACK_PUSH(s, (UWord) ptr); } break; @@ -1741,7 +1777,7 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) } case EXPORT_DEF: { - Export* exp = (Export *) (export_val(obj))[1]; + Export* exp = *((Export **) (export_val(obj) + 1)); if ((dflags & DFLAG_EXPORT_PTR_TAG) != 0) { *ep++ = EXPORT_EXT; ep = enc_atom(acmp, exp->code[0], ep, dflags); @@ -1770,8 +1806,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) int ei; *ep++ = NEW_FUN_EXT; - ESTACK_PUSH(s, ENC_PATCH_FUN_SIZE); - ESTACK_PUSH(s, (Eterm) ep); /* Position for patching in size */ + WSTACK_PUSH(s, ENC_PATCH_FUN_SIZE); + WSTACK_PUSH(s, (UWord) ep); /* Position for patching in size */ ep += 4; *ep = funp->arity; ep += 1; @@ -1788,8 +1824,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) fun_env: for (ei = funp->num_free-1; ei > 0; ei--) { - ESTACK_PUSH(s, ENC_TERM); - ESTACK_PUSH(s, funp->env[ei]); + WSTACK_PUSH(s, ENC_TERM); + WSTACK_PUSH(s, (UWord) funp->env[ei]); } if (funp->num_free != 0) { obj = funp->env[0]; @@ -1832,7 +1868,7 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) break; } } - DESTROY_ESTACK(s); + DESTROY_WSTACK(s); return ep; } @@ -1874,69 +1910,30 @@ is_external_string(Eterm list, int* p_is_string) return len; } -/* Assumes that the ones to undo are preluding the lists. */ +/* Assumes that the ones to undo are preluding the list. */ static void undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end) { const Uint area_sz = (end - start) * sizeof(Eterm); - struct proc_bin* mso; - struct proc_bin** mso_nextp = NULL; -#ifndef HYBRID /* FIND ME! */ - struct erl_fun_thing* funs; - struct erl_fun_thing** funs_nextp = NULL; -#endif - struct external_thing_* ext; - struct external_thing_** ext_nextp = NULL; - - for (mso = off_heap->mso; ; mso=mso->next) { - if (!in_area(mso, start, area_sz)) { - if (mso_nextp != NULL) { - *mso_nextp = NULL; - erts_cleanup_mso(off_heap->mso); - off_heap->mso = mso; + struct erl_off_heap_header* hdr; + struct erl_off_heap_header** hdr_nextp = NULL; + + for (hdr = off_heap->first; ; hdr=hdr->next) { + if (!in_area(hdr, start, area_sz)) { + if (hdr_nextp != NULL) { + *hdr_nextp = NULL; + erts_cleanup_offheap(off_heap); + off_heap->first = hdr; } break; } - mso_nextp = &mso->next; + hdr_nextp = &hdr->next; } -#ifndef HYBRID /* FIND ME! */ - for (funs = off_heap->funs; ; funs=funs->next) { - if (!in_area(funs, start, area_sz)) { - if (funs_nextp != NULL) { - *funs_nextp = NULL; - erts_cleanup_funs(off_heap->funs); - off_heap->funs = funs; - } - break; - } - funs_nextp = &funs->next; - } -#endif - for (ext = off_heap->externals; ; ext=ext->next) { - if (!in_area(ext, start, area_sz)) { - if (ext_nextp != NULL) { - *ext_nextp = NULL; - erts_cleanup_externals(off_heap->externals); - off_heap->externals = ext; - } - break; - } - ext_nextp = &ext->next; - } - - /* Assert that the ones to undo were indeed preluding the lists. */ + /* Assert that the ones to undo were indeed preluding the list. */ #ifdef DEBUG - for (mso = off_heap->mso; mso != NULL; mso=mso->next) { - ASSERT(!in_area(mso, start, area_sz)); - } -# ifndef HYBRID /* FIND ME! */ - for (funs = off_heap->funs; funs != NULL; funs=funs->next) { - ASSERT(!in_area(funs, start, area_sz)); - } -# endif - for (ext = off_heap->externals; ext != NULL; ext=ext->next) { - ASSERT(!in_area(ext, start, area_sz)); + for (hdr = off_heap->first; hdr != NULL; hdr = hdr->next) { + ASSERT(!in_area(hdr, start, area_sz)); } #endif /* DEBUG */ } @@ -1952,11 +1949,11 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et register Eterm* hp = *hpp; /* Please don't take the address of hp */ Eterm* next = objp; - *next = (Eterm) NULL; + *next = (Eterm) (UWord) NULL; while (next != NULL) { objp = next; - next = (Eterm *) (*objp); + next = (Eterm *) EXPAND_POINTER(*objp); switch (*ep++) { case INTEGER_EXT: @@ -1964,7 +1961,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et Sint sn = get_int32(ep); ep += 4; -#if defined(ARCH_64) +#if defined(ARCH_64) && !HALFWORD_HEAP *objp = make_small(sn); #else if (MY_IS_SSMALL(sn)) { @@ -2061,7 +2058,7 @@ dec_term_atom_common: hp += n; objp = hp - 1; while (n-- > 0) { - objp[0] = (Eterm) next; + objp[0] = (Eterm) COMPRESS_POINTER(next); next = objp; objp--; } @@ -2079,12 +2076,12 @@ dec_term_atom_common: *objp = make_list(hp); hp += 2*n; objp = hp - 2; - objp[0] = (Eterm) (objp+1); - objp[1] = (Eterm) next; + objp[0] = (Eterm) COMPRESS_POINTER((objp+1)); + objp[1] = (Eterm) COMPRESS_POINTER(next); next = objp; objp -= 2; while (--n > 0) { - objp[0] = (Eterm) next; + objp[0] = (Eterm) COMPRESS_POINTER(next); objp[1] = make_list(objp + 2); next = objp; objp -= 2; @@ -2180,11 +2177,11 @@ dec_term_atom_common: hp += EXTERNAL_THING_HEAD_SIZE + 1; etp->header = make_external_port_header(1); - etp->next = off_heap->externals; + etp->next = off_heap->first; etp->node = node; etp->data.ui[0] = num; - off_heap->externals = etp; + off_heap->first = (struct erl_off_heap_header*)etp; *objp = make_external_port(etp); } @@ -2238,32 +2235,39 @@ dec_term_atom_common: node = erts_find_or_insert_node(sysname, cre); if(node == erts_this_node) { RefThing *rtp = (RefThing *) hp; - hp += REF_THING_HEAD_SIZE; -#ifdef ARCH_64 + ref_num = (Uint32 *) (hp + REF_THING_HEAD_SIZE); + +#if defined(ARCH_64) && !HALFWORD_HEAP + hp += REF_THING_HEAD_SIZE + ref_words/2 + 1; rtp->header = make_ref_thing_header(ref_words/2 + 1); #else + hp += REF_THING_HEAD_SIZE + ref_words; rtp->header = make_ref_thing_header(ref_words); #endif *objp = make_internal_ref(rtp); } else { ExternalThing *etp = (ExternalThing *) hp; - hp += EXTERNAL_THING_HEAD_SIZE; - -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP + hp += EXTERNAL_THING_HEAD_SIZE + ref_words/2 + 1; +#else + hp += EXTERNAL_THING_HEAD_SIZE + ref_words; +#endif + +#if defined(ARCH_64) && !HALFWORD_HEAP etp->header = make_external_ref_header(ref_words/2 + 1); #else etp->header = make_external_ref_header(ref_words); #endif - etp->next = off_heap->externals; + etp->next = off_heap->first; etp->node = node; - off_heap->externals = etp; + off_heap->first = (struct erl_off_heap_header*)etp; *objp = make_external_ref(etp); + ref_num = &(etp->data.ui32[0]); } - ref_num = (Uint32 *) hp; -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP *(ref_num++) = ref_words /* 32-bit arity */; #endif ref_num[0] = r0; @@ -2271,12 +2275,9 @@ dec_term_atom_common: ref_num[i] = get_int32(ep); ep += 4; } -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP if ((1 + ref_words) % 2) ref_num[ref_words] = 0; - hp += ref_words/2 + 1; -#else - hp += ref_words; #endif break; } @@ -2304,8 +2305,8 @@ dec_term_atom_common: hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = n; - pb->next = off_heap->mso; - off_heap->mso = pb; + pb->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)pb; pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -2341,8 +2342,8 @@ dec_term_atom_common: pb = (ProcBin *) hp; pb->thing_word = HEADER_PROC_BIN; pb->size = n; - pb->next = off_heap->mso; - off_heap->mso = pb; + pb->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)pb; pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -2398,7 +2399,12 @@ dec_term_atom_common: } *objp = make_export(hp); *hp++ = HEADER_EXPORT; +#if HALFWORD_HEAP + *((UWord *) (UWord) hp) = (UWord) erts_export_get_or_make_stub(mod, name, arity); + hp += 2; +#else *hp++ = (Eterm) erts_export_get_or_make_stub(mod, name, arity); +#endif break; } break; @@ -2457,8 +2463,8 @@ dec_term_atom_common: * It is safe to link the fun into the fun list only when * no more validity tests can fail. */ - funp->next = off_heap->funs; - off_heap->funs = funp; + funp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)funp; #endif funp->fe = erts_put_fun_entry2(module, old_uniq, old_index, @@ -2474,11 +2480,11 @@ dec_term_atom_common: /* Environment */ for (i = num_free-1; i >= 0; i--) { - funp->env[i] = (Eterm) next; + funp->env[i] = (Eterm) COMPRESS_POINTER(next); next = funp->env + i; } /* Creator */ - funp->creator = (Eterm) next; + funp->creator = (Eterm) COMPRESS_POINTER(next); next = &(funp->creator); break; } @@ -2535,8 +2541,8 @@ dec_term_atom_common: * It is safe to link the fun into the fun list only when * no more validity tests can fail. */ - funp->next = off_heap->funs; - off_heap->funs = funp; + funp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)funp; #endif old_uniq = unsigned_val(temp); @@ -2549,7 +2555,7 @@ dec_term_atom_common: /* Environment */ for (i = num_free-1; i >= 0; i--) { - funp->env[i] = (Eterm) next; + funp->env[i] = (Eterm) COMPRESS_POINTER(next); next = funp->env + i; } break; @@ -2580,26 +2586,35 @@ dec_term_atom_common: static Uint encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) { - DECLARE_ESTACK(s); + DECLARE_WSTACK(s); Uint m, i, arity; Uint result = 0; +#if HALFWORD_HEAP + UWord wobj = 0; +#endif goto L_jump_start; outer_loop: - while (!ESTACK_ISEMPTY(s)) { - obj = ESTACK_POP(s); - + while (!WSTACK_ISEMPTY(s)) { +#if HALFWORD_HEAP + obj = (Eterm) (wobj = WSTACK_POP(s)); +#else + obj = WSTACK_POP(s); +#endif handle_popped_obj: - if (is_CP(obj)) { + if (is_CP(obj)) { /* Does not look for CP, looks for "no tag" */ +#if HALFWORD_HEAP + Eterm* ptr = (Eterm *) wobj; +#else Eterm* ptr = (Eterm *) obj; - +#endif /* * Pointer into a tuple. */ obj = *ptr--; if (!is_header(obj)) { - ESTACK_PUSH(s, (Eterm)ptr); + WSTACK_PUSH(s, (UWord)ptr); } else { /* Reached tuple header */ ASSERT(header_is_arityval(obj)); @@ -2611,7 +2626,7 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) tl = CDR(cons); obj = CAR(cons); - ESTACK_PUSH(s, tl); + WSTACK_PUSH(s, tl); } else if (is_nil(obj)) { result++; goto outer_loop; @@ -2647,17 +2662,22 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) if ((Uint)val < 256) result += 1 + 1; /* SMALL_INTEGER_EXT */ - else if (sizeof(Sint) == 4 || IS_SSMALL28(val)) + else if (sizeof(Sint) == 4 || IS_SSMALL32(val)) result += 1 + 4; /* INTEGER_EXT */ else { - Eterm tmp_big[2]; + DeclareTmpHeapNoproc(tmp_big,2); + UseTmpHeapNoproc(2); i = big_bytes(small_to_big(val, tmp_big)); result += 1 + 1 + 1 + i; /* SMALL_BIG_EXT */ + UnUseTmpHeapNoproc(2); } } break; case BIG_DEF: - if ((i = big_bytes(obj)) < 256) + i = big_bytes(obj); + if (sizeof(Sint)==4 && i <= 4 && (big_digit(obj,0)-big_sign(obj)) < (1<<31)) + result += 1 + 4; /* INTEGER_EXT */ + else if (i < 256) result += 1 + 1 + 1 + i; /* tag,size,sign,digits */ else result += 1 + 4 + 1 + i; /* tag,size,sign,digits */ @@ -2698,7 +2718,11 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) result += 1 + 4; } ptr += arity; +#if HALFWORD_HEAP + obj = (Eterm) (wobj = (UWord) ptr); +#else obj = (Eterm) ptr; +#endif goto handle_popped_obj; } break; @@ -2740,14 +2764,14 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) if (is_not_list(obj)) { /* Push any non-list terms on the stack */ - ESTACK_PUSH(s, obj); + WSTACK_PUSH(s, obj); } else { /* Lists must be handled specially. */ if ((m = is_string(obj)) && (m < MAX_STRING_LEN)) { result += m + 2 + 1; } else { result += 5; - ESTACK_PUSH(s, obj); + WSTACK_PUSH(s, obj); } } } @@ -2760,8 +2784,12 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) case EXPORT_DEF: { - Export* ep = (Export *) (export_val(obj))[1]; + Export* ep = *((Export **) (export_val(obj) + 1)); +#if HALFWORD_HEAP + result += 2; +#else result += 1; +#endif result += encode_size_struct2(acmp, ep->code[0], dflags); result += encode_size_struct2(acmp, ep->code[1], dflags); result += encode_size_struct2(acmp, make_small(ep->code[2]), dflags); @@ -2774,7 +2802,7 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) } } - DESTROY_ESTACK(s); + DESTROY_WSTACK(s); return result; } @@ -2886,7 +2914,7 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins) ep += 2; atom_extra_skip = 1 + 4*id_words; /* In case it is an external ref */ -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP heap_size += EXTERNAL_THING_HEAD_SIZE + id_words/2 + 1; #else heap_size += EXTERNAL_THING_HEAD_SIZE + id_words; @@ -2961,7 +2989,11 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins) break; case EXPORT_EXT: terms += 3; +#if HALFWORD_HEAP + heap_size += 3; +#else heap_size += 2; +#endif break; case NEW_FUN_EXT: { diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index eada6d4f27..cee48bbeb0 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -211,8 +211,8 @@ ERTS_GLB_INLINE void * erts_dist_ext_trailer(ErtsDistExternal *edep) { void *res = (void *) (edep->ext_endp - + ERTS_WORD_ALIGN_PAD_SZ(edep->ext_endp)); - ASSERT((((Uint) res) % sizeof(Uint)) == 0); + + ERTS_EXTRA_DATA_ALIGN_SZ(edep->ext_endp)); + ASSERT((((UWord) res) % sizeof(Uint)) == 0); return res; } diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index cab249a53f..280421952e 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -76,12 +76,6 @@ typedef struct line_buf { /* Buffer used in line oriented I/O */ The rest is the overflow buffer. */ } LineBuf; -/* Temporary object header, auto-deallocated when NIF returns. */ -struct enif_tmp_obj_t { - struct enif_tmp_obj_t* next; - void (*dtor)(struct enif_tmp_obj_t*); - /*char data[];*/ -}; struct enif_environment_t /* ErlNifEnv */ { struct erl_module_nif* mod_nif; @@ -403,7 +397,7 @@ extern Eterm erts_ddll_monitor_driver(Process *p, /* Add fields in ERTS_INTERNAL_BINARY_FIELDS, otherwise the drivers crash */ #define ERTS_INTERNAL_BINARY_FIELDS \ - Uint flags; \ + UWord flags; \ erts_refc_t refc; \ ERTS_BINARY_STRUCT_ALIGNMENT @@ -470,7 +464,10 @@ typedef union { typedef struct proc_bin { Eterm thing_word; /* Subtag REFC_BINARY_SUBTAG. */ Uint size; /* Binary size in bytes. */ - struct proc_bin *next; /* Pointer to next ProcBin. */ +#if HALFWORD_HEAP + void* dummy_ptr_padding__; +#endif + struct erl_off_heap_header *next; Binary *val; /* Pointer to Binary structure. */ byte *bytes; /* Pointer to the actual data bytes. */ Uint flags; /* Flag word. */ @@ -500,8 +497,8 @@ erts_mk_magic_binary_term(Eterm **hpp, ErlOffHeap *ohp, Binary *mbp) pb->thing_word = HEADER_PROC_BIN; pb->size = 0; - pb->next = ohp->mso; - ohp->mso = pb; + pb->next = ohp->first; + ohp->first = (struct erl_off_heap_header*) pb; pb->val = mbp; pb->bytes = (byte *) mbp->orig_bytes; pb->flags = 0; @@ -518,6 +515,15 @@ erts_mk_magic_binary_term(Eterm **hpp, ErlOffHeap *ohp, Binary *mbp) && (thing_subtag(*binary_val((T))) == REFC_BINARY_SUBTAG) \ && (((ProcBin *) binary_val((T)))->val->flags & BIN_FLAG_MAGIC)) + +union erl_off_heap_ptr { + struct erl_off_heap_header* hdr; + ProcBin *pb; + struct erl_fun_thing* fun; + struct external_thing_* ext; + Eterm* ep; +}; + /* arrays that get malloced at startup */ extern Port* erts_port; extern erts_smp_atomic_t erts_ports_alive; @@ -723,6 +729,60 @@ do { \ #define ESTACK_POP(s) (*(--ESTK_CONCAT(s,_sp))) +void erl_grow_wstack(UWord** start, UWord** sp, UWord** end); +#define WSTK_CONCAT(a,b) a##b +#define WSTK_SUBSCRIPT(s,i) *((UWord *)((byte *)WSTK_CONCAT(s,_start) + (i))) +#define DEF_WSTACK_SIZE (16) + +#define DECLARE_WSTACK(s) \ + UWord WSTK_CONCAT(s,_default_stack)[DEF_WSTACK_SIZE]; \ + UWord* WSTK_CONCAT(s,_start) = WSTK_CONCAT(s,_default_stack); \ + UWord* WSTK_CONCAT(s,_sp) = WSTK_CONCAT(s,_start); \ + UWord* WSTK_CONCAT(s,_end) = WSTK_CONCAT(s,_start) + DEF_WSTACK_SIZE + +#define DESTROY_WSTACK(s) \ +do { \ + if (WSTK_CONCAT(s,_start) != WSTK_CONCAT(s,_default_stack)) { \ + erts_free(ERTS_ALC_T_ESTACK, WSTK_CONCAT(s,_start)); \ + } \ +} while(0) + +#define WSTACK_PUSH(s, x) \ +do { \ + if (WSTK_CONCAT(s,_sp) == WSTK_CONCAT(s,_end)) { \ + erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp), \ + &WSTK_CONCAT(s,_end)); \ + } \ + *WSTK_CONCAT(s,_sp)++ = (x); \ +} while(0) + +#define WSTACK_PUSH2(s, x, y) \ +do { \ + if (WSTK_CONCAT(s,_sp) > WSTK_CONCAT(s,_end) - 2) { \ + erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp), \ + &WSTK_CONCAT(s,_end)); \ + } \ + *WSTK_CONCAT(s,_sp)++ = (x); \ + *WSTK_CONCAT(s,_sp)++ = (y); \ +} while(0) + +#define WSTACK_PUSH3(s, x, y, z) \ +do { \ + if (WSTK_CONCAT(s,_sp) > WSTK_CONCAT(s,_end) - 3) { \ + erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp), \ + &WSTK_CONCAT(s,_end)); \ + } \ + *WSTK_CONCAT(s,_sp)++ = (x); \ + *WSTK_CONCAT(s,_sp)++ = (y); \ + *WSTK_CONCAT(s,_sp)++ = (z); \ +} while(0) + +#define WSTACK_COUNT(s) (WSTK_CONCAT(s,_sp) - WSTK_CONCAT(s,_start)) + +#define WSTACK_ISEMPTY(s) (WSTK_CONCAT(s,_sp) == WSTK_CONCAT(s,_start)) +#define WSTACK_POP(s) (*(--WSTK_CONCAT(s,_sp))) + + /* port status flags */ #define ERTS_PORT_SFLG_CONNECTED ((Uint32) (1 << 0)) @@ -775,7 +835,6 @@ Eterm erts_new_heap_binary(Process *p, byte *buf, int len, byte** datap); Eterm erts_new_mso_binary(Process*, byte*, int); Eterm new_binary(Process*, byte*, int); Eterm erts_realloc_binary(Eterm bin, size_t size); -void erts_cleanup_mso(ProcBin* pb); /* erl_bif_info.c */ @@ -802,7 +861,7 @@ void erts_system_profile_clear(Process *c_p); int erts_load_module(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm* mod, byte* code, int size); void init_load(void); -Eterm* find_function_from_pc(Eterm* pc); +BeamInstr* find_function_from_pc(BeamInstr* pc); Eterm erts_module_info_0(Process* p, Eterm module); Eterm erts_module_info_1(Process* p, Eterm module, Eterm what); Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info); @@ -833,6 +892,8 @@ Eterm copy_object(Eterm, Process*); Uint size_object(Eterm); Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*); Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*); +void move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, + Eterm* refs, unsigned nrefs); #ifdef HYBRID #define RRMA_DEFAULT_SIZE 256 @@ -965,7 +1026,7 @@ void print_pass_through(int, byte*, int); /* beam_emu.c */ int catchlevel(Process*); -void init_emulator(_VOID_); +void init_emulator(void); void process_main(void); Eterm build_stacktrace(Process* c_p, Eterm exc); Eterm expand_error_value(Process* c_p, Uint freason, Eterm Value); @@ -1030,6 +1091,7 @@ Eterm erts_heap_sizes(Process* p); void erts_offset_off_heap(ErlOffHeap *, Sint, Eterm*, Eterm*); void erts_offset_heap_ptr(Eterm*, Uint, Sint, Eterm*, Eterm*); void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*); +void erts_free_heap_frags(Process* p); #ifdef HYBRID int erts_global_garbage_collect(Process*, int, Eterm*, int); @@ -1144,12 +1206,11 @@ erts_smp_port_unlock(Port *prt) { #ifdef ERTS_SMP long refc; + erts_smp_mtx_unlock(prt->lock); refc = erts_smp_atomic_dectest(&prt->refc); ASSERT(refc >= 0); if (refc == 0) erts_port_cleanup(prt); - else - erts_smp_mtx_unlock(prt->lock); #endif } @@ -1171,7 +1232,7 @@ erts_smp_port_unlock(Port *prt) ERTS_INVALID_PORT_OPT((PP), (ID), ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP) #define ERTS_PORT_SCHED_ID(P, ID) \ - ((Uint) erts_prtsd_set((P), ERTS_PSD_SCHED_ID, (void *) (ID))) + ((Uint) (UWord) erts_prtsd_set((P), ERTS_PSD_SCHED_ID, (void *) (UWord) (ID))) #ifdef ERTS_SMP Port *erts_de2port(DistEntry *, Process *, ErtsProcLocks); @@ -1449,7 +1510,6 @@ void p_slpq(_VOID_); void erts_silence_warn_unused_result(long unused); void erts_cleanup_offheap(ErlOffHeap *offheap); -void erts_cleanup_externals(ExternalThing *); Uint erts_fit_in_bits(Uint); int list_length(Eterm); @@ -1463,6 +1523,7 @@ Uint32 make_hash(Eterm); Eterm erts_bld_atom(Uint **hpp, Uint *szp, char *str); Eterm erts_bld_uint(Uint **hpp, Uint *szp, Uint ui); +Eterm erts_bld_uword(Uint **hpp, Uint *szp, UWord uw); Eterm erts_bld_uint64(Uint **hpp, Uint *szp, Uint64 ui64); Eterm erts_bld_sint64(Uint **hpp, Uint *szp, Sint64 si64); Eterm erts_bld_cons(Uint **hpp, Uint *szp, Eterm car, Eterm cdr); @@ -1481,7 +1542,7 @@ erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length, Eterm atoms[], Uint uints1[], Uint uints2[]); Eterm store_external_or_ref_in_proc_(Process *, Eterm); -Eterm store_external_or_ref_(Uint **, ExternalThing **, Eterm); +Eterm store_external_or_ref_(Uint **, ErlOffHeap*, Eterm); #define NC_HEAP_SIZE(NC) \ (ASSERT_EXPR(is_node_container((NC))), \ @@ -1515,7 +1576,9 @@ Sint cmp(Eterm, Eterm); #define CMP_EQ(a,b) ((a) == (b) || cmp_eq((a),(b))) #define CMP_NE(a,b) ((a) != (b) && cmp_ne((a),(b))) +/* duplicates from big.h */ int term_to_Uint(Eterm term, Uint *up); +int term_to_UWord(Eterm, UWord*); #ifdef HAVE_ERTS_NOW_CPU extern int erts_cpu_timestamp; @@ -1525,6 +1588,10 @@ void erts_init_bif_chksum(void); /* erl_bif_re.c */ void erts_init_bif_re(void); Sint erts_re_set_loop_limit(Sint limit); +/* erl_bif_binary.c */ +void erts_init_bif_binary(void); +Sint erts_binary_set_loop_limit(Sint limit); + /* erl_unicode.c */ void erts_init_unicode(void); Sint erts_unicode_set_loop_limit(Sint limit); @@ -1554,12 +1621,12 @@ void erts_queue_error_logger_message(Eterm, Eterm, ErlHeapFragment *); void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *); void trace_send(Process*, Eterm, Eterm); void trace_receive(Process*, Eterm); -Uint32 erts_call_trace(Process *p, Eterm mfa[], Binary *match_spec, Eterm* args, +Uint32 erts_call_trace(Process *p, BeamInstr mfa[], Binary *match_spec, Eterm* args, int local, Eterm *tracer_pid); -void erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid); -void erts_trace_exception(Process* p, Eterm mfa[], Eterm class, Eterm value, +void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid); +void erts_trace_exception(Process* p, BeamInstr mfa[], Eterm class, Eterm value, Eterm *tracer); -void erts_trace_return_to(Process *p, Uint *pc); +void erts_trace_return_to(Process *p, BeamInstr *pc); void trace_sched(Process*, Eterm); void trace_proc(Process*, Process*, Eterm, Eterm); void trace_proc_spawn(Process*, Eterm pid, Eterm mod, Eterm func, Eterm args); @@ -1589,7 +1656,7 @@ Uint erts_trace_flag2bit(Eterm flag); int erts_trace_flags(Eterm List, Uint *pMask, Eterm *pTracer, int *pCpuTimestamp); Eterm erts_bif_trace(int bif_index, Process* p, - Eterm arg1, Eterm arg2, Eterm arg3, Uint *I); + Eterm arg1, Eterm arg2, Eterm arg3, BeamInstr *I); #ifdef ERTS_SMP void erts_send_pending_trace_msgs(ErtsSchedulerData *esdp); @@ -1606,7 +1673,7 @@ void bin_write(int, void*, byte*, int); int intlist_to_buf(Eterm, char*, int); /* most callers pass plain char*'s */ struct Sint_buf { -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP char s[22]; #else char s[12]; @@ -1619,7 +1686,7 @@ int io_list_to_buf(Eterm, char*, int); int io_list_to_buf2(Eterm, char*, int); int io_list_len(Eterm); int is_string(Eterm); -void erl_at_exit(FUNCTION(void,(*),(void*)), void*); +void erl_at_exit(void (*) (void*), void*); Eterm collect_memory(Process *); void dump_memory_to_fd(int); int dump_memory_data(const char *); @@ -1654,6 +1721,8 @@ Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live); Eterm erts_gc_float_1(Process* p, Eterm* reg, Uint live); Eterm erts_gc_round_1(Process* p, Eterm* reg, Uint live); Eterm erts_gc_trunc_1(Process* p, Eterm* reg, Uint live); +Eterm erts_gc_binary_part_3(Process* p, Eterm* reg, Uint live); +Eterm erts_gc_binary_part_2(Process* p, Eterm* reg, Uint live); Uint erts_current_reductions(Process* current, Process *p); @@ -1663,7 +1732,6 @@ int erts_print_system_version(int to, void *arg, Process *c_p); * Interface to erl_init */ void erl_init(void); -void erts_first_process(Eterm modname, void* code, unsigned size, int argc, char** argv); #define seq_trace_output(token, msg, type, receiver, process) \ seq_trace_output_generic((token), (msg), (type), (receiver), (process), NIL) @@ -1683,8 +1751,10 @@ struct trace_pattern_flags { unsigned int local : 1; /* Local call trace breakpoint */ unsigned int meta : 1; /* Metadata trace breakpoint */ unsigned int call_count : 1; /* Fast call count breakpoint */ + unsigned int call_time : 1; /* Fast call time breakpoint */ }; extern const struct trace_pattern_flags erts_trace_pattern_flags_off; +extern int erts_call_time_breakpoint_tracing; int erts_set_trace_pattern(Eterm* mfa, int specified, Binary* match_prog_set, Binary *meta_match_prog_set, int on, struct trace_pattern_flags, @@ -1728,18 +1798,20 @@ extern void erts_match_prog_foreach_offheap(Binary *b, void (*)(ErlOffHeap *, void *), void *); -#define MATCH_SET_RETURN_TRACE 0x1 /* return trace requested */ -#define MATCH_SET_RETURN_TO_TRACE 0x2 /* Misleading name, it is not actually - set by the match program, but by the - breakpoint functions */ -#define MATCH_SET_EXCEPTION_TRACE 0x4 /* exception trace requested */ +#define MATCH_SET_RETURN_TRACE (0x1) /* return trace requested */ +#define MATCH_SET_RETURN_TO_TRACE (0x2) /* Misleading name, it is not actually + set by the match program, but by the + breakpoint functions */ +#define MATCH_SET_EXCEPTION_TRACE (0x4) /* exception trace requested */ #define MATCH_SET_RX_TRACE (MATCH_SET_RETURN_TRACE|MATCH_SET_EXCEPTION_TRACE) /* * Flag values when tracing bif + * Future note: flag field is 8 bits */ -#define BIF_TRACE_AS_LOCAL 0x1 -#define BIF_TRACE_AS_GLOBAL 0x2 -#define BIF_TRACE_AS_META 0x4 +#define BIF_TRACE_AS_LOCAL (0x1) +#define BIF_TRACE_AS_GLOBAL (0x2) +#define BIF_TRACE_AS_META (0x4) +#define BIF_TRACE_AS_CALL_TIME (0x8) extern erts_driver_t vanilla_driver; extern erts_driver_t spawn_driver; @@ -1839,4 +1911,61 @@ erts_alloc_message_heap(Uint size, #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ -#endif +#if !HEAP_ON_C_STACK +# if defined(DEBUG) +# define DeclareTmpHeap(VariableName,Size,Process) \ + Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,Process) +# define DeclareTmpHeapNoproc(VariableName,Size) \ + Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,NULL) +# define UseTmpHeap(Size,Proc) \ + do { \ + erts_debug_use_tmp_heap((Size),(Proc)); \ + } while (0) +# define UnUseTmpHeap(Size,Proc) \ + do { \ + erts_debug_unuse_tmp_heap((Size),(Proc)); \ + } while (0) +# define UseTmpHeapNoproc(Size) \ + do { \ + erts_debug_use_tmp_heap(Size,NULL); \ + } while (0) +# define UnUseTmpHeapNoproc(Size) \ + do { \ + erts_debug_unuse_tmp_heap(Size,NULL); \ + } while (0) +# else +# define DeclareTmpHeap(VariableName,Size,Process) \ + Eterm *VariableName = (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used) +# define DeclareTmpHeapNoproc(VariableName,Size) \ + Eterm *VariableName = (erts_get_scheduler_data()->tmp_heap)+(erts_get_scheduler_data()->num_tmp_heap_used) +# define UseTmpHeap(Size,Proc) \ + do { \ + ERTS_PROC_GET_SCHDATA(Proc)->num_tmp_heap_used += (Size); \ + } while (0) +# define UnUseTmpHeap(Size,Proc) \ + do { \ + ERTS_PROC_GET_SCHDATA(Proc)->num_tmp_heap_used -= (Size); \ + } while (0) +# define UseTmpHeapNoproc(Size) \ + do { \ + erts_get_scheduler_data()->num_tmp_heap_used += (Size); \ + } while (0) +# define UnUseTmpHeapNoproc(Size) \ + do { \ + erts_get_scheduler_data()->num_tmp_heap_used -= (Size); \ + } while (0) + + +# endif + +#else +# define DeclareTmpHeap(VariableName,Size,Process) \ + Eterm VariableName[Size] +# define DeclareTmpHeapNoproc(VariableName,Size) \ + Eterm VariableName[Size] +# define UseTmpHeap(Size,Proc) /* Nothing */ +# define UnUseTmpHeap(Size,Proc) /* Nothing */ +# define UseTmpHeapNoproc(Size) /* Nothing */ +# define UnUseTmpHeapNoproc(Size) /* Nothing */ +#endif /* HEAP_ON_C_STACK */ +#endif /* !__GLOBAL_H__ */ diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 3309b77086..79022d5dd7 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -72,6 +72,15 @@ erts_driver_t fd_driver; static int init_driver(erts_driver_t *, ErlDrvEntry *, DE_Handle *); static void terminate_port(Port *p); static void pdl_init(void); +#ifdef ERTS_SMP +static void driver_monitor_lock_pdl(Port *p); +static void driver_monitor_unlock_pdl(Port *p); +#define DRV_MONITOR_LOCK_PDL(Port) driver_monitor_lock_pdl(Port) +#define DRV_MONITOR_UNLOCK_PDL(Port) driver_monitor_unlock_pdl(Port) +#else +#define DRV_MONITOR_LOCK_PDL(Port) /* nothing */ +#define DRV_MONITOR_UNLOCK_PDL(Port) /* nothing */ +#endif static ERTS_INLINE ErlIOQueue* drvport2ioq(ErlDrvPort drvport) @@ -271,10 +280,36 @@ erts_test_next_port(int set, Uint next) return res; } + +static void port_cleanup(Port *prt); + +#ifdef ERTS_SMP + +static void +sched_port_cleanup(void *vprt) +{ + Port *prt = (Port *) vprt; + erts_smp_mtx_lock(prt->lock); + port_cleanup(prt); +} + +#endif + void erts_port_cleanup(Port *prt) { #ifdef ERTS_SMP + if (erts_smp_mtx_trylock(prt->lock) == EBUSY) + erts_schedule_misc_op(sched_port_cleanup, (void *) prt); + else +#endif + port_cleanup(prt); +} + +void +port_cleanup(Port *prt) +{ +#ifdef ERTS_SMP Uint32 port_specific; erts_smp_mtx_t *mtx; #endif @@ -1540,14 +1575,14 @@ static void deliver_read_message(Port* prt, Eterm to, pb = (ProcBin *) hp; pb->thing_word = HEADER_PROC_BIN; pb->size = len; - pb->next = ohp->mso; - ohp->mso = pb; + pb->next = ohp->first; + ohp->first = (struct erl_off_heap_header*)pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; hp += PROC_BIN_SIZE; - ohp->overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(ohp, pb->size / sizeof(Eterm)); listp = make_binary(pb); } @@ -1690,14 +1725,14 @@ deliver_vec_message(Port* prt, /* Port */ } pb->thing_word = HEADER_PROC_BIN; pb->size = iov->iov_len; - pb->next = ohp->mso; - ohp->mso = pb; + pb->next = ohp->first; + ohp->first = (struct erl_off_heap_header*)pb; pb->val = ErlDrvBinary2Binary(b); pb->bytes = base; pb->flags = 0; hp += PROC_BIN_SIZE; - ohp->overhead += iov->iov_len / sizeof(Eterm); + OH_OVERHEAD(ohp, iov->iov_len / sizeof(Eterm)); if (listp == NIL) { /* compatible with deliver_bin_message */ listp = make_binary(pb); @@ -1998,12 +2033,13 @@ erts_do_exit_port(Port *p, Eterm from, Eterm reason) p->nlinks = NULL; erts_sweep_links(lnk, &sweep_one_link, &sc); } + DRV_MONITOR_LOCK_PDL(p); { ErtsMonitor *moni = p->monitors; p->monitors = NULL; erts_sweep_monitors(moni, &sweep_one_monitor, NULL); } - + DRV_MONITOR_UNLOCK_PDL(p); if ((p->status & ERTS_PORT_SFLG_DISTRIBUTION) && p->dist_entry) { erts_do_net_exits(p->dist_entry, rreason); @@ -2223,12 +2259,12 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist) ProcBin* pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); pb->thing_word = HEADER_PROC_BIN; pb->size = dbin->orig_size; - pb->next = MSO(p).mso; - MSO(p).mso = pb; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*)pb; pb->val = ErlDrvBinary2Binary(dbin); pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; - MSO(p).overhead += dbin->orig_size / sizeof(Eterm); + OH_OVERHEAD(&(MSO(p)), dbin->orig_size / sizeof(Eterm)); return make_binary(pb); } port_resp = dbin->orig_bytes; @@ -2997,14 +3033,14 @@ driver_deliver_term(ErlDrvPort port, driver_binary_inc_refc(b); /* caller will free binary */ pb->thing_word = HEADER_PROC_BIN; pb->size = size; - pb->next = ohp->mso; - ohp->mso = pb; + pb->next = ohp->first; + ohp->first = (struct erl_off_heap_header*)pb; pb->val = ErlDrvBinary2Binary(b); pb->bytes = ((byte*) b->orig_bytes) + offset; pb->flags = 0; mess = make_binary(pb); hp += PROC_BIN_SIZE; - ohp->overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(ohp, pb->size / sizeof(Eterm)); } ptr += 3; break; @@ -3036,12 +3072,12 @@ driver_deliver_term(ErlDrvPort port, hp += PROC_BIN_SIZE; pbp->thing_word = HEADER_PROC_BIN; pbp->size = size; - pbp->next = ohp->mso; - ohp->mso = pbp; + pbp->next = ohp->first; + ohp->first = (struct erl_off_heap_header*)pbp; pbp->val = bp; pbp->bytes = (byte*) bp->orig_bytes; pbp->flags = 0; - ohp->overhead += (pbp->size / sizeof(Eterm)); + OH_OVERHEAD(ohp, pbp->size / sizeof(Eterm)); mess = make_binary(pbp); } ptr += 2; @@ -3536,6 +3572,32 @@ static ERTS_INLINE void pdl_destroy(ErlDrvPDL pdl) erts_free(ERTS_ALC_T_PORT_DATA_LOCK, pdl); } +#ifdef ERTS_SMP + +static void driver_monitor_lock_pdl(Port *p) { + if (p->port_data_lock) { + driver_pdl_lock(p->port_data_lock); + } + /* Now we either have the port lock or the port_data_lock */ + ERTS_LC_ASSERT(!p->port_data_lock + || erts_lc_mtx_is_locked(&(p->port_data_lock->mtx))); + ERTS_SMP_LC_ASSERT(p->port_data_lock + || erts_lc_is_port_locked(p)); +} + +static void driver_monitor_unlock_pdl(Port *p) { + /* We should either have the port lock or the port_data_lock */ + ERTS_LC_ASSERT(!p->port_data_lock + || erts_lc_mtx_is_locked(&(p->port_data_lock->mtx))); + ERTS_SMP_LC_ASSERT(p->port_data_lock + || erts_lc_is_port_locked(p)); + if (p->port_data_lock) { + driver_pdl_unlock(p->port_data_lock); + } +} + +#endif + /* * exported driver_pdl_* functions ... */ @@ -3994,7 +4056,7 @@ drv_cancel_timer(Port *prt) erts_port_task_abort(prt->id, &prt->timeout_task); } -int driver_set_timer(ErlDrvPort ix, Uint t) +int driver_set_timer(ErlDrvPort ix, UWord t) { Port* prt = erts_drvport2port(ix); @@ -4053,12 +4115,16 @@ driver_read_timer(ErlDrvPort ix, unsigned long* t) int driver_get_now(ErlDrvNowData *now_data) { + Uint mega,secs,micro; ERTS_SMP_CHK_NO_PROC_LOCKS; if (now_data == NULL) { return -1; } - get_now(&(now_data->megasecs),&(now_data->secs),&(now_data->microsecs)); + get_now(&mega,&secs,µ); + now_data->megasecs = (unsigned long) mega; + now_data->secs = (unsigned long) secs; + now_data->microsecs = (unsigned long) micro; return 0; } @@ -4072,14 +4138,15 @@ static void ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon) memcpy(mon,refp,sizeof(RefThing)); } -int driver_monitor_process(ErlDrvPort port, - ErlDrvTermData process, - ErlDrvMonitor *monitor) + +static int do_driver_monitor_process(Port *prt, + Eterm *buf, + ErlDrvTermData process, + ErlDrvMonitor *monitor) { - Port *prt = erts_drvport2port(port); Process *rp; Eterm ref; - Eterm buf[REF_THING_SIZE]; + if (prt->drv_ptr->process_exit == NULL) { return -1; } @@ -4089,22 +4156,76 @@ int driver_monitor_process(ErlDrvPort port, if (!rp) { return 1; } + ref = erts_make_ref_in_buffer(buf); erts_add_monitor(&(prt->monitors), MON_ORIGIN, ref, rp->id, NIL); erts_add_monitor(&(rp->monitors), MON_TARGET, ref, prt->id, NIL); - + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); ref_to_driver_monitor(ref,monitor); return 0; } -int driver_demonitor_process(ErlDrvPort port, - const ErlDrvMonitor *monitor) +/* + * This can be called from a non scheduler thread iff a port_data_lock exists + */ +int driver_monitor_process(ErlDrvPort port, + ErlDrvTermData process, + ErlDrvMonitor *monitor) +{ + Port *prt; + int ret; + Uint32 status; + ErtsSchedulerData *sched = erts_get_scheduler_data(); + int ix = (int) port; + if (ix < 0 || erts_max_ports <= ix) { + return -1; + } + prt = &erts_port[ix]; + + DRV_MONITOR_LOCK_PDL(prt); + + if (sched) { + status = erts_port[ix].status; + } else { + erts_smp_port_state_lock(prt); + status = erts_port[ix].status; + erts_smp_port_state_unlock(prt); + } + + if (status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) { + DRV_MONITOR_UNLOCK_PDL(prt); + return -1; + } + + /* Now (in SMP) we should have either the port lock (if we have a scheduler) or the port data lock + (if we're a driver thread) */ + ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock)); + +#if !HEAP_ON_C_STACK + if (!sched) { + /* Need a separate allocation for the ref :( */ + Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM, + sizeof(Eterm)*REF_THING_SIZE); + ret = do_driver_monitor_process(prt,buf,process,monitor); + erts_free(ERTS_ALC_T_TEMP_TERM,buf); + } else +#endif + { + DeclareTmpHeapNoproc(buf,REF_THING_SIZE); + UseTmpHeapNoproc(REF_THING_SIZE); + ret = do_driver_monitor_process(prt,buf,process,monitor); + UnUseTmpHeapNoproc(REF_THING_SIZE); + } + DRV_MONITOR_UNLOCK_PDL(prt); + return ret; +} + +static int do_driver_demonitor_process(Port *prt, Eterm *buf, + const ErlDrvMonitor *monitor) { - Port *prt = erts_drvport2port(port); Process *rp; Eterm ref; - Eterm buf[REF_THING_SIZE]; ErtsMonitor *mon; Eterm to; @@ -4137,12 +4258,60 @@ int driver_demonitor_process(ErlDrvPort port, return 0; } -ErlDrvTermData driver_get_monitored_process(ErlDrvPort port, +int driver_demonitor_process(ErlDrvPort port, + const ErlDrvMonitor *monitor) +{ + Port *prt; + int ret; + Uint32 status; + ErtsSchedulerData *sched = erts_get_scheduler_data(); + int ix = (int) port; + if (ix < 0 || erts_max_ports <= ix) { + return -1; + } + prt = &erts_port[ix]; + + DRV_MONITOR_LOCK_PDL(prt); + + if (sched) { + status = erts_port[ix].status; + } else { + erts_smp_port_state_lock(prt); + status = erts_port[ix].status; + erts_smp_port_state_unlock(prt); + } + + if (status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) { + DRV_MONITOR_UNLOCK_PDL(prt); + return -1; + } + + /* Now we should have either the port lock (if we have a scheduler) or the port data lock + (if we're a driver thread) */ + ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock)); +#if !HEAP_ON_C_STACK + if (!sched) { + /* Need a separate allocation for the ref :( */ + Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM, + sizeof(Eterm)*REF_THING_SIZE); + ret = do_driver_demonitor_process(prt,buf,monitor); + erts_free(ERTS_ALC_T_TEMP_TERM,buf); + } else +#endif + { + DeclareTmpHeapNoproc(buf,REF_THING_SIZE); + UseTmpHeapNoproc(REF_THING_SIZE); + ret = do_driver_demonitor_process(prt,buf,monitor); + UnUseTmpHeapNoproc(REF_THING_SIZE); + } + DRV_MONITOR_UNLOCK_PDL(prt); + return ret; +} + +static ErlDrvTermData do_driver_get_monitored_process(Port *prt, Eterm *buf, const ErlDrvMonitor *monitor) { - Port *prt = erts_drvport2port(port); Eterm ref; - Eterm buf[REF_THING_SIZE]; ErtsMonitor *mon; Eterm to; @@ -4158,6 +4327,59 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort port, return (ErlDrvTermData) to; } + +ErlDrvTermData driver_get_monitored_process(ErlDrvPort port, + const ErlDrvMonitor *monitor) +{ + Port *prt; + ErlDrvTermData ret; + Uint32 status; + ErtsSchedulerData *sched = erts_get_scheduler_data(); + int ix = (int) port; + if (ix < 0 || erts_max_ports <= ix) { + return driver_term_nil; + } + prt = &erts_port[ix]; + + DRV_MONITOR_LOCK_PDL(prt); + + if (sched) { + status = erts_port[ix].status; + } else { + erts_smp_port_state_lock(prt); + status = erts_port[ix].status; + erts_smp_port_state_unlock(prt); + } + + if (status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) { + DRV_MONITOR_UNLOCK_PDL(prt); + return driver_term_nil; + } + + /* Now we should have either the port lock (if we have a scheduler) or the port data lock + (if we're a driver thread) */ + ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock)); + +#if !HEAP_ON_C_STACK + if (!sched) { + /* Need a separate allocation for the ref :( */ + Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM, + sizeof(Eterm)*REF_THING_SIZE); + ret = do_driver_get_monitored_process(prt,buf,monitor); + erts_free(ERTS_ALC_T_TEMP_TERM,buf); + } else +#endif + { + DeclareTmpHeapNoproc(buf,REF_THING_SIZE); + UseTmpHeapNoproc(REF_THING_SIZE); + ret = do_driver_get_monitored_process(prt,buf,monitor); + UnUseTmpHeapNoproc(REF_THING_SIZE); + } + DRV_MONITOR_UNLOCK_PDL(prt); + return ret; +} + + int driver_compare_monitors(const ErlDrvMonitor *monitor1, const ErlDrvMonitor *monitor2) { @@ -4173,18 +4395,22 @@ void erts_fire_port_monitor(Port *prt, Eterm ref) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); ASSERT(prt->drv_ptr != NULL); - + DRV_MONITOR_LOCK_PDL(prt); if (erts_lookup_monitor(prt->monitors,ref) == NULL) { + DRV_MONITOR_UNLOCK_PDL(prt); return; } callback = prt->drv_ptr->process_exit; ASSERT(callback != NULL); ref_to_driver_monitor(ref,&drv_monitor); + DRV_MONITOR_UNLOCK_PDL(prt); fpe_was_unmasked = erts_block_fpe(); (*callback)((ErlDrvData) (prt->drv_data), &drv_monitor); erts_unblock_fpe(fpe_was_unmasked); + DRV_MONITOR_LOCK_PDL(prt); /* remove monitor *after* callback */ rmon = erts_remove_monitor(&(prt->monitors),ref); + DRV_MONITOR_UNLOCK_PDL(prt); if (rmon) { erts_destroy_monitor(rmon); } diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 87d13b3607..694e4ab72f 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -29,8 +29,8 @@ typedef struct erl_module { IndexSlot slot; /* Must be located at top of struct! */ int module; /* Atom index for module (not tagged). */ - Eterm* code; - Eterm* old_code; + BeamInstr* code; + BeamInstr* old_code; int code_length; /* Length of loaded code in bytes. */ int old_code_length; /* Length of old loaded code in bytes */ unsigned catches, old_catches; diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index ce1df74f03..a2439d5582 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -1,19 +1,19 @@ # # %CopyrightBegin% -# -# Copyright Ericsson AB 1997-2009. All Rights Reserved. -# +# +# Copyright Ericsson AB 1997-2010. All Rights Reserved. +# # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in # compliance with the License. You should have received a copy of the # Erlang Public License along with this software. If not, it can be # retrieved online at http://www.erlang.org/. -# +# # Software distributed under the License is distributed on an "AS IS" # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See # the License for the specific language governing rights and limitations # under the License. -# +# # %CopyrightEnd% # @@ -60,12 +60,17 @@ func_info M=a a==am_module_info A=u==0 | label L | move n r => too_old_compiler func_info M=a a==am_module_info A=u==1 | label L | move n r => too_old_compiler # The undocumented and unsupported guard BIF is_constant/1 was removed -# in R13. The is_constant/2 operation is marked as obosolete in genop.tab, +# in R13. The is_constant/2 operation is marked as obsolete in genop.tab, # so the loader will automatically generate a too_old_compiler message # it is used, but we need to handle the is_constant/1 BIF specially here. bif1 Fail u$func:erlang:is_constant/1 Src Dst => too_old_compiler +# Since the constant pool was introduced in R12B, empty tuples ({}) +# are literals. Therefore we no longer need to allow put_tuple/2 +# with a tuple size of zero. + +put_tuple u==0 d => too_old_compiler # # All the other instructions. @@ -79,6 +84,8 @@ i_trace_breakpoint i_mtrace_breakpoint i_debug_breakpoint i_count_breakpoint +i_time_breakpoint +i_return_time_trace i_return_to_trace i_yield i_global_cons @@ -115,12 +122,6 @@ init Y1 | init Y2 => init2 Y1 Y2 %macro: init2 Init2 -pack %macro: init3 Init3 -pack -# -# Warning: The put_string instruction is specially treated in the loader. -# Don't change the instruction format unless you change the loader too. -# -put_string I I d - # Selecting values select_val S=q Fail=f Size=u Rest=* => const_select_val(S, Fail, Size, Rest) @@ -335,11 +336,8 @@ i_is_eq_immed f y c # Putting things. # -put_tuple u==0 Dst => i_put_tuple_only u Dst put_tuple Arity Dst | put V => i_put_tuple Arity V Dst -i_put_tuple_only A d - %macro: i_put_tuple PutTuple -pack i_put_tuple A x x i_put_tuple A y x @@ -1184,12 +1182,6 @@ i_bs_init_bits_fail_heap I j I d i_bs_init_bits I I d i_bs_init_bits_heap I I I d -bs_bits_to_bytes Fail Src Dst => i_bs_bits_to_bytes Src Fail Dst - -i_bs_bits_to_bytes r j d -i_bs_bits_to_bytes x j d -i_bs_bits_to_bytes y j d - bs_add Fail S1=i==0 S2 Unit=u==1 D => move S2 D bs_add Fail S1 S2 Unit D => i_fetch S1 S2 | i_bs_add Fail Unit D @@ -1311,7 +1303,7 @@ fmul p FR1 FR2 FR3 => i_fmul FR1 FR2 FR3 fdiv p FR1 FR2 FR3 => i_fdiv FR1 FR2 FR3 fnegate p FR1 FR2 => i_fnegate FR1 FR2 -fconv Int=iq Dst=l => move Int x | fconv x Dst +fconv Arg=iqan Dst=l => move Arg x | fconv x Dst fmove q l fmove d l @@ -1397,34 +1389,60 @@ bif1 Fail u$bif:erlang:trunc/1 s d => too_old_compiler # Guard BIFs. # gc_bif1 Fail I Bif=u$bif:erlang:length/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) gc_bif1 Fail I Bif=u$bif:erlang:size/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) gc_bif1 Fail I Bif=u$bif:erlang:bit_size/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) gc_bif1 Fail I Bif=u$bif:erlang:byte_size/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) gc_bif1 Fail I Bif=u$bif:erlang:abs/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) gc_bif1 Fail I Bif=u$bif:erlang:float/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) gc_bif1 Fail I Bif=u$bif:erlang:round/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) gc_bif1 Fail I Bif=u$bif:erlang:trunc/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) + +gc_bif2 Fail I Bif=u$bif:erlang:binary_part/2 S1 S2 Dst=d => \ + gen_guard_bif2(Fail, I, Bif, S1, S2, Dst) + +gc_bif3 Fail I Bif=u$bif:erlang:binary_part/3 S1 S2 S3 Dst=d => \ + gen_guard_bif3(Fail, I, Bif, S1, S2, S3, Dst) i_gc_bif1 Fail Bif V=q Live D => move V x | i_gc_bif1 Fail Bif x Live D i_gc_bif1 j I s I d +ii_gc_bif2/6 + +ii_gc_bif2 Fail Bif S1 S2 Live D => i_fetch S1 S2 | i_gc_bif2 Fail Bif Live D + +i_gc_bif2 j I I d + +ii_gc_bif3/7 + +ii_gc_bif3 Fail Bif S1 S2 S3 Live D => move S1 x | i_fetch S2 S3 | i_gc_bif3 Fail Bif x Live D + +i_gc_bif3 j I s I d # # R13B03 # on_load + +# +# R14A. +# +recv_mark f + +recv_set Fail | label Lbl | loop_rec Lf Reg => \ + i_recv_set | label Lbl | loop_rec Lf Reg +i_recv_set diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c index 8c8029d450..5bcd567b5f 100644 --- a/erts/emulator/beam/packet_parser.c +++ b/erts/emulator/beam/packet_parser.c @@ -679,7 +679,7 @@ int packet_parse_http(const char* buf, int len, int* statep, while (n && SP(ptr)) { ptr++; n--; } - if (ptr==p0) return -1; + if (ptr==p0 && n>0) return -1; /* NOTE: the syntax allows empty reason phrases */ (*statep) = !0; diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c index 964c10a380..c9bb7bbe91 100644 --- a/erts/emulator/beam/register.c +++ b/erts/emulator/beam/register.c @@ -39,8 +39,6 @@ static Hash process_reg; static erts_smp_rwmtx_t regtab_rwmtx; -#define reg_lock_init() erts_smp_rwmtx_init(®tab_rwmtx, \ - "reg_tab") #define reg_try_read_lock() erts_smp_rwmtx_tryrlock(®tab_rwmtx) #define reg_try_write_lock() erts_smp_rwmtx_tryrwlock(®tab_rwmtx) #define reg_read_lock() erts_smp_rwmtx_rlock(®tab_rwmtx) @@ -147,8 +145,11 @@ static void reg_free(RegProc *obj) void init_register_table(void) { HashFunctions f; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; - reg_lock_init(); + erts_smp_rwmtx_init_opt(®tab_rwmtx, &rwmtx_opt, "reg_tab"); f.hash = (H_FUN) reg_hash; f.cmp = (HCMP_FUN) reg_cmp; @@ -476,8 +477,9 @@ int erts_unregister_name(Process *c_p, * on c_prt. */ - if (!c_p) + if (!c_p) { c_p_locks = 0; + } current_c_p_locks = c_p_locks; restart: @@ -489,9 +491,15 @@ int erts_unregister_name(Process *c_p, if (is_non_value(name)) { /* Unregister current process name */ ASSERT(c_p); - if (c_p->reg) +#ifdef ERTS_SMP + if (current_c_p_locks != c_p_locks) { + erts_smp_proc_lock(c_p, c_p_locks); + current_c_p_locks = c_p_locks; + } +#endif + if (c_p->reg) { r.name = c_p->reg->name; - else { + } else { /* Name got unregistered while main lock was released */ res = 0; goto done; @@ -533,24 +541,25 @@ int erts_unregister_name(Process *c_p, } } else if (rp->p) { - Process* p = rp->p; + #ifdef ERTS_SMP erts_proc_safelock(c_p, current_c_p_locks, c_p_locks, rp->p, - 0, + (c_p == rp->p) ? current_c_p_locks : 0, ERTS_PROC_LOCK_MAIN); current_c_p_locks = c_p_locks; #endif - p->reg = NULL; + rp->p->reg = NULL; + if (IS_TRACED_FL(rp->p, F_TRACE_PROCS)) { + trace_proc(c_p, rp->p, am_unregister, r.name); + } #ifdef ERTS_SMP - if (rp->p != c_p) + if (rp->p != c_p) { erts_smp_proc_unlock(rp->p, ERTS_PROC_LOCK_MAIN); -#endif - if (IS_TRACED_FL(p, F_TRACE_PROCS)) { - trace_proc(c_p, p, am_unregister, r.name); } +#endif } hash_erase(&process_reg, (void*) &r); res = 1; @@ -560,14 +569,17 @@ int erts_unregister_name(Process *c_p, reg_write_unlock(); if (c_prt != port) { - if (port) + if (port) { erts_smp_port_unlock(port); - if (c_prt) + } + if (c_prt) { erts_smp_port_lock(c_prt); + } } #ifdef ERTS_SMP - if (c_p && !current_c_p_locks) + if (c_p && !current_c_p_locks) { erts_smp_proc_lock(c_p, c_p_locks); + } #endif return res; } diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 4b949523fa..0031568af6 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -1,37 +1,30 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ #ifndef __SYS_H__ #define __SYS_H__ + #if defined(VALGRIND) && !defined(NO_FPE_SIGNALS) # define NO_FPE_SIGNALS #endif -/* Never use elib-malloc when purify-memory-tracing */ -#if defined(PURIFY) -#undef ENABLE_ELIB_MALLOC -#undef ELIB_HEAP_SBRK -#undef ELIB_ALLOC_IS_CLIB -#endif - - /* xxxP __VXWORKS__ */ #ifdef VXWORKS #include <vxWorks.h> @@ -57,8 +50,6 @@ # include "erl_win_sys.h" #elif defined (VXWORKS) # include "erl_vxworks_sys.h" -#elif defined (_OSE_) -# include "erl_ose_sys.h" #else # include "erl_unix_sys.h" #ifndef UNIX @@ -172,23 +163,6 @@ void erl_assert_error(char* expr, char* file, int line); #include <stdarg.h> -#if defined(__STDC__) || defined(_MSC_VER) -# define EXTERN_FUNCTION(t, f, x) extern t f x -# define FUNCTION(t, f, x) t f x -# define _DOTS_ ... -# define _VOID_ void -#elif defined(__cplusplus) -# define EXTERN_FUNCTION(f, x) extern "C" { f x } -# define FUNCTION(t, f, x) t f x -# define _DOTS_ ... -# define _VOID_ void -#else -# define EXTERN_FUNCTION(t, f, x) extern t f (/*x*/) -# define FUNCTION(t, f, x) t f (/*x*/) -# define _DOTS_ -# define _VOID_ -#endif - /* This isn't sys-dependent, but putting it here benefits sys.c and drivers - allow use of 'const' regardless of compiler */ @@ -198,7 +172,7 @@ void erl_assert_error(char* expr, char* file, int line); #ifdef VXWORKS /* Replace VxWorks' printf with a real one that does fprintf(stdout, ...) */ -EXTERN_FUNCTION(int, real_printf, (const char *fmt, ...)); +int real_printf(const char *fmt, ...); # define printf real_printf #endif @@ -230,9 +204,14 @@ EXTERN_FUNCTION(int, real_printf, (const char *fmt, ...)); ** Data types: ** ** Eterm: A tagged erlang term (possibly 64 bits) +** BeamInstr: A beam code instruction unit, possibly larger than Eterm, not smaller. ** UInt: An unsigned integer exactly as large as an Eterm. ** SInt: A signed integer exactly as large as an eterm and therefor large ** enough to hold the return value of the signed_val() macro. +** UWord: An unsigned integer at least as large as a void * and also as large +** or larger than an Eterm +** SWord: A signed integer at least as large as a void * and also as large +** or larger than an Eterm ** Uint32: An unsigned integer of 32 bits exactly ** Sint32: A signed integer of 32 bits exactly ** Uint16: An unsigned integer of 16 bits exactly @@ -253,11 +232,43 @@ EXTERN_FUNCTION(int, real_printf, (const char *fmt, ...)); #else #error Neither 32 nor 64 bit architecture #endif +#ifdef ARCH_64 +# ifdef HALFWORD_HEAP_EMULATOR +# define HALFWORD_HEAP 1 +# define HALFWORD_ASSERT 0 +# else +# define HALFWORD_HEAP 0 +# define HALFWORD_ASSERT 0 +# endif +#endif #if SIZEOF_VOID_P != SIZEOF_SIZE_T #error sizeof(void*) != sizeof(size_t) #endif +#if HALFWORD_HEAP + +#if SIZEOF_INT == 4 +typedef unsigned int Eterm; +typedef unsigned int Uint; +typedef int Sint; +#define ERTS_SIZEOF_ETERM SIZEOF_INT +#else +#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint' +#endif + +#if SIZEOF_VOID_P == SIZEOF_LONG +typedef unsigned long UWord; +typedef long SWord; +#elif SIZEOF_VOID_P == SIZEOF_INT +typedef unsigned int UWord; +typedef int SWord; +#else +#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint' +#endif + +#else /* !HALFWORD_HEAP */ + #if SIZEOF_VOID_P == SIZEOF_LONG typedef unsigned long Eterm; typedef unsigned long Uint; @@ -272,6 +283,13 @@ typedef int Sint; #error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint' #endif +typedef Uint UWord; +typedef Sint SWord; + +#endif /* HALFWORD_HEAP */ + +typedef UWord BeamInstr; + #ifndef HAVE_INT64 #if SIZEOF_LONG == 8 #define HAVE_INT64 1 @@ -316,15 +334,8 @@ typedef unsigned char byte; #error 64-bit architecture, but no appropriate type to use for Uint64 and Sint64 found #endif -#if defined(ARCH_64) -# define ERTS_WORD_ALIGN_PAD_SZ(X) \ +# define ERTS_EXTRA_DATA_ALIGN_SZ(X) \ (((size_t) 8) - (((size_t) (X)) & ((size_t) 7))) -#elif defined(ARCH_32) -# define ERTS_WORD_ALIGN_PAD_SZ(X) \ - (((size_t) 4) - (((size_t) (X)) & ((size_t) 3))) -#else -#error "Not supported..." -#endif #include "erl_lock_check.h" #include "erl_smp.h" @@ -413,13 +424,6 @@ extern volatile int erts_writing_erl_crash_dump; in non-blocking mode - and ioctl FIONBIO on AIX *doesn't* work for pipes or ttys (O_NONBLOCK does)!!! For now, we'll use FIONBIO for AIX. */ -# ifdef _OSE_ -static const int zero_value = 0, one_value = 1; -# define SET_BLOCKING(fd) ioctl((fd), FIONBIO, (char*)&zero_value) -# define SET_NONBLOCKING(fd) ioctl((fd), FIONBIO, (char*)&one_value) -# define ERRNO_BLOCK EWOULDBLOCK -# else - # ifdef __WIN32__ static unsigned long zero_value = 0, one_value = 1; @@ -460,7 +464,6 @@ static const int zero_value = 0, one_value = 1; # endif /* !NB_FIONBIO */ # endif /* _WXWORKS_ */ # endif /* !__WIN32__ */ -# endif /* _OSE_ */ #endif /* WANT_NONBLOCKING */ extern erts_cpu_info_t *erts_cpuinfo; /* erl_init.c */ @@ -538,8 +541,6 @@ typedef struct preload { * None of the drivers use all of the fields. */ -/* OSE: Want process_type and priority in here as well! Needs updates in erl_bif_ports.c! */ - typedef struct _SysDriverOpts { int ifd; /* Input file descriptor (fd driver). */ int ofd; /* Outputfile descriptor (fd driver). */ @@ -556,12 +557,6 @@ typedef struct _SysDriverOpts { char *wd; /* Working directory. */ unsigned spawn_type; /* Bitfield of ERTS_SPAWN_DRIVER | ERTS_SPAWN_EXTERNAL | both*/ - -#ifdef _OSE_ - enum PROCESS_TYPE process_type; - OSPRIORITY priority; -#endif /* _OSE_ */ - } SysDriverOpts; extern char *erts_default_arg0; @@ -621,7 +616,7 @@ extern char *erts_sys_ddll_error(int code); /* - * System interfaces for startup/sae code (functions found in respective sys.c) + * System interfaces for startup. */ @@ -638,18 +633,14 @@ extern void erts_sys_pre_init(void); extern void erl_sys_init(void); extern void erl_sys_args(int *argc, char **argv); extern void erl_sys_schedule(int); -#ifdef _OSE_ -extern void erl_sys_init_final(void); -#else -void sys_tty_reset(void); -#endif +void sys_tty_reset(int); -EXTERN_FUNCTION(int, sys_max_files, (_VOID_)); +int sys_max_files(void); void sys_init_io(void); Preload* sys_preloaded(void); -EXTERN_FUNCTION(unsigned char*, sys_preload_begin, (Preload*)); -EXTERN_FUNCTION(void, sys_preload_end, (Preload*)); -EXTERN_FUNCTION(int, sys_get_key, (int)); +unsigned char* sys_preload_begin(Preload*); +void sys_preload_end(Preload*); +int sys_get_key(int); void elapsed_time_both(unsigned long *ms_user, unsigned long *ms_sys, unsigned long *ms_user_diff, unsigned long *ms_sys_diff); void wall_clock_elapsed_time_both(unsigned long *ms_total, @@ -666,7 +657,7 @@ int local_to_univ(Sint *year, Sint *month, Sint *day, Sint *hour, Sint *minute, Sint *second, int isdst); void get_now(Uint*, Uint*, Uint*); void get_sys_now(Uint*, Uint*, Uint*); -EXTERN_FUNCTION(void, set_break_quit, (void (*)(void), void (*)(void))); +void set_break_quit(void (*)(void), void (*)(void)); void os_flavor(char*, unsigned); void os_version(int*, int*, int*); @@ -706,7 +697,7 @@ int erts_write_env(char *key, char *value); #define ERTS_DEFAULT_MMAP_THRESHOLD (128 * 1024) #define ERTS_DEFAULT_MMAP_MAX 64 -EXTERN_FUNCTION(int, sys_alloc_opt, (int, int)); +int sys_alloc_opt(int, int); typedef struct { Sint trim_threshold; @@ -715,7 +706,7 @@ typedef struct { Sint mmap_max; } SysAllocStat; -EXTERN_FUNCTION(void, sys_alloc_stat, (SysAllocStat *)); +void sys_alloc_stat(SysAllocStat *); /* Block the whole system... */ @@ -1104,13 +1095,10 @@ erts_refc_read(erts_refc_t *refcp, long min_val) extern int erts_use_kernel_poll; #endif -void elib_ensure_initialized(void); - - -#if (defined(VXWORKS) || defined(_OSE_)) +#if defined(VXWORKS) /* NOTE! sys_calloc2 does not exist on other platforms than VxWorks and OSE */ -EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint)); +void* sys_calloc2(Uint, Uint); #endif /* VXWORKS || OSE */ @@ -1150,14 +1138,14 @@ EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint)); /* Standard set of integer macros .. */ -#define get_int64(s) ((((unsigned char*) (s))[0] << 56) | \ - (((unsigned char*) (s))[1] << 48) | \ - (((unsigned char*) (s))[2] << 40) | \ - (((unsigned char*) (s))[3] << 32) | \ - (((unsigned char*) (s))[4] << 24) | \ - (((unsigned char*) (s))[5] << 16) | \ - (((unsigned char*) (s))[6] << 8) | \ - (((unsigned char*) (s))[7])) +#define get_int64(s) (((Uint64)(((unsigned char*) (s))[0]) << 56) | \ + (((Uint64)((unsigned char*) (s))[1]) << 48) | \ + (((Uint64)((unsigned char*) (s))[2]) << 40) | \ + (((Uint64)((unsigned char*) (s))[3]) << 32) | \ + (((Uint64)((unsigned char*) (s))[4]) << 24) | \ + (((Uint64)((unsigned char*) (s))[5]) << 16) | \ + (((Uint64)((unsigned char*) (s))[6]) << 8) | \ + (((Uint64)((unsigned char*) (s))[7]))) #define put_int64(i, s) do {((char*)(s))[0] = (char)((Sint64)(i) >> 56) & 0xff;\ ((char*)(s))[1] = (char)((Sint64)(i) >> 48) & 0xff;\ @@ -1202,8 +1190,8 @@ EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint)); */ #ifdef DEBUG -EXTERN_FUNCTION(void, erl_debug, (char* format, ...)); -EXTERN_FUNCTION(void, erl_bin_write, (unsigned char *, int, int)); +void erl_debug(char* format, ...); +void erl_bin_write(unsigned char *, int, int); # define DEBUGF(x) erl_debug x #else diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 31efddc0f2..ab5e8b5d4a 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -32,6 +32,7 @@ #include "erl_binary.h" #include "erl_bits.h" #include "packet_parser.h" +#include "erl_gc.h" #define ERTS_WANT_DB_INTERNAL__ #include "erl_db.h" #include "erl_threads.h" @@ -47,11 +48,11 @@ #undef M_MMAP_THRESHOLD #undef M_MMAP_MAX -#if !defined(ELIB_ALLOC_IS_CLIB) && defined(__GLIBC__) && defined(HAVE_MALLOC_H) +#if defined(__GLIBC__) && defined(HAVE_MALLOC_H) #include <malloc.h> #endif -#if defined(ELIB_ALLOC_IS_CLIB) || !defined(HAVE_MALLOPT) +#if !defined(HAVE_MALLOPT) #undef HAVE_MALLOPT #define HAVE_MALLOPT 0 #endif @@ -125,7 +126,7 @@ erts_heap_alloc(Process* p, Uint need) n = need; bp = MBUF(p); - if (bp != NULL && need <= (bp->size - bp->used_size)) { + if (bp != NULL && need <= (bp->alloc_size - bp->used_size)) { Eterm* ret = bp->mem + bp->used_size; bp->used_size += need; return ret; @@ -158,16 +159,11 @@ erts_heap_alloc(Process* p, Uint need) bp->next = MBUF(p); MBUF(p) = bp; - bp->size = n; + bp->alloc_size = n; bp->used_size = n; MBUF_SIZE(p) += n; - bp->off_heap.mso = NULL; -#ifndef HYBRID /* FIND ME! */ - bp->off_heap.funs = NULL; -#endif - bp->off_heap.externals = NULL; + bp->off_heap.first = NULL; bp->off_heap.overhead = 0; - return bp->mem; } @@ -204,6 +200,25 @@ erl_grow_stack(Eterm** start, Eterm** sp, Eterm** end) *end = *start + new_size; *sp = *start + sp_offs; } +/* + * Helper function for the ESTACK macros defined in global.h. + */ +void +erl_grow_wstack(UWord** start, UWord** sp, UWord** end) +{ + Uint old_size = (*end - *start); + Uint new_size = old_size * 2; + Uint sp_offs = *sp - *start; + if (new_size > 2 * DEF_ESTACK_SIZE) { + *start = erts_realloc(ERTS_ALC_T_ESTACK, (void *) *start, new_size*sizeof(UWord)); + } else { + UWord* new_ptr = erts_alloc(ERTS_ALC_T_ESTACK, new_size*sizeof(UWord)); + sys_memcpy(new_ptr, *start, old_size*sizeof(UWord)); + *start = new_ptr; + } + *end = *start + new_size; + *sp = *start + sp_offs; +} /* CTYPE macros */ @@ -354,6 +369,31 @@ erts_bld_uint(Uint **hpp, Uint *szp, Uint ui) return res; } +/* + * Erts_bld_uword is more or less similar to erts_bld_uint, but a pointer + * can safely be passed. + */ + +Eterm +erts_bld_uword(Uint **hpp, Uint *szp, UWord uw) +{ + Eterm res = THE_NON_VALUE; + if (IS_USMALL(0, uw)) { + if (hpp) + res = make_small((Uint) uw); + } + else { + if (szp) + *szp += BIG_UWORD_HEAP_SIZE(uw); + if (hpp) { + res = uword_to_big(uw, *hpp); + *hpp += BIG_UWORD_HEAP_SIZE(uw); + } + } + return res; +} + + Eterm erts_bld_uint64(Uint **hpp, Uint *szp, Uint64 ui64) { @@ -364,7 +404,7 @@ erts_bld_uint64(Uint **hpp, Uint *szp, Uint64 ui64) } else { if (szp) - *szp = ERTS_UINT64_HEAP_SIZE(ui64); + *szp += ERTS_UINT64_HEAP_SIZE(ui64); if (hpp) res = erts_uint64_to_big(ui64, hpp); } @@ -381,7 +421,7 @@ erts_bld_sint64(Uint **hpp, Uint *szp, Sint64 si64) } else { if (szp) - *szp = ERTS_SINT64_HEAP_SIZE(si64); + *szp += ERTS_SINT64_HEAP_SIZE(si64); if (hpp) res = erts_sint64_to_big(si64, hpp); } @@ -465,7 +505,7 @@ erts_bld_string_n(Uint **hpp, Uint *szp, const char *str, Sint len) if (hpp) { res = NIL; while (--i >= 0) { - res = CONS(*hpp, make_small(str[i]), res); + res = CONS(*hpp, make_small((byte) str[i]), res); *hpp += 2; } } @@ -711,7 +751,7 @@ hash_binary_bytes(Eterm bin, Uint sz, Uint32 hash) Uint32 make_hash(Eterm term_arg) { - DECLARE_ESTACK(stack); + DECLARE_WSTACK(stack); Eterm term = term_arg; Eterm hash = 0; unsigned op; @@ -770,7 +810,7 @@ tail_recur: Uint y2 = y1 < 0 ? -(Uint)y1 : y1; UINT32_HASH_STEP(y2, FUNNY_NUMBER2); -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP if (y2 >> 32) UINT32_HASH_STEP(y2 >> 32, FUNNY_NUMBER2); #endif @@ -787,7 +827,7 @@ tail_recur: } case EXPORT_DEF: { - Export* ep = (Export *) (export_val(term))[1]; + Export* ep = *((Export **) (export_val(term) + 1)); hash = hash * FUNNY_NUMBER11 + ep->code[2]; hash = hash*FUNNY_NUMBER1 + @@ -809,7 +849,7 @@ tail_recur: hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq; if (num_free > 0) { if (num_free > 1) { - ESTACK_PUSH3(stack, (Eterm) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP); + WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP); } term = funp->env[0]; goto tail_recur; @@ -837,9 +877,9 @@ tail_recur: } case MAKE_HASH_CDR_PRE_OP: - term = ESTACK_POP(stack); + term = (Eterm) WSTACK_POP(stack); if (is_not_list(term)) { - ESTACK_PUSH(stack, MAKE_HASH_CDR_POST_OP); + WSTACK_PUSH(stack, (UWord) MAKE_HASH_CDR_POST_OP); goto tail_recur; } /* fall through */ @@ -854,13 +894,13 @@ tail_recur: hash = hash*FUNNY_NUMBER2 + unsigned_val(*list); if (is_not_list(CDR(list))) { - ESTACK_PUSH(stack, MAKE_HASH_CDR_POST_OP); + WSTACK_PUSH(stack, MAKE_HASH_CDR_POST_OP); term = CDR(list); goto tail_recur; } list = list_val(CDR(list)); } - ESTACK_PUSH2(stack, CDR(list), MAKE_HASH_CDR_PRE_OP); + WSTACK_PUSH2(stack, CDR(list), MAKE_HASH_CDR_PRE_OP); term = CAR(list); goto tail_recur; } @@ -888,7 +928,7 @@ tail_recur: } d = BIG_DIGIT(ptr, k); k = sizeof(ErtsDigit); -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP if (!(d >> 32)) k /= 2; #endif @@ -904,21 +944,21 @@ tail_recur: Eterm* ptr = tuple_val(term); Uint arity = arityval(*ptr); - ESTACK_PUSH3(stack, arity, (Eterm)(ptr+1), arity); + WSTACK_PUSH3(stack, (UWord) arity, (UWord)(ptr+1), (UWord) arity); op = MAKE_HASH_TUPLE_OP; }/*fall through*/ case MAKE_HASH_TUPLE_OP: case MAKE_HASH_FUN_OP: { - Uint i = ESTACK_POP(stack); - Eterm* ptr = (Eterm*) ESTACK_POP(stack); + Uint i = (Uint) WSTACK_POP(stack); + Eterm* ptr = (Eterm*) WSTACK_POP(stack); if (i != 0) { term = *ptr; - ESTACK_PUSH3(stack, (Eterm)(ptr+1), i-1, op); + WSTACK_PUSH3(stack, (UWord)(ptr+1), (UWord) i-1, (UWord) op); goto tail_recur; } if (op == MAKE_HASH_TUPLE_OP) { - Uint32 arity = ESTACK_POP(stack); + Uint32 arity = (Uint32) WSTACK_POP(stack); hash = hash*FUNNY_NUMBER9 + arity; } break; @@ -928,10 +968,10 @@ tail_recur: erl_exit(1, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op); return 0; } - if (ESTACK_ISEMPTY(stack)) break; - op = ESTACK_POP(stack); + if (WSTACK_ISEMPTY(stack)) break; + op = WSTACK_POP(stack); } - DESTROY_ESTACK(stack); + DESTROY_WSTACK(stack); return hash; #undef UINT32_HASH_STEP @@ -1002,7 +1042,7 @@ Uint32 make_hash2(Eterm term) { Uint32 hash; - Eterm tmp_big[2]; + DeclareTmpHeapNoproc(tmp_big,2); /* (HCONST * {2, ..., 14}) mod 2^32 */ #define HCONST_2 0x3c6ef372UL @@ -1041,7 +1081,6 @@ make_hash2(Eterm term) } while(0) #define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2) - /* Optimization. Simple cases before declaration of estack. */ if (primary_tag(term) == TAG_PRIMARY_IMMED1) { switch (term & _TAG_IMMED1_MASK) { @@ -1070,6 +1109,7 @@ make_hash2(Eterm term) Eterm tmp; DECLARE_ESTACK(s); + UseTmpHeapNoproc(2); hash = 0; for (;;) { switch (primary_tag(term)) { @@ -1123,7 +1163,7 @@ make_hash2(Eterm term) break; case EXPORT_SUBTAG: { - Export* ep = (Export *) (export_val(term))[1]; + Export* ep = *((Export **) (export_val(term) + 1)); UINT32_HASH_2 (ep->code[2], @@ -1314,6 +1354,7 @@ make_hash2(Eterm term) hash2_common: if (ESTACK_ISEMPTY(s)) { DESTROY_ESTACK(s); + UnUseTmpHeapNoproc(2); return hash; } term = ESTACK_POP(s); @@ -1332,7 +1373,7 @@ make_hash2(Eterm term) Uint32 make_broken_hash(Eterm term) { Uint32 hash = 0; - DECLARE_ESTACK(stack); + DECLARE_WSTACK(stack); unsigned op; tail_recur: op = tag_val_def(term); @@ -1346,7 +1387,7 @@ tail_recur: (atom_tab(atom_val(term))->slot.bucket.hvalue); break; case SMALL_DEF: -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP { Sint y1 = signed_val(term); Uint y2 = y1 < 0 ? -(Uint)y1 : y1; @@ -1399,7 +1440,7 @@ tail_recur: case EXPORT_DEF: { - Export* ep = (Export *) (export_val(term))[1]; + Export* ep = *((Export **) (export_val(term) + 1)); hash = hash * FUNNY_NUMBER11 + ep->code[2]; hash = hash*FUNNY_NUMBER1 + @@ -1421,7 +1462,7 @@ tail_recur: hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq; if (num_free > 0) { if (num_free > 1) { - ESTACK_PUSH3(stack, (Eterm) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP); + WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP); } term = funp->env[0]; goto tail_recur; @@ -1456,16 +1497,17 @@ tail_recur: break; case MAKE_HASH_CDR_PRE_OP: - term = ESTACK_POP(stack); + term = (Eterm) WSTACK_POP(stack); if (is_not_list(term)) { - ESTACK_PUSH(stack, MAKE_HASH_CDR_POST_OP); + WSTACK_PUSH(stack, (UWord) MAKE_HASH_CDR_POST_OP); goto tail_recur; } /*fall through*/ case LIST_DEF: { Eterm* list = list_val(term); - ESTACK_PUSH2(stack, CDR(list), MAKE_HASH_CDR_PRE_OP); + WSTACK_PUSH2(stack, (UWord) CDR(list), + (UWord) MAKE_HASH_CDR_PRE_OP); term = CAR(list); goto tail_recur; } @@ -1538,21 +1580,21 @@ tail_recur: Eterm* ptr = tuple_val(term); Uint arity = arityval(*ptr); - ESTACK_PUSH3(stack, arity, (Eterm)(ptr+1), arity); + WSTACK_PUSH3(stack, (UWord) arity, (UWord) (ptr+1), (UWord) arity); op = MAKE_HASH_TUPLE_OP; }/*fall through*/ case MAKE_HASH_TUPLE_OP: case MAKE_HASH_FUN_OP: { - Uint i = ESTACK_POP(stack); - Eterm* ptr = (Eterm*) ESTACK_POP(stack); + Uint i = (Uint) WSTACK_POP(stack); + Eterm* ptr = (Eterm*) WSTACK_POP(stack); if (i != 0) { term = *ptr; - ESTACK_PUSH3(stack, (Eterm)(ptr+1), i-1, op); + WSTACK_PUSH3(stack, (UWord)(ptr+1), (UWord) i-1, (UWord) op); goto tail_recur; } if (op == MAKE_HASH_TUPLE_OP) { - Uint32 arity = ESTACK_POP(stack); + Uint32 arity = (UWord) WSTACK_POP(stack); hash = hash*FUNNY_NUMBER9 + arity; } break; @@ -1562,11 +1604,11 @@ tail_recur: erl_exit(1, "Invalid tag in make_broken_hash\n"); return 0; } - if (ESTACK_ISEMPTY(stack)) break; - op = ESTACK_POP(stack); + if (WSTACK_ISEMPTY(stack)) break; + op = (Uint) WSTACK_POP(stack); } - DESTROY_ESTACK(stack); + DESTROY_WSTACK(stack); return hash; #undef MAKE_HASH_TUPLE_OP @@ -1869,7 +1911,7 @@ erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *dsbufp) int eq(Eterm a, Eterm b) { - DECLARE_ESTACK(stack); + DECLARE_WSTACK(stack); Sint sz; Eterm* aa; Eterm* bb; @@ -1887,7 +1929,7 @@ tailrecur_ne: Eterm atmp = CAR(aval); Eterm btmp = CAR(bval); if (atmp != btmp) { - ESTACK_PUSH2(stack,CDR(bval),CDR(aval)); + WSTACK_PUSH2(stack,(UWord) CDR(bval),(UWord) CDR(aval)); a = atmp; b = btmp; goto tailrecur_ne; @@ -1957,8 +1999,8 @@ tailrecur_ne: case EXPORT_SUBTAG: { if (is_export(b)) { - Export* a_exp = (Export *) (export_val(a))[1]; - Export* b_exp = (Export *) (export_val(b))[1]; + Export* a_exp = *((Export **) (export_val(a) + 1)); + Export* b_exp = *((Export **) (export_val(b) + 1)); if (a_exp == b_exp) goto pop_next; } break; /* not equal */ @@ -2130,32 +2172,32 @@ term_array: /* arrays in 'aa' and 'bb', length in 'sz' */ goto not_equal; } if (i > 1) { /* push the rest */ - ESTACK_PUSH3(stack, i-1, (Eterm)(bp+1), - ((Eterm)(ap+1)) | TAG_PRIMARY_HEADER); + WSTACK_PUSH3(stack, i-1, (UWord)(bp+1), + ((UWord)(ap+1)) | TAG_PRIMARY_HEADER); /* We (ab)use TAG_PRIMARY_HEADER to recognize a term_array */ } goto tailrecur_ne; } pop_next: - if (!ESTACK_ISEMPTY(stack)) { - Eterm something = ESTACK_POP(stack); - if (primary_tag(something) == TAG_PRIMARY_HEADER) { /* a term_array */ + if (!WSTACK_ISEMPTY(stack)) { + UWord something = WSTACK_POP(stack); + if (primary_tag((Eterm) something) == TAG_PRIMARY_HEADER) { /* a term_array */ aa = (Eterm*) something; - bb = (Eterm*) ESTACK_POP(stack); - sz = ESTACK_POP(stack); + bb = (Eterm*) WSTACK_POP(stack); + sz = WSTACK_POP(stack); goto term_array; } a = something; - b = ESTACK_POP(stack); + b = WSTACK_POP(stack); goto tailrecur; } - DESTROY_ESTACK(stack); + DESTROY_WSTACK(stack); return 1; not_equal: - DESTROY_ESTACK(stack); + DESTROY_WSTACK(stack); return 0; } @@ -2210,7 +2252,7 @@ static int cmp_atoms(Eterm a, Eterm b) Sint cmp(Eterm a, Eterm b) { - DECLARE_ESTACK(stack); + DECLARE_WSTACK(stack); Eterm* aa; Eterm* bb; int i; @@ -2327,7 +2369,7 @@ tailrecur_ne: Eterm atmp = CAR(aa); Eterm btmp = CAR(bb); if (atmp != btmp) { - ESTACK_PUSH2(stack,CDR(bb),CDR(aa)); + WSTACK_PUSH2(stack,(UWord) CDR(bb),(UWord) CDR(aa)); a = atmp; b = btmp; goto tailrecur_ne; @@ -2392,8 +2434,8 @@ tailrecur_ne: a_tag = EXPORT_DEF; goto mixed_types; } else { - Export* a_exp = (Export *) (export_val(a))[1]; - Export* b_exp = (Export *) (export_val(b))[1]; + Export* a_exp = *((Export **) (export_val(a) + 1)); + Export* b_exp = *((Export **) (export_val(b) + 1)); if ((j = cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) { RETURN_NEQ(j); @@ -2581,7 +2623,11 @@ tailrecur_ne: { FloatDef f1, f2; Eterm big; - Eterm big_buf[2]; +#if HEAP_ON_C_STACK + Eterm big_buf[2]; /* If HEAP_ON_C_STACK */ +#else + Eterm *big_buf = erts_get_scheduler_data()->cmp_tmp_heap; +#endif switch(_NUMBER_CODE(a_tag, b_tag)) { case SMALL_BIG: @@ -2644,7 +2690,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */ } } else { /* (ab)Use TAG_PRIMARY_HEADER to recognize a term_array */ - ESTACK_PUSH3(stack, i, (Eterm)bb, (Eterm)aa | TAG_PRIMARY_HEADER); + WSTACK_PUSH3(stack, i, (UWord)bb, (UWord)aa | TAG_PRIMARY_HEADER); goto tailrecur_ne; } } @@ -2654,20 +2700,20 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */ goto tailrecur; pop_next: - if (!ESTACK_ISEMPTY(stack)) { - Eterm something = ESTACK_POP(stack); - if (primary_tag(something) == TAG_PRIMARY_HEADER) { /* a term_array */ + if (!WSTACK_ISEMPTY(stack)) { + UWord something = WSTACK_POP(stack); + if (primary_tag((Eterm) something) == TAG_PRIMARY_HEADER) { /* a term_array */ aa = (Eterm*) something; - bb = (Eterm*) ESTACK_POP(stack); - i = ESTACK_POP(stack); + bb = (Eterm*) WSTACK_POP(stack); + i = WSTACK_POP(stack); goto term_array; } - a = something; - b = ESTACK_POP(stack); + a = (Eterm) something; + b = (Eterm) WSTACK_POP(stack); goto tailrecur; } - DESTROY_ESTACK(stack); + DESTROY_WSTACK(stack); return 0; not_equal: @@ -2678,21 +2724,8 @@ not_equal: } -void -erts_cleanup_externals(ExternalThing *etp) -{ - ExternalThing *tetp; - - tetp = etp; - - while(tetp) { - erts_deref_node_entry(tetp->node); - tetp = tetp->next; - } -} - Eterm -store_external_or_ref_(Uint **hpp, ExternalThing **etpp, Eterm ns) +store_external_or_ref_(Uint **hpp, ErlOffHeap* oh, Eterm ns) { Uint i; Uint size; @@ -2711,8 +2744,8 @@ store_external_or_ref_(Uint **hpp, ExternalThing **etpp, Eterm ns) erts_refc_inc(&((ExternalThing *) to_hp)->node->refc, 2); - ((ExternalThing *) to_hp)->next = *etpp; - *etpp = (ExternalThing *) to_hp; + ((struct erl_off_heap_header*) to_hp)->next = oh->first; + oh->first = (struct erl_off_heap_header*) to_hp; return make_external(to_hp); } @@ -2741,7 +2774,7 @@ store_external_or_ref_in_proc_(Process *proc, Eterm ns) sz = NC_HEAP_SIZE(ns); ASSERT(sz > 0); hp = HAlloc(proc, sz); - return store_external_or_ref_(&hp, &MSO(proc).externals, ns); + return store_external_or_ref_(&hp, &MSO(proc), ns); } void bin_write(int to, void *to_arg, byte* buf, int sz) diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c index 95510a16b2..c450f10f48 100644 --- a/erts/emulator/drivers/common/efile_drv.c +++ b/erts/emulator/drivers/common/efile_drv.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ /* @@ -53,6 +53,8 @@ #define FILE_IPREAD 27 #define FILE_ALTNAME 28 #define FILE_READ_LINE 29 +#define FILE_FDATASYNC 30 +#define FILE_FADVISE 31 /* Return codes */ @@ -102,7 +104,7 @@ #include <ctype.h> #include <sys/types.h> -extern void erl_exit(int n, char *fmt, _DOTS_); +void erl_exit(int n, char *fmt, ...); static ErlDrvSysInfo sys_info; @@ -196,9 +198,9 @@ enum e_timer {timer_idle, timer_again, timer_write}; struct t_data; typedef struct { - Sint fd; + SWord fd; ErlDrvPort port; - unsigned key; /* Async queue key */ + unsigned int key; /* Async queue key */ unsigned flags; /* Original flags from FILE_OPEN. */ void (*invoke)(void *); struct t_data *d; @@ -306,7 +308,7 @@ struct t_data int result_ok; Efile_error errInfo; int flags; - Sint fd; + SWord fd; /**/ Efile_info info; EFILE_DIR_HANDLE dir_handle; /* Handle to open directory. */ @@ -357,6 +359,11 @@ struct t_data struct t_readdir_buf *first_buf; struct t_readdir_buf *last_buf; } read_dir; + struct { + Sint64 offset; + Sint64 length; + int advise; + } fadvise; } c; char b[1]; }; @@ -605,7 +612,7 @@ file_start(ErlDrvPort port, char* command) } desc->fd = FILE_FD_INVALID; desc->port = port; - desc->key = (unsigned) (Uint) port; + desc->key = (unsigned int) (UWord) port; desc->flags = 0; desc->invoke = NULL; desc->d = NULL; @@ -630,7 +637,7 @@ static void free_data(void *data) EF_FREE(data); } -static void do_close(int flags, Sint fd) { +static void do_close(int flags, SWord fd) { if (flags & EFILE_COMPRESSED) { erts_gzclose((gzFile)(fd)); } else { @@ -709,7 +716,7 @@ static void reply_Uint_posix_error(file_descriptor *desc, Uint num, TRACE_C('N'); response[0] = FILE_RESP_NUMERR; -#if SIZEOF_VOID_P == 4 +#if SIZEOF_VOID_P == 4 || HALFWORD_HEAP put_int32(0, response+1); #else put_int32(num>>32, response+1); @@ -767,7 +774,7 @@ static int reply_Uint(file_descriptor *desc, Uint result) { TRACE_C('R'); tmp[0] = FILE_RESP_NUMBER; -#if SIZEOF_VOID_P == 4 +#if SIZEOF_VOID_P == 4 || HALFWORD_HEAP put_int32(0, tmp+1); #else put_int32(result>>32, tmp+1); @@ -883,6 +890,15 @@ static void invoke_chdir(void *data) invoke_name(data, efile_chdir); } +static void invoke_fdatasync(void *data) +{ + struct t_data *d = (struct t_data *) data; + int fd = (int) d->fd; + + d->again = 0; + d->result_ok = efile_fdatasync(&d->errInfo, fd); +} + static void invoke_fsync(void *data) { struct t_data *d = (struct t_data *) data; @@ -1620,7 +1636,7 @@ static void invoke_open(void *data) status = efile_may_openfile(&d->errInfo, d->b); if (status || (d->errInfo.posix_errno != EISDIR)) { mode = (d->flags & EFILE_MODE_READ) ? "rb" : "wb"; - d->fd = (Sint) erts_gzopen(d->b, mode); + d->fd = (SWord) erts_gzopen(d->b, mode); if ((gzFile)d->fd) { status = 1; } else { @@ -1637,6 +1653,18 @@ static void invoke_open(void *data) d->result_ok = status; } +static void invoke_fadvise(void *data) +{ + struct t_data *d = (struct t_data *) data; + int fd = (int) d->fd; + off_t offset = (off_t) d->c.fadvise.offset; + off_t length = (off_t) d->c.fadvise.length; + int advise = (int) d->c.fadvise.advise; + + d->again = 0; + d->result_ok = efile_fadvise(&d->errInfo, fd, offset, length, advise); +} + static void free_readdir(void *data) { struct t_data *d = (struct t_data *) data; @@ -1919,12 +1947,14 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) case FILE_RMDIR: case FILE_CHDIR: case FILE_DELETE: + case FILE_FDATASYNC: case FILE_FSYNC: case FILE_TRUNCATE: case FILE_LINK: case FILE_SYMLINK: case FILE_RENAME: case FILE_WRITE_INFO: + case FILE_FADVISE: reply(desc, d->result_ok, &d->errInfo); free_data(data); break; @@ -2209,6 +2239,18 @@ file_output(ErlDrvData e, char* buf, int count) goto done; } + case FILE_FDATASYNC: + { + d = EF_SAFE_ALLOC(sizeof(struct t_data)); + + d->fd = fd; + d->command = command; + d->invoke = invoke_fdatasync; + d->free = free_data; + d->level = 2; + goto done; + } + case FILE_FSYNC: { d = EF_SAFE_ALLOC(sizeof(struct t_data)); @@ -2332,6 +2374,21 @@ file_output(ErlDrvData e, char* buf, int count) goto done; } + case FILE_FADVISE: + { + d = EF_SAFE_ALLOC(sizeof(struct t_data)); + + d->fd = fd; + d->command = command; + d->invoke = invoke_fadvise; + d->free = free_data; + d->level = 2; + d->c.fadvise.offset = get_int64((uchar*) buf); + d->c.fadvise.length = get_int64(((uchar*) buf) + sizeof(Sint64)); + d->c.fadvise.advise = get_int32(((uchar*) buf) + 2 * sizeof(Sint64)); + goto done; + } + } /* diff --git a/erts/emulator/drivers/common/erl_efile.h b/erts/emulator/drivers/common/erl_efile.h index 9aa941e550..ac95c1f949 100644 --- a/erts/emulator/drivers/common/erl_efile.h +++ b/erts/emulator/drivers/common/erl_efile.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1997-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ /* @@ -32,7 +32,8 @@ #define EFILE_MODE_READ_WRITE 3 #define EFILE_MODE_APPEND 4 #define EFILE_COMPRESSED 8 -#define EFILE_NO_TRUNCATE 16 /* Special for reopening on VxWorks */ +#define EFILE_MODE_EXCL 16 +#define EFILE_NO_TRUNCATE 32 /* Special for reopening on VxWorks */ /* * Seek modes for efile_seek(). @@ -126,6 +127,7 @@ int efile_readdir(Efile_error* errInfo, char* name, int efile_openfile(Efile_error* errInfo, char* name, int flags, int* pfd, Sint64* pSize); void efile_closefile(int fd); +int efile_fdatasync(Efile_error* errInfo, int fd); int efile_fsync(Efile_error* errInfo, int fd); int efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, char *name, int info_for_link); @@ -150,3 +152,5 @@ int efile_altname(Efile_error* errInfo, char *name, int efile_link(Efile_error* errInfo, char* old, char* new); int efile_symlink(Efile_error* errInfo, char* old, char* new); int efile_may_openfile(Efile_error* errInfo, char *name); +int efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, Sint64 length, + int advise); diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index c6e23ee647..059288d1cb 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1997-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -41,14 +41,12 @@ #define STRINGIFY_1(b) IDENTITY(#b) #define STRINGIFY(a) STRINGIFY_1(a) -#ifndef _OSE_ #ifdef HAVE_UNISTD_H #include <unistd.h> #endif #ifdef HAVE_SYS_UIO_H #include <sys/uio.h> #endif -#endif /* All platforms fail on malloc errors. */ @@ -57,6 +55,21 @@ #include "erl_driver.h" +/* The IS_SOCKET_ERROR macro below is used for portability reasons. While + POSIX specifies that errors from socket-related system calls should be + indicated with a -1 return value, some users have experienced non-Windows + OS kernels that return negative values other than -1. While one can argue + that such kernels are technically broken, comparing against values less + than 0 covers their out-of-spec return values without imposing incorrect + semantics on systems that manage to correctly return -1 for errors, thus + increasing Erlang's portability. +*/ +#ifdef __WIN32__ +#define IS_SOCKET_ERROR(val) ((val) == SOCKET_ERROR) +#else +#define IS_SOCKET_ERROR(val) ((val) < 0) +#endif + #ifdef __WIN32__ #define STRNCASECMP strncasecmp @@ -186,18 +199,8 @@ static unsigned long one_value = 1; #include <netdb.h> #endif -#ifndef _OSE_ #include <sys/socket.h> #include <netinet/in.h> -#else -/* datatypes and macros from Solaris socket.h */ -struct linger { - int l_onoff; /* option on/off */ - int l_linger; /* linger time */ -}; -#define SO_OOBINLINE 0x0100 /* leave received OOB data in line */ -#define SO_LINGER 0x0080 /* linger on close if data present */ -#endif #ifdef VXWORKS #include <rpc/rpctypes.h> @@ -206,12 +209,10 @@ struct linger { #include <rpc/types.h> #endif -#ifndef _OSE_ #include <netinet/tcp.h> #include <arpa/inet.h> -#endif -#if (!defined(VXWORKS) && !defined(_OSE_)) +#if (!defined(VXWORKS)) #include <sys/param.h> #ifdef HAVE_ARPA_NAMESER_H #include <arpa/nameser.h> @@ -226,33 +227,11 @@ struct linger { #include <sys/ioctl.h> #endif -#ifndef _OSE_ #include <net/if.h> -#else -#define IFF_MULTICAST 0x00000800 -#endif - -#ifdef _OSE_ -#include "inet.h" -#include "ineterr.h" -#include "ose_inet_drv.h" -#include "nameser.h" -#include "resolv.h" -#define SET_ASYNC(s) setsockopt((s), SOL_SOCKET, SO_OSEEVENT, (&(s)), sizeof(int)) - -extern void select_release(void); - -#endif /* _OSE_ */ - -/* Solaris headers, only to be used with SFK */ -#ifdef _OSE_SFK_ -#include <ctype.h> -#include <string.h> -#endif /* SCTP support -- currently for UNIX platforms only: */ #undef HAVE_SCTP -#if (!defined(VXWORKS) && !defined(_OSE_) && !defined(__WIN32__) && defined(HAVE_SCTP_H)) +#if (!defined(VXWORKS) && !defined(__WIN32__) && defined(HAVE_SCTP_H)) #include <netinet/sctp.h> @@ -315,7 +294,7 @@ static int (*p_sctp_bindx)(int sd, struct sockaddr *addrs, #define DEBUGF(X) printf X #endif -#if !defined(__WIN32__) && !defined(HAVE_STRNCASECMP) +#if !defined(HAVE_STRNCASECMP) #define STRNCASECMP my_strncasecmp static int my_strncasecmp(const char *s1, const char *s2, size_t n) @@ -335,6 +314,7 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) #define INVALID_SOCKET -1 #define INVALID_EVENT -1 #define SOCKET_ERROR -1 + #define SOCKET int #define HANDLE long int #define FD_READ ERL_DRV_READ @@ -362,20 +342,6 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) #define sock_htons(x) htons((x)) #define sock_htonl(x) htonl((x)) -#ifdef _OSE_ -#define sock_accept(s, addr, len) ose_inet_accept((s), (addr), (len)) -#define sock_send(s,buf,len,flag) ose_inet_send((s),(buf),(len),(flag)) -#define sock_sendto(s,buf,blen,flag,addr,alen) \ - ose_inet_sendto((s),(buf),(blen),(flag),(addr),(alen)) -#define sock_sendv(s, vec, size, np, flag) \ - (*(np) = ose_inet_sendv((s), (SysIOVec*)(vec), (size))) -#define sock_open(af, type, proto) ose_inet_socket((af), (type), (proto)) -#define sock_close(s) ose_inet_close((s)) -#define sock_hostname(buf, len) ose_gethostname((buf), (len)) -#define sock_getservbyname(name,proto) ose_getservbyname((name), (proto)) -#define sock_getservbyport(port,proto) ose_getservbyport((port), (proto)) - -#else #define sock_accept(s, addr, len) accept((s), (addr), (len)) #define sock_send(s,buf,len,flag) send((s),(buf),(len),(flag)) #define sock_sendto(s,buf,blen,flag,addr,alen) \ @@ -391,7 +357,6 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) #define sock_hostname(buf, len) gethostname((buf), (len)) #define sock_getservbyname(name,proto) getservbyname((name), (proto)) #define sock_getservbyport(port,proto) getservbyport((port), (proto)) -#endif /* _OSE_ */ #define sock_recv(s,buf,len,flag) recv((s),(buf),(len),(flag)) #define sock_recvfrom(s,buf,blen,flag,addr,alen) \ @@ -402,13 +367,8 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) #define sock_create_event(d) ((d)->s) /* return file descriptor */ #define sock_close_event(e) /* do nothing */ -#ifdef _OSE_ -#define inet_driver_select(port, e, mode, on) \ - ose_inet_select(port, e, mode, on) -#else #define inet_driver_select(port, e, mode, on) \ driver_select(port, e, mode | (on?ERL_DRV_USE:0), on) -#endif /* _OSE_ */ #define sock_select(d, flags, onoff) do { \ (d)->event_mask = (onoff) ? \ @@ -1085,7 +1045,7 @@ struct erl_drv_entry inet_driver_entry = }; /* XXX: is this a driver interface function ??? */ -extern void erl_exit(int n, char*, _DOTS_); +void erl_exit(int n, char*, ...); /* * Malloc wrapper, @@ -3480,13 +3440,9 @@ static int inet_init() INIT_ATOM(scheme); /* add TCP, UDP and SCTP drivers */ -#ifdef _OSE_ - add_ose_tcp_drv_entry(&tcp_inet_driver_entry); - add_ose_udp_drv_entry(&udp_inet_driver_entry); -#else add_driver_entry(&tcp_inet_driver_entry); add_driver_entry(&udp_inet_driver_entry); -# ifdef HAVE_SCTP +#ifdef HAVE_SCTP /* Check the size of SCTP AssocID -- currently both this driver and the Erlang part require 32 bit: */ ASSERT(sizeof(sctp_assoc_t)==ASSOC_ID_LEN); @@ -3501,8 +3457,8 @@ static int inet_init() add_driver_entry(&sctp_inet_driver_entry); } } -# endif -#endif /* _OSE_ */ +#endif + /* remove the dummy inet driver */ remove_driver_entry(&inet_driver_entry); return 0; @@ -3744,7 +3700,7 @@ static int inet_ctl_fdopen(inet_descriptor* desc, int domain, int type, unsigned int sz = sizeof(name); /* check that it is a socket and that the socket is bound */ - if (sock_name(s, (struct sockaddr*) &name, &sz) == SOCKET_ERROR) + if (IS_SOCKET_ERROR(sock_name(s, (struct sockaddr*) &name, &sz))) return ctl_error(sock_errno(), rbuf, rsize); desc->s = s; if ((desc->event = sock_create_event(desc)) == INVALID_EVENT) @@ -3756,7 +3712,7 @@ static int inet_ctl_fdopen(inet_descriptor* desc, int domain, int type, desc->state = INET_STATE_BOUND; /* assume bound */ if (type == SOCK_STREAM) { /* check if connected */ sz = sizeof(name); - if (sock_peer(s, (struct sockaddr*) &name, &sz) != SOCKET_ERROR) + if (!IS_SOCKET_ERROR(sock_peer(s, (struct sockaddr*) &name, &sz))) desc->state = INET_STATE_CONNECTED; } @@ -5687,8 +5643,8 @@ static int inet_fill_opts(inet_descriptor* desc, buf += arg_sz; len -= arg_sz; } - if (sock_getopt(desc->s,proto,type,arg_ptr,&arg_sz) == - SOCKET_ERROR) { + if (IS_SOCKET_ERROR(sock_getopt(desc->s,proto,type, + arg_ptr,&arg_sz))) { TRUNCATE_TO(0,ptr); continue; } @@ -5705,7 +5661,7 @@ static int inet_fill_opts(inet_descriptor* desc, RETURN_ERROR(); } /* We have 5 bytes allocated to ptr */ - if (sock_getopt(desc->s,proto,type,arg_ptr,&arg_sz) == SOCKET_ERROR) { + if (IS_SOCKET_ERROR(sock_getopt(desc->s,proto,type,arg_ptr,&arg_sz))) { TRUNCATE_TO(0,ptr); continue; } @@ -6771,7 +6727,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len, if (len != 0) return ctl_error(EINVAL, rbuf, rsize); - if (sock_hostname(tbuf, MAXHOSTNAMELEN) == SOCKET_ERROR) + if (IS_SOCKET_ERROR(sock_hostname(tbuf, MAXHOSTNAMELEN))) return ctl_error(sock_errno(), rbuf, rsize); return ctl_reply(INET_REP_OK, tbuf, strlen(tbuf), rbuf, rsize); } @@ -6788,7 +6744,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len, return ctl_error(ENOTCONN, rbuf, rsize); if ((ptr = desc->peer_ptr) == NULL) { ptr = &peer; - if (sock_peer(desc->s, (struct sockaddr*)ptr,&sz) == SOCKET_ERROR) + if (IS_SOCKET_ERROR(sock_peer(desc->s, (struct sockaddr*)ptr,&sz))) return ctl_error(sock_errno(), rbuf, rsize); } if (inet_get_address(desc->sfamily, tbuf, ptr, &sz) < 0) @@ -6825,7 +6781,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len, if ((ptr = desc->name_ptr) == NULL) { ptr = &name; - if (sock_name(desc->s, (struct sockaddr*)ptr, &sz) == SOCKET_ERROR) + if (IS_SOCKET_ERROR(sock_name(desc->s, (struct sockaddr*)ptr, &sz))) return ctl_error(sock_errno(), rbuf, rsize); } if (inet_get_address(desc->sfamily, tbuf, ptr, &sz) < 0) @@ -6864,7 +6820,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len, if (inet_set_address(desc->sfamily, &local, buf, &len) == NULL) return ctl_error(EINVAL, rbuf, rsize); - if (sock_bind(desc->s,(struct sockaddr*) &local, len) == SOCKET_ERROR) + if (IS_SOCKET_ERROR(sock_bind(desc->s,(struct sockaddr*) &local, len))) return ctl_error(sock_errno(), rbuf, rsize); desc->state = INET_STATE_BOUND; @@ -7297,7 +7253,7 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len, if (len != 2) return ctl_error(EINVAL, rbuf, rsize); backlog = get_int16(buf); - if (sock_listen(desc->inet.s, backlog) == SOCKET_ERROR) + if (IS_SOCKET_ERROR(sock_listen(desc->inet.s, backlog))) return ctl_error(sock_errno(), rbuf, rsize); desc->inet.state = TCP_STATE_LISTEN; return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize); @@ -7331,7 +7287,7 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len, code = sock_connect(desc->inet.s, (struct sockaddr*) &desc->inet.remote, len); - if ((code == SOCKET_ERROR) && + if (IS_SOCKET_ERROR(code) && ((sock_errno() == ERRNO_BLOCK) || /* Winsock2 */ (sock_errno() == EINPROGRESS))) { /* Unix & OSE!! */ sock_select(INETP(desc), FD_CONNECT, 1); @@ -7518,7 +7474,6 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len, tcp_deliver(desc, 0); return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize); } -#ifndef _OSE_ case TCP_REQ_SHUTDOWN: { int how; DEBUGF(("tcp_inet_ctl(%ld): FDOPEN\r\n", (long)desc->inet.port)); @@ -7535,7 +7490,6 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len, return ctl_error(sock_errno(), rbuf, rsize); } } -#endif default: DEBUGF(("tcp_inet_ctl(%ld): %u\r\n", (long)desc->inet.port, cmd)); return inet_ctl(INETP(desc), cmd, buf, len, rbuf, rsize); @@ -8009,7 +7963,7 @@ static int tcp_recv(tcp_descriptor* desc, int request_len) n = sock_recv(desc->inet.s, desc->i_ptr, nread, 0); - if (n == SOCKET_ERROR) { + if (IS_SOCKET_ERROR(n)) { int err = sock_errno(); if (err == ECONNRESET) { DEBUGF((" => detected close (connreset)\r\n")); @@ -8511,8 +8465,8 @@ static int tcp_sendv(tcp_descriptor* desc, ErlIOVec* ev) (long)desc->inet.port, desc->inet.s, h_len, len)); if (desc->tcp_add_flags & TCP_ADDF_DELAY_SEND) { n = 0; - } else if (sock_sendv(desc->inet.s, ev->iov, vsize, &n, 0) - == SOCKET_ERROR) { + } else if (IS_SOCKET_ERROR(sock_sendv(desc->inet.s, ev->iov, + vsize, &n, 0))) { if ((sock_errno() != ERRNO_BLOCK) && (sock_errno() != EINTR)) { int err = sock_errno(); DEBUGF(("tcp_sendv(%ld): s=%d, " @@ -8605,7 +8559,7 @@ static int tcp_send(tcp_descriptor* desc, char* ptr, int len) if (desc->tcp_add_flags & TCP_ADDF_DELAY_SEND) { sock_send(desc->inet.s, buf, 0, 0); n = 0; - } else if (sock_sendv(desc->inet.s,iov,2,&n,0) == SOCKET_ERROR) { + } else if (IS_SOCKET_ERROR(sock_sendv(desc->inet.s,iov,2,&n,0))) { if ((sock_errno() != ERRNO_BLOCK) && (sock_errno() != EINTR)) { int err = sock_errno(); DEBUGF(("tcp_send(%ld): s=%d,sock_sendv(size=2) errno = %d\r\n", @@ -8678,7 +8632,7 @@ static int tcp_inet_output(tcp_descriptor* desc, HANDLE event) int code = sock_peer(desc->inet.s, (struct sockaddr*) &desc->inet.remote, &sz); - if (code == SOCKET_ERROR) { + if (IS_SOCKET_ERROR(code)) { desc->inet.state = TCP_STATE_BOUND; /* restore state */ ret = async_error(INETP(desc), sock_errno()); goto done; @@ -8719,7 +8673,7 @@ static int tcp_inet_output(tcp_descriptor* desc, HANDLE event) vsize = vsize > MAX_VSIZE ? MAX_VSIZE : vsize; DEBUGF(("tcp_inet_output(%ld): s=%d, About to send %d items\r\n", (long)desc->inet.port, desc->inet.s, vsize)); - if (sock_sendv(desc->inet.s, iov, vsize, &n, 0)==SOCKET_ERROR) { + if (IS_SOCKET_ERROR(sock_sendv(desc->inet.s, iov, vsize, &n, 0))) { if ((sock_errno() != ERRNO_BLOCK) && (sock_errno() != EINTR)) { DEBUGF(("tcp_inet_output(%ld): sock_sendv(%d) errno = %d\r\n", (long)desc->inet.port, vsize, sock_errno())); @@ -8988,7 +8942,7 @@ static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len, sock_select(desc, FD_CONNECT, 1); code = sock_connect(desc->s, &remote.sa, len); - if ((code == SOCKET_ERROR) && (sock_errno() == EINPROGRESS)) { + if (IS_SOCKET_ERROR(code) && (sock_errno() == EINPROGRESS)) { /* XXX: Unix only -- WinSock would have a different cond! */ desc->state = SCTP_STATE_CONNECTING; if (timeout != INET_INFINITY) @@ -9028,7 +8982,7 @@ static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len, code = sock_connect(desc->s, (struct sockaddr*) &desc->remote, len); - if (code == SOCKET_ERROR) { + if (IS_SOCKET_ERROR(code)) { sock_connect(desc->s, (struct sockaddr*) NULL, 0); desc->state &= ~INET_F_ACTIVE; return ctl_error(sock_errno(), rbuf, rsize); @@ -9062,7 +9016,7 @@ static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len, return ctl_error(EINVAL, rbuf, rsize); flag = get_int8(buf); - if (sock_listen(desc->s, flag) == SOCKET_ERROR) + if (IS_SOCKET_ERROR(sock_listen(desc->s, flag))) return ctl_error(sock_errno(), rbuf, rsize); desc->state = SCTP_STATE_LISTEN; /* XXX: not used? */ @@ -9267,7 +9221,7 @@ static void packet_inet_command(ErlDrvData e, char* buf, int len) check_result_code: /* "code" analysis is the same for both SCTP and UDP cases above: */ #endif - if (code == SOCKET_ERROR) { + if (IS_SOCKET_ERROR(code)) { int err = sock_errno(); inet_reply_error(desc, err); } @@ -9366,7 +9320,7 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event) check_result: #endif /* Analyse the result: */ - if (n == SOCKET_ERROR + if (IS_SOCKET_ERROR(n) #ifdef HAVE_SCTP || (short_recv = (IS_SCTP(desc) && !(mhdr.msg_flags & MSG_EOR))) /* NB: here we check for EOR not being set -- this is an error as @@ -9379,11 +9333,13 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event) if (err != ERRNO_BLOCK) { if (!desc->active) { #ifdef HAVE_SCTP - if (short_recv) + if (short_recv) { async_error_am(desc, am_short_recv); - else -#else + } else { async_error(desc, err); + } +#else + async_error(desc, err); #endif driver_cancel_timer(desc->port); sock_select(desc,FD_READ,0); @@ -9481,7 +9437,7 @@ static int packet_inet_output(udp_descriptor* udesc, HANDLE event) int code = sock_peer(desc->s, (struct sockaddr*) &desc->remote, &sz); - if (code == SOCKET_ERROR) { + if (IS_SOCKET_ERROR(code)) { desc->state = PACKET_STATE_BOUND; /* restore state */ ret = async_error(desc, sock_errno()); goto done; @@ -9922,23 +9878,26 @@ int erts_sock_connect(erts_sock_t socket, byte *ip_addr, int len, Uint16 port) if (!inet_set_address(AF_INET, &addr, buf, &blen)) return 0; - if (SOCKET_ERROR == sock_connect(s, + if (IS_SOCKET_ERROR(sock_connect(s, (struct sockaddr *) &addr, - sizeof(struct sockaddr_in))) + sizeof(struct sockaddr_in)))) return 0; return 1; } Sint erts_sock_send(erts_sock_t socket, const void *buf, Sint len) { - return (Sint) sock_send((SOCKET) socket, buf, (size_t) len, 0); + Sint result = (Sint) sock_send((SOCKET) socket, buf, (size_t) len, 0); + if (IS_SOCKET_ERROR(result)) + return SOCKET_ERROR; + return result; } int erts_sock_gethostname(char *buf, int bufsz) { - if (sock_hostname(buf, bufsz) == SOCKET_ERROR) - return -1; + if (IS_SOCKET_ERROR(sock_hostname(buf, bufsz))) + return SOCKET_ERROR; return 0; } diff --git a/erts/emulator/drivers/common/ram_file_drv.c b/erts/emulator/drivers/common/ram_file_drv.c index 4a39a156e6..abedcc933a 100644 --- a/erts/emulator/drivers/common/ram_file_drv.c +++ b/erts/emulator/drivers/common/ram_file_drv.c @@ -35,6 +35,7 @@ #define RAM_FILE_TRUNCATE 14 #define RAM_FILE_PREAD 17 #define RAM_FILE_PWRITE 18 +#define RAM_FILE_FDATASYNC 19 /* other operations */ #define RAM_FILE_GET 30 @@ -45,6 +46,8 @@ #define RAM_FILE_UUENCODE 35 /* uuencode file */ #define RAM_FILE_UUDECODE 36 /* uudecode file */ #define RAM_FILE_SIZE 37 /* get file size */ +#define RAM_FILE_ADVISE 38 /* predeclare the access + * pattern for file data */ /* possible new operations include: DES_ENCRYPT DES_DECRYPT @@ -558,6 +561,13 @@ static void rfile_command(ErlDrvData e, char* buf, int count) numeric_reply(f, 0); /* 0 is not used */ break; + case RAM_FILE_FDATASYNC: + if (f->flags == 0) + error_reply(f, EBADF); + else + reply(f, 1, 0); + break; + case RAM_FILE_FSYNC: if (f->flags == 0) error_reply(f, EBADF); @@ -685,6 +695,13 @@ static void rfile_command(ErlDrvData e, char* buf, int count) case RAM_FILE_UUDECODE: /* uudecode file */ ram_file_uudecode(f); break; + + case RAM_FILE_ADVISE: + if (f->flags == 0) + error_reply(f, EBADF); + else + reply(f, 1, 0); + break; } /* * Ignore anything else -- let the caller hang. diff --git a/erts/emulator/drivers/common/zlib_drv.c b/erts/emulator/drivers/common/zlib_drv.c index 723efeaa13..f50899a730 100644 --- a/erts/emulator/drivers/common/zlib_drv.c +++ b/erts/emulator/drivers/common/zlib_drv.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2003-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2003-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -115,7 +115,7 @@ typedef struct { static int zlib_inflate(ZLibData* d, int flush); static int zlib_deflate(ZLibData* d, int flush); -#if defined(_OSE_) || defined(__WIN32__) +#if defined(__WIN32__) static int i32(char* buf) #else static inline int i32(char* buf) diff --git a/erts/emulator/drivers/unix/mem_drv.c b/erts/emulator/drivers/unix/mem_drv.c deleted file mode 100644 index 1417ca1121..0000000000 --- a/erts/emulator/drivers/unix/mem_drv.c +++ /dev/null @@ -1,145 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - -/* Purpose: Access to elib memory statistics */ - -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "sys.h" -#include "erl_driver.h" -#include "elib_stat.h" - -#define MAP_BUF_SIZE 1000 /* Max map size */ -#define HISTO_BUF_SIZE 100 /* Max histogram buckets */ - -static ErlDrvData mem_start(ErlDrvPort); -static int mem_init(void); -static void mem_stop(ErlDrvData); -static void mem_command(ErlDrvData, char*, int); - -const struct driver_entry mem_driver_entry = { - mem_init, - mem_start, - mem_stop, - mem_command, - NULL, - NULL, - "mem_drv" -}; - -static int mem_init(void) -{ - return 0; -} - -static ErlDrvData mem_start(ErlDrvPort port, char* buf) -{ - return (ErlDrvData)port; -} - -static void mem_stop(ErlDrvData port) -{ -} - -void putint32(p, v) -byte* p; int v; -{ - p[0] = (v >> 24) & 0xff; - p[1] = (v >> 16) & 0xff; - p[2] = (v >> 8) & 0xff; - p[3] = (v) & 0xff; -} - -int getint16(p) -byte* p; -{ - return (p[0] << 8) | p[1]; -} - -/* -** Command: -** m L1 L0 -> a heap map of length L1*256 + L0 is returned -** s -> X3 X2 X1 X0 Y3 Y2 Y1 Y0 Z3 Z2 Z1 Z0 -** X == Total heap size bytes -** Y == Total free bytes -** Z == Size of largest free block in bytes -** -** h L1 L0 B0 -> Generate a logarithm historgram base B with L buckets -** l L1 L0 S0 -> Generate a linear histogram with step S with L buckets -*/ -unsigned char outbuf[HISTO_BUF_SIZE*2*4]; - -static void mem_command(ErlDrvData port, char* buf, int count) -{ - if ((count == 1) && buf[0] == 's') { - struct elib_stat info; - char v[3*4]; - - elib_stat(&info); - - putint32(v, info.mem_total*4); - putint32(v+4, info.mem_free*4); - putint32(v+8, info.max_free*4); - driver_output((ErlDrvPort)port, v, 12); - return; - } - else if ((count == 3) && buf[0] == 'm') { - char w[MAP_BUF_SIZE]; - int n = getint16(buf+1); - - if (n > MAP_BUF_SIZE) - n = MAP_BUF_SIZE; - elib_heap_map(w, n); - driver_output((ErlDrvPort)port, w, n); - return; - } - else if ((count == 4) && (buf[0] == 'h' || buf[0] == 'l')) { - unsigned long vf[HISTO_BUF_SIZE]; - unsigned long va[HISTO_BUF_SIZE]; - int n = getint16(buf+1); - int base = (unsigned char) buf[3]; - - if (n >= HISTO_BUF_SIZE) - n = HISTO_BUF_SIZE; - if (buf[0] == 'l') - base = -base; - if (elib_histo(vf, va, n, base) < 0) { - driver_failure((ErlDrvPort)port, -1); - return; - } - else { - char* p = outbuf; - int i; - - for (i = 0; i < n; i++) { - putint32(p, vf[i]); - p += 4; - } - for (i = 0; i < n; i++) { - putint32(p, va[i]); - p += 4; - } - driver_output((ErlDrvPort)port, outbuf, n*8); - } - return; - } - driver_failure((ErlDrvPort)port, -1); -} diff --git a/erts/emulator/drivers/unix/ttsl_drv.c b/erts/emulator/drivers/unix/ttsl_drv.c index 4cd54c073f..d782b044a9 100644 --- a/erts/emulator/drivers/unix/ttsl_drv.c +++ b/erts/emulator/drivers/unix/ttsl_drv.c @@ -314,7 +314,7 @@ static ErlDrvData ttysl_start(ErlDrvPort port, char* buf) sys_sigset(SIGCONT, cont); sys_sigset(SIGWINCH, winch); - driver_select(port, (ErlDrvEvent)(Uint)ttysl_fd, ERL_DRV_READ|ERL_DRV_USE, 1); + driver_select(port, (ErlDrvEvent)(UWord)ttysl_fd, ERL_DRV_READ|ERL_DRV_USE, 1); ttysl_port = port; /* we need to know this when we enter the break handler */ @@ -394,7 +394,7 @@ static void ttysl_stop(ErlDrvData ttysl_data) stop_lbuf(); stop_termcap(); tty_reset(ttysl_fd); - driver_select(ttysl_port, (ErlDrvEvent)(Uint)ttysl_fd, ERL_DRV_READ|ERL_DRV_USE, 0); + driver_select(ttysl_port, (ErlDrvEvent)(UWord)ttysl_fd, ERL_DRV_READ|ERL_DRV_USE, 0); sys_sigset(SIGCONT, SIG_DFL); sys_sigset(SIGWINCH, SIG_DFL); } @@ -685,7 +685,7 @@ static void ttysl_from_tty(ErlDrvData ttysl_data, ErlDrvEvent fd) utf8buf_size = 0; } - if ((i = read((int)(Sint)fd, (char *) p, left)) >= 0) { + if ((i = read((int)(SWord)fd, (char *) p, left)) >= 0) { if (p != b) { i += (p - b); } diff --git a/erts/emulator/drivers/unix/unix_efile.c b/erts/emulator/drivers/unix/unix_efile.c index d395b68691..b19f632f52 100644 --- a/erts/emulator/drivers/unix/unix_efile.c +++ b/erts/emulator/drivers/unix/unix_efile.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1997-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ /* @@ -34,17 +34,6 @@ #include <sys/uio.h> #endif -#ifdef _OSE_ -#include "efs.h" -#include <sys/types.h> -#include <sys/stat.h> -#include <unistd.h> - -#ifdef _OSE_SFK_ -#include <string.h> -#endif -#endif /* _OSE_ */ - #if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__) #define DARWIN 1 #endif @@ -88,23 +77,6 @@ extern STATUS copy(char *, char *); * Macros for testing file types. */ -#ifdef _OSE_ - -#define ISDIR(st) S_ISDIR(((st).st_mode)) -#define ISREG(st) S_ISREG(((st).st_mode)) -#define ISDEV(st) (S_ISCHR(((st).st_mode)) || S_ISBLK(((st).st_mode))) -#define ISLNK(st) S_ISLNK(((st).st_mode)) -#ifdef NO_UMASK -#define FILE_MODE (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) -#define DIR_MODE (S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH) -#else -#define FILE_MODE (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH) -#define DIR_MODE (S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IWGRP | S_IXGRP | S_IROTH | \ - S_IWOTH | S_IXOTH) -#endif - -#else /* !_OSE_ */ - #define ISDIR(st) (((st).st_mode & S_IFMT) == S_IFDIR) #define ISREG(st) (((st).st_mode & S_IFMT) == S_IFREG) #define ISDEV(st) \ @@ -118,8 +90,6 @@ extern STATUS copy(char *, char *); #define DIR_MODE 0777 #endif -#endif /* _OSE_ */ - #ifdef VXWORKS /* Currently only used on vxworks */ #define EF_ALLOC(S) driver_alloc((S)) @@ -128,7 +98,7 @@ extern STATUS copy(char *, char *); #define EF_SAFE_REALLOC(P, S) ef_safe_realloc((P), (S)) #define EF_FREE(P) do { if((P)) driver_free((P)); } while(0) -extern void erl_exit(int n, char *fmt, _DOTS_); +void erl_exit(int n, char *fmt, ...); static void *ef_safe_alloc(Uint s) { @@ -157,7 +127,7 @@ static void *ef_safe_realloc(void *op, Uint s) (s[0] == '.' && (s[1] == '\0' || (s[1] == '.' && s[2] == '\0'))) #ifdef VXWORKS -static FUNCTION(int, vxworks_to_posix, (int vx_errno)); +static int vxworks_to_posix(int vx_errno); #endif /* @@ -176,7 +146,7 @@ static FUNCTION(int, vxworks_to_posix, (int vx_errno)); #define CHECK_PATHLEN(X,Y) /* Nothing */ #endif -static FUNCTION(int, check_error, (int result, Efile_error* errInfo)); +static int check_error(int result, Efile_error* errInfo); static int check_error(int result, Efile_error *errInfo) @@ -361,15 +331,6 @@ path_size(char *pathname) #endif /* VXWORKS */ -#ifdef _OSE_ -static int -ose_enotsup(Efile_error *errInfo) -{ - errInfo->posix_errno = errInfo->os_errno = ENOTSUP; - return 0; -} -#endif /* _OSE_ */ - int efile_mkdir(Efile_error* errInfo, /* Where to return error codes. */ char* name) /* Name of directory to create. */ @@ -446,18 +407,12 @@ efile_delete_file(Efile_error* errInfo, /* Where to return error codes. */ char* name) /* Name of file to delete. */ { CHECK_PATHLEN(name,errInfo); -#ifdef _OSE_ - if (remove(name) == 0) { - return 1; - } -#else if (unlink(name) == 0) { return 1; } if (errno == EISDIR) { /* Linux sets the wrong error code. */ errno = EPERM; } -#endif return check_error(-1, errInfo); } @@ -524,7 +479,7 @@ efile_rename(Efile_error* errInfo, /* Where to return error codes. */ if (errno == ENOTEMPTY) { errno = EEXIST; } -#if defined (sparc) && !defined(VXWORKS) && !defined(_OSE_) +#if defined (sparc) && !defined(VXWORKS) /* * SunOS 4.1.4 reports overwriting a non-empty directory with a * directory as EINVAL instead of EEXIST (first rule out the correct @@ -751,6 +706,9 @@ efile_openfile(Efile_error* errInfo, /* Where to return error codes. */ #endif } + if (flags & EFILE_MODE_EXCL) { + mode |= O_EXCL; + } #ifdef VXWORKS if (*name != '/') { @@ -819,6 +777,17 @@ efile_closefile(int fd) } int +efile_fdatasync(Efile_error *errInfo, /* Where to return error codes. */ + int fd) /* File descriptor for file to sync data. */ +{ +#ifdef HAVE_FDATASYNC + return check_error(fdatasync(fd), errInfo); +#else + return efile_fsync(errInfo, fd); +#endif +} + +int efile_fsync(Efile_error *errInfo, /* Where to return error codes. */ int fd) /* File descriptor for file to sync. */ { @@ -855,7 +824,7 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, CHECK_PATHLEN(name, errInfo); if (info_for_link) { -#if (defined(VXWORKS) || defined(_OSE_)) +#if (defined(VXWORKS)) result = stat(name, &statbuf); #else result = lstat(name, &statbuf); @@ -939,11 +908,7 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, pInfo->mode = statbuf.st_mode; pInfo->links = statbuf.st_nlink; pInfo->major_device = statbuf.st_dev; -#ifdef _OSE_ - pInfo->minor_device = 0; -#else pInfo->minor_device = statbuf.st_rdev; -#endif pInfo->inode = statbuf.st_ino; pInfo->uid = statbuf.st_uid; pInfo->gid = statbuf.st_gid; @@ -989,11 +954,9 @@ efile_write_info(Efile_error *errInfo, Efile_info *pInfo, char *name) * you don't try to chown a file to someone besides youself. */ -#ifndef _OSE_ if (chown(name, pInfo->uid, pInfo->gid) && errno != EPERM) { return check_error(-1, errInfo); } -#endif if (pInfo->mode != -1) { mode_t newMode = pInfo->mode & (S_ISUID | S_ISGID | @@ -1008,8 +971,6 @@ efile_write_info(Efile_error *errInfo, Efile_info *pInfo, char *name) #endif /* !VXWORKS */ -#ifndef _OSE_ - if (pInfo->accessTime.year != -1 && pInfo->modifyTime.year != -1) { struct utimbuf tval; struct tm timebuf; @@ -1041,7 +1002,6 @@ efile_write_info(Efile_error *errInfo, Efile_info *pInfo, char *name) return check_error(utime(name, &tval), errInfo); #endif } -#endif /* !_OSE_ */ return 1; } @@ -1451,9 +1411,6 @@ efile_truncate_file(Efile_error* errInfo, int *fd, int flags) int efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size) { -#ifdef _OSE_ - return ose_enotsup(errInfo); -#else #ifdef VXWORKS return vxworks_enotsup(errInfo); #else @@ -1466,7 +1423,6 @@ efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size) buffer[len] = '\0'; return 1; #endif -#endif } int @@ -1479,27 +1435,30 @@ efile_altname(Efile_error* errInfo, char* name, char* buffer, size_t size) int efile_link(Efile_error* errInfo, char* old, char* new) { -#ifdef _OSE_ - return ose_enotsup(errInfo); -#else #ifdef VXWORKS return vxworks_enotsup(errInfo); #else return check_error(link(old, new), errInfo); #endif -#endif } int efile_symlink(Efile_error* errInfo, char* old, char* new) { -#ifdef _OSE_ - return ose_enotsup(errInfo); -#else #ifdef VXWORKS return vxworks_enotsup(errInfo); #else return check_error(symlink(old, new), errInfo); #endif +} + +int +efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, + Sint64 length, int advise) +{ +#ifdef HAVE_POSIX_FADVISE + return check_error(posix_fadvise(fd, offset, length, advise), errInfo); +#else + return check_error(0, errInfo); #endif } diff --git a/erts/emulator/drivers/win32/mem_drv.c b/erts/emulator/drivers/win32/mem_drv.c deleted file mode 100644 index fa7c46eca8..0000000000 --- a/erts/emulator/drivers/win32/mem_drv.c +++ /dev/null @@ -1,141 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ -/* Purpose: Access to elib memory statistics */ - -#include "sys.h" -#include "erl_driver.h" -#include "elib_stat.h" - -#define MAP_BUF_SIZE 1000 /* Max map size */ -#define HISTO_BUF_SIZE 100 /* Max histogram buckets */ - -static ErlDrvData mem_start(ErlDrvPort, char*); -static int mem_init(void); -static void mem_stop(ErlDrvData); -static void mem_command(ErlDrvData); - -ErlDrvEntry mem_driver_entry = { - mem_init, - mem_start, - mem_stop, - mem_command, - NULL, - NULL, - "mem_drv" -}; - -static int mem_init(void) -{ - return 0; -} - -static ErlDrvData mem_start(ErlDrvPort port, char* buf) -{ - return (ErlDrvData)port; -} - -static void mem_stop(ErlDrvData port) -{ -} - -void putint32(p, v) -byte* p; int v; -{ - p[0] = (v >> 24) & 0xff; - p[1] = (v >> 16) & 0xff; - p[2] = (v >> 8) & 0xff; - p[3] = (v) & 0xff; -} - -int getint16(p) -byte* p; -{ - return (p[0] << 8) | p[1]; -} - -/* -** Command: -** m L1 L0 -> a heap map of length L1*256 + L0 is returned -** s -> X3 X2 X1 X0 Y3 Y2 Y1 Y0 Z3 Z2 Z1 Z0 -** X == Total heap size bytes -** Y == Total free bytes -** Z == Size of largest free block in bytes -** -** h L1 L0 B0 -> Generate a logarithm histogram base B with L buckets -** l L1 L0 S0 -> Generate a linear histogram with step S with L buckets -*/ -unsigned char outbuf[HISTO_BUF_SIZE*2*4]; - -static void mem_command(ErlDrvData port, char* buf, int count) -{ - if ((count == 1) && buf[0] == 's') { - struct elib_stat info; - char v[3*4]; - - elib_stat(&info); - - putint32(v, info.mem_total*4); - putint32(v+4, info.mem_free*4); - putint32(v+8, info.max_free*4); - driver_output((ErlDrvPort)port, v, 12); - return; - } - else if ((count == 3) && buf[0] == 'm') { - char w[MAP_BUF_SIZE]; - int n = getint16(buf+1); - - if (n > MAP_BUF_SIZE) - n = MAP_BUF_SIZE; - elib_heap_map(w, n); - driver_output((ErlDrvPort)port, w, n); - return; - } - else if ((count == 4) && (buf[0] == 'h' || buf[0] == 'l')) { - unsigned long vf[HISTO_BUF_SIZE]; - unsigned long va[HISTO_BUF_SIZE]; - int n = getint16(buf+1); - int base = (unsigned char) buf[3]; - - if (n >= HISTO_BUF_SIZE) - n = HISTO_BUF_SIZE; - if (buf[0] == 'l') - base = -base; - if (elib_histo(vf, va, n, base) < 0) { - driver_failure((ErlDrvPort)port, -1); - return; - } - else { - char* p = outbuf; - int i; - - for (i = 0; i < n; i++) { - putint32(p, vf[i]); - p += 4; - } - for (i = 0; i < n; i++) { - putint32(p, va[i]); - p += 4; - } - driver_output((ErlDrvPort)port, outbuf, n*8); - } - return; - } - driver_failure((ErlDrvPort)port, -1); -} - diff --git a/erts/emulator/drivers/win32/win_efile.c b/erts/emulator/drivers/win32/win_efile.c index 89aaad31da..04bd1139f5 100644 --- a/erts/emulator/drivers/win32/win_efile.c +++ b/erts/emulator/drivers/win32/win_efile.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1997-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ /* @@ -689,7 +689,11 @@ Sint64* pSize; /* Where to store the size of the file. */ if (flags & EFILE_MODE_APPEND) { crFlags = OPEN_ALWAYS; } - fd = CreateFile(name, access, FILE_SHARE_READ | FILE_SHARE_WRITE, + if (flags & EFILE_MODE_EXCL) { + crFlags = CREATE_NEW; + } + fd = CreateFile(name, access, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, crFlags, FILE_ATTRIBUTE_NORMAL, NULL); /* @@ -764,6 +768,15 @@ int fd; /* File descriptor for file to close. */ } int +efile_fdatasync(errInfo, fd) +Efile_error* errInfo; /* Where to return error codes. */ +int fd; /* File descriptor for file to sync. */ +{ + /* Not available in Windows, just call regular fsync */ + return efile_fsync(errInfo, fd); +} + +int efile_fsync(errInfo, fd) Efile_error* errInfo; /* Where to return error codes. */ int fd; /* File descriptor for file to sync. */ @@ -1424,3 +1437,12 @@ efile_symlink(Efile_error* errInfo, char* old, char* new) errno = ENOTSUP; return check_error(-1, errInfo); } + +int +efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, + Sint64 length, int advise) +{ + /* posix_fadvise is not available on Windows, do nothing */ + errno = ERROR_SUCCESS; + return check_error(0, errInfo); +} diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index b0abfd2310..2a877d8ace 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -440,9 +440,12 @@ BIF_RETTYPE hipe_bifs_alloc_data_2(BIF_ALIST_2) align != sizeof(long) && align != sizeof(double))) BIF_ERROR(BIF_P, BADARG); nrbytes = unsigned_val(BIF_ARG_2); + if (nrbytes == 0) + BIF_RET(make_small(0)); block = erts_alloc(ERTS_ALC_T_HIPE, nrbytes); if ((unsigned long)block & (align-1)) - fprintf(stderr, "Yikes! erts_alloc() returned misaligned address %p\r\n", block); + fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte aligned\r\n", + __FUNCTION__, (unsigned long)nrbytes, block, (unsigned long)align); BIF_RET(address_to_term(block, BIF_P)); } diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c index e57e293547..6c9e1d9ba7 100644 --- a/erts/emulator/hipe/hipe_gc.c +++ b/erts/emulator/hipe/hipe_gc.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2004-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ /* $Id$ @@ -86,7 +86,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) if (is_boxed(gval)) { Eterm *ptr = boxed_val(gval); Eterm val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *nsp_i = val; } else if (in_area(ptr, src, src_size) || @@ -96,7 +96,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) } else if (is_list(gval)) { Eterm *ptr = list_val(gval); Eterm val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { *nsp_i = ptr[1]; } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { @@ -193,7 +193,7 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) if (is_boxed(gval)) { Eterm *ptr = boxed_val(gval); Eterm val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *nsp_i = val; } else if (in_area(ptr, heap, mature_size)) { @@ -205,7 +205,7 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) } else if (is_list(gval)) { Eterm *ptr = list_val(gval); Eterm val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { *nsp_i = ptr[1]; } else if (in_area(ptr, heap, mature_size)) { MOVE_CONS(ptr, val, old_htop, nsp_i); diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c index a77aec7919..900dfc5a8a 100644 --- a/erts/emulator/hipe/hipe_mkliterals.c +++ b/erts/emulator/hipe/hipe_mkliterals.c @@ -261,7 +261,7 @@ static const struct literal { /* Field offsets in a process struct */ { "P_HP", offsetof(struct process, htop) }, { "P_HP_LIMIT", offsetof(struct process, stop) }, - { "P_OFF_HEAP_MSO", offsetof(struct process, off_heap.mso) }, + { "P_OFF_HEAP_FIRST", offsetof(struct process, off_heap.first) }, { "P_MBUF", offsetof(struct process, mbuf) }, { "P_ID", offsetof(struct process, id) }, { "P_FLAGS", offsetof(struct process, flags) }, @@ -456,7 +456,7 @@ static const struct rts_param { } rts_params[] = { { 1, "P_OFF_HEAP_FUNS", #if !defined(HYBRID) - 1, offsetof(struct process, off_heap.funs) + 1, offsetof(struct process, off_heap.first) #endif }, diff --git a/erts/emulator/obsolete/driver.h b/erts/emulator/obsolete/driver.h deleted file mode 100644 index 708fe68e1a..0000000000 --- a/erts/emulator/obsolete/driver.h +++ /dev/null @@ -1,263 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ -/* - * OLD, OBSOLETE include file for erlang driver writers. - * New drivers should use erl_driver.h instead. - */ - -#ifndef __DRIVER_H__ -#define __DRIVER_H__ - -#include <stdlib.h> -#include "driver_int.h" - -#undef _ANSI_ARGS_ -#undef CONST - -#if ((defined(__STDC__) || defined(SABER)) && !defined(NO_PROTOTYPE)) || defined(__cplusplus) || defined(USE_PROTOTYPE) -# define _USING_PROTOTYPES_ 1 -# define _ANSI_ARGS_(x) x -# define CONST const -#else -# define _ANSI_ARGS_(x) () -# define CONST -#endif - -#ifdef __cplusplus -# define EXTERN extern "C" -#else -# define EXTERN extern -#endif - -/* Values for mode arg to driver_select() */ - -#define DO_READ (1 << 0) -#define DO_WRITE (1 << 1) - -/* Flags for set_port_control_flags() */ -#define PORT_CONTROL_FLAG_BINARY 1 -#define PORT_CONTROL_FLAG_HEAVY 2 - -/* This macro is used to name a dynamic driver's init function in */ -/* a way that doesn't lead to conflicts. This is crucial when using */ -/* operating systems that has one namespace for all symbols */ -/* (e.g. VxWorks). Example: if you have an dynamic driver C source */ -/* file named echo_drv.c, you use the macro like this: */ -/* int DRIVER_INIT(echo_drv)(void *handle) */ -#if defined(VXWORKS) -# define DRIVER_INIT(DRIVER_NAME) DRIVER_NAME ## _init -#elif defined(__WIN32__) -# define DRIVER_INIT(DRIVER_NAME) __declspec(dllexport) driver_init -#else -# define DRIVER_INIT(DRIVER_NAME) driver_init -#endif - -typedef int (*F_PTR)(); /* a function pointer */ -typedef long (*L_PTR)(); /* pointer to a function returning long */ - -extern int null_func(); - -/* This structure MUST match Binary in global.h exactly!!! */ -typedef struct driver_binary { - int orig_size; /* total length of binary */ - char orig_bytes[1]; /* the data (char instead of byte!) */ -} DriverBinary; - -typedef struct { - int vsize; /* length of vectors */ - int size; /* total size in bytes */ - SysIOVec* iov; - DriverBinary** binv; -} ErlIOVec; - -/* - * OLD, OBSOLETE driver entry structure. - */ - -typedef struct driver_entry { - F_PTR init; /* called at system start up (no args) */ - L_PTR start; /* called when some one does an open_port - args: port, command (nul-terminated), - additional/alternate args for fd/vanilla/spawn driver. - return value -1 means failure, other - is saved and passed to the other funcs */ - F_PTR stop; /* called when port is closed, and when the - emulator is halted - arg: start_return */ - F_PTR output; /* called when we have output from erlang to the port - args: start_return, buf, buflen */ - F_PTR ready_input; /* called when we have input from one of the driver's - file descriptors - args: start_return, fd */ - F_PTR ready_output; /* called when output is possible to one of the driver's - file descriptors - args: start_return, fd */ - char *driver_name; /* name supplied as {driver,Name,Args} to open_port */ - - F_PTR finish; /* called before unloading (DYNAMIC DRIVERS ONLY) */ - void *handle; /* file handle (DYNAMIC DRIVERS ONLY) */ - F_PTR control; /* "ioctl" for drivers (invoked by port_command/3) */ - F_PTR timeout; /* Reserved */ - F_PTR outputv; /* Reserved */ - F_PTR ready_async; /* Completion routine for driver_async */ - F_PTR padding1[3]; /* pad to match size of modern driver struct */ - int padding2[4]; /* more pad */ - F_PTR padding3[3]; /* even more padding */ -} DriverEntry; - - -/* These are the kernel functions available for driver writers */ - -EXTERN int driver_select _ANSI_ARGS_((int,int,int,int)); - -EXTERN int driver_output _ANSI_ARGS_((int, char*, int)); -EXTERN int driver_output2 _ANSI_ARGS_((int, char*, int, char*, int)); -EXTERN int driver_output_binary _ANSI_ARGS_((int, char*, int, - DriverBinary*, int, int)); -EXTERN int driver_outputv _ANSI_ARGS_((int, char*,int,ErlIOVec*,int)); - -EXTERN int driver_vec_to_buf _ANSI_ARGS_((ErlIOVec*, char*, int)); - -EXTERN int driver_set_timer _ANSI_ARGS_((int, unsigned long)); -EXTERN int driver_cancel_timer _ANSI_ARGS_((int)); - -/* - * The following functions are used to initiate a close of a port - * from a driver. - */ -EXTERN int driver_failure_eof _ANSI_ARGS_((int)); -EXTERN int driver_failure_atom _ANSI_ARGS_((int, char *)); -EXTERN int driver_failure_posix _ANSI_ARGS_((int, int)); -EXTERN int driver_failure _ANSI_ARGS_((int, int)); -EXTERN int driver_exit _ANSI_ARGS_ ((int, int)); - -EXTERN char* erl_errno_id _ANSI_ARGS_((int error)); -EXTERN void set_busy_port _ANSI_ARGS_((int, int)); -EXTERN void add_driver_entry _ANSI_ARGS_((DriverEntry *)); -EXTERN int remove_driver_entry _ANSI_ARGS_((DriverEntry *)); -EXTERN void set_port_control_flags _ANSI_ARGS_((int, int)); - -/* Binary interface */ -/* NOTE: DO NOT overwrite a binary with new data (if the data is delivered); -** since the binary is a shared object it MUST be written once. -*/ - -EXTERN DriverBinary* driver_alloc_binary _ANSI_ARGS_((int)); -EXTERN DriverBinary* driver_realloc_binary _ANSI_ARGS_((DriverBinary*, int)); -EXTERN void driver_free_binary _ANSI_ARGS_((DriverBinary*)); - - -/* Queue interface */ -EXTERN int driver_enqv _ANSI_ARGS_((int, ErlIOVec*, int)); -EXTERN int driver_pushqv _ANSI_ARGS_((int, ErlIOVec*, int)); -EXTERN int driver_deq _ANSI_ARGS_((int, int)); -EXTERN SysIOVec* driver_peekq _ANSI_ARGS_((int, int*)); -EXTERN int driver_sizeq _ANSI_ARGS_((int)); -EXTERN int driver_enq_bin _ANSI_ARGS_((int, DriverBinary*, int, int)); -EXTERN int driver_enq _ANSI_ARGS_((int, char*, int)); -EXTERN int driver_pushq_bin _ANSI_ARGS_((int, DriverBinary*, int, int)); -EXTERN int driver_pushq _ANSI_ARGS_((int, char*, int)); - -/* Memory management */ -EXTERN void *driver_alloc _ANSI_ARGS_((size_t)); -EXTERN void *driver_realloc _ANSI_ARGS_((void*, size_t)); -EXTERN void driver_free _ANSI_ARGS_((void*)); - -/* Shared / dynamic link libraries */ -EXTERN void *driver_dl_open _ANSI_ARGS_((char *)); -EXTERN void *driver_dl_sym _ANSI_ARGS_((void *, char *)); -EXTERN int driver_dl_close _ANSI_ARGS_((void *)); -EXTERN char *driver_dl_error _ANSI_ARGS_((void)); - -/* Async IO functions */ -EXTERN long driver_async _ANSI_ARGS_((int, - unsigned int*, - void (*)(void*), - void *, - void (*)(void*))); -EXTERN int driver_async_cancel _ANSI_ARGS_((long)); - -EXTERN int driver_lock_driver _ANSI_ARGS_((int)); - -/* Threads */ -typedef void* erl_mutex_t; -typedef void* erl_cond_t; -typedef void* erl_thread_t; - -EXTERN erl_mutex_t erts_mutex_create _ANSI_ARGS_((void)); -EXTERN int erts_mutex_destroy _ANSI_ARGS_((erl_mutex_t)); -EXTERN int erts_mutex_lock _ANSI_ARGS_((erl_mutex_t)); -EXTERN int erts_mutex_unlock _ANSI_ARGS_((erl_mutex_t)); - -EXTERN erl_cond_t erts_cond_create _ANSI_ARGS_((void)); -EXTERN int erts_cond_destroy _ANSI_ARGS_((erl_cond_t)); -EXTERN int erts_cond_signal _ANSI_ARGS_((erl_cond_t)); -EXTERN int erts_cond_broadcast _ANSI_ARGS_((erl_cond_t)); -EXTERN int erts_cond_wait _ANSI_ARGS_((erl_cond_t, erl_mutex_t)); -EXTERN int erts_cond_timedwait _ANSI_ARGS_((erl_cond_t, erl_mutex_t, long)); - -EXTERN int erts_thread_create _ANSI_ARGS_((erl_thread_t*, - void* (*func)(void*), - void* arg, - int detached)); -EXTERN erl_thread_t erts_thread_self _ANSI_ARGS_((void)); -EXTERN void erts_thread_exit _ANSI_ARGS_((void*)); -EXTERN int erts_thread_join _ANSI_ARGS_((erl_thread_t, void**)); -EXTERN int erts_thread_kill _ANSI_ARGS_((erl_thread_t)); - - -typedef unsigned long DriverTermData; - -#define TERM_DATA(x) ((DriverTermData) (x)) - -/* Possible types to send from driver Argument type */ -#define ERL_DRV_NIL ((DriverTermData) 1) /* None */ -#define ERL_DRV_ATOM ((DriverTermData) 2) /* driver_mk_atom(string) */ -#define ERL_DRV_INT ((DriverTermData) 3) /* int */ -#define ERL_DRV_PORT ((DriverTermData) 4) /* driver_mk_port(ix) */ -#define ERL_DRV_BINARY ((DriverTermData) 5) /* ErlDriverBinary*, int */ -#define ERL_DRV_STRING ((DriverTermData) 6) /* char*, int */ -#define ERL_DRV_TUPLE ((DriverTermData) 7) /* int */ -#define ERL_DRV_LIST ((DriverTermData) 8) /* int */ -#define ERL_DRV_STRING_CONS ((DriverTermData) 9) /* char*, int */ -#define ERL_DRV_PID ((DriverTermData) 10) /* driver_connected,... */ - -/* DriverTermData is the type to use for casts when building - * terms that should be sent to connected process, - * for instance a tuple on the form {tcp, Port, [Tag|Binary]} - * - * DriverTermData spec[] = { - * ERL_DRV_ATOM, driver_mk_atom("tcp"), - * ERL_DRV_PORT, driver_mk_port(drv->ix), - * ERL_DRV_INT, REPLY_TAG, - * ERL_DRV_BIN, 50, TERM_DATA(buffer), - * ERL_DRV_LIST, 2, - * ERL_DRV_TUPLE, 3, - * } - * - */ - -EXTERN DriverTermData driver_mk_atom _ANSI_ARGS_ ((char*)); -EXTERN DriverTermData driver_mk_port _ANSI_ARGS_ ((int)); -EXTERN DriverTermData driver_connected _ANSI_ARGS_((int)); -EXTERN DriverTermData driver_caller _ANSI_ARGS_((int)); - -EXTERN int driver_output_term _ANSI_ARGS_((int, DriverTermData *, int)); -EXTERN int driver_send_term _ANSI_ARGS_((int, DriverTermData, DriverTermData *, int)); - -#endif - - diff --git a/erts/emulator/pcre/pcre_compile.c b/erts/emulator/pcre/pcre_compile.c index 29743362d4..9508c5a697 100644 --- a/erts/emulator/pcre/pcre_compile.c +++ b/erts/emulator/pcre/pcre_compile.c @@ -92,6 +92,11 @@ is 4 there is plenty of room. */ #define COMPILE_WORK_SIZE (4096) +/* The overrun tests check for a slightly smaller size so that they detect the +overrun before it actually does run off the end of the data block. */ + +#define WORK_SIZE_CHECK (COMPILE_WORK_SIZE - 100) + /* Table for handling escaped characters in the range '0'-'z'. Positive returns are simple data values; negative values are for special things like \d and so @@ -2445,7 +2450,7 @@ for (;; ptr++) #ifdef DEBUG if (code > cd->hwm) cd->hwm = code; /* High water info */ #endif - if (code > cd->start_workspace + COMPILE_WORK_SIZE) /* Check for overrun */ + if (code > cd->start_workspace + WORK_SIZE_CHECK) /* Check for overrun */ { *errorcodeptr = ERR52; goto FAILED; @@ -2494,7 +2499,7 @@ for (;; ptr++) /* In the real compile phase, just check the workspace used by the forward reference list. */ - else if (cd->hwm > cd->start_workspace + COMPILE_WORK_SIZE) + else if (cd->hwm > cd->start_workspace + WORK_SIZE_CHECK) { *errorcodeptr = ERR52; goto FAILED; diff --git a/erts/emulator/pcre/pcre_exec.c b/erts/emulator/pcre/pcre_exec.c index 51625130c3..3fe13ca32e 100644 --- a/erts/emulator/pcre/pcre_exec.c +++ b/erts/emulator/pcre/pcre_exec.c @@ -5191,7 +5191,6 @@ for(;;) EDEBUGF(("Loop limit break detected")); return PCRE_ERROR_LOOP_LIMIT; RESTART_INTERRUPTED: - md->match_call_count = 0; md->loop_limit = extra_data->loop_limit; rc = match(NULL,NULL,NULL,0,md,0,NULL,0,0); *extra_data->loop_counter_return = diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index 5dfd66bd7c..1c4c37b01a 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -83,10 +83,20 @@ static int is_cache_check_scheduled; static int is_cache_check_requested; #endif +#if HALFWORD_HEAP +static int initialize_pmmap(void); +static void *pmmap(size_t size); +static int pmunmap(void *p, size_t size); +static void *pmremap(void *old_address, size_t old_size, + size_t new_size); +#endif + #if HAVE_MMAP /* Mmap ... */ #define MMAP_PROT (PROT_READ|PROT_WRITE) + + #ifdef MAP_ANON # define MMAP_FLAGS (MAP_ANON|MAP_PRIVATE) # define MMAP_FD (-1) @@ -102,7 +112,11 @@ static int mmap_fd; # define HAVE_MSEG_RECREATE 0 #endif +#if HALFWORD_HEAP +#define CAN_PARTLY_DESTROY 0 +#else #define CAN_PARTLY_DESTROY 1 +#endif #else /* #if HAVE_MMAP */ #define CAN_PARTLY_DESTROY 0 #error "Not supported" @@ -232,6 +246,7 @@ static void thread_safe_init(void) { erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms"); erts_mtx_init(&mseg_mutex, "mseg"); + #ifdef ERTS_THREADS_NO_SMP main_tid = erts_thr_self(); #endif @@ -302,10 +317,20 @@ mseg_create(Uint size) #if defined(ERTS_MSEG_FAKE_SEGMENTS) seg = erts_sys_alloc(ERTS_ALC_N_INVALID, NULL, size); #elif HAVE_MMAP +#if HALFWORD_HEAP + seg = pmmap(size); +#else seg = (void *) mmap((void *) 0, (size_t) size, MMAP_PROT, MMAP_FLAGS, MMAP_FD, 0); if (seg == (void *) MAP_FAILED) seg = NULL; +#endif +#if HALFWORD_HEAP + if ((unsigned long) seg & CHECK_POINTER_MASK) { + erts_fprintf(stderr,"Pointer mask failure (0x%08lx)\n",(unsigned long) seg); + return NULL; + } +#endif #else #error "Missing mseg_create() implementation" #endif @@ -325,9 +350,11 @@ mseg_destroy(void *seg, Uint size) #ifdef DEBUG int res = #endif - +#if HALFWORD_HEAP + pmunmap((void *) seg, size); +#else munmap((void *) seg, size); - +#endif ASSERT(size % page_size == 0); ASSERT(res == 0); #else @@ -351,12 +378,18 @@ mseg_recreate(void *old_seg, Uint old_size, Uint new_size) #if defined(ERTS_MSEG_FAKE_SEGMENTS) new_seg = erts_sys_realloc(ERTS_ALC_N_INVALID, NULL, old_seg, new_size); #elif HAVE_MREMAP +#if HALFWORD_HEAP + new_seg = (void *) pmremap((void *) old_seg, + (size_t) old_size, + (size_t) new_size); +#else new_seg = (void *) mremap((void *) old_seg, (size_t) old_size, (size_t) new_size, MREMAP_MAYMOVE); if (new_seg == (void *) MAP_FAILED) new_seg = NULL; +#endif #else #error "Missing mseg_recreate() implementation" #endif @@ -1324,6 +1357,10 @@ erts_mseg_init(ErtsMsegInit_t *init) erl_exit(ERTS_ABORT_EXIT, "erts_mseg: unable to open /dev/zero\n"); #endif +#if HAVE_MMAP && HALFWORD_HEAP + initialize_pmmap(); +#endif + page_size = GET_PAGE_SIZE; page_shift = 1; @@ -1446,3 +1483,419 @@ erts_mseg_test(unsigned long op, } +#if HALFWORD_HEAP +/* + * Very simple page oriented mmap replacer. Works in the lower + * 32 bit address range of a 64bit program. + * Implements anonymous mmap mremap and munmap with address order first fit. + * The free list is expected to be very short... + * To be used for compressed pointers in Erlang halfword emulator + * implementation. The MacOS X version is more of a toy, it's not really + * for production as the halfword erlang VM relies on Linux specific memory + * mapping tricks. + */ + +/*#define HARDDEBUG 1*/ + +#ifdef __APPLE__ +#define MAP_ANONYMOUS MAP_ANON +#endif + +#define INIT_LOCK() do {erts_mtx_init(&pmmap_mutex, "pmmap");} while(0) + +#define TAKE_LOCK() do {erts_mtx_lock(&pmmap_mutex);} while(0) + +#define RELEASE_LOCK() do {erts_mtx_unlock(&pmmap_mutex);} while(0) + +static erts_mtx_t pmmap_mutex; /* Also needed when !USE_THREADS */ + +typedef struct _free_block { + unsigned long num; /*pages*/ + struct _free_block *next; +} FreeBlock; + +/* Assigned once and for all */ +static size_t pagsz; + +/* Protect with lock */ +static FreeBlock *first; + +static size_t round_up_to_pagesize(size_t size) +{ + size_t x = size / pagsz; + + if ((size % pagsz)) { + ++x; + } + + return pagsz * x; +} + +static size_t round_down_to_pagesize(size_t size) +{ + size_t x = size / pagsz; + + return pagsz * x; +} + +static void *do_map(void *ptr, size_t sz) +{ + void *res; + + if (round_up_to_pagesize(sz) != sz) { +#ifdef HARDDEBUG + fprintf(stderr,"Mapping of address %p with size %ld " + "does not map complete pages\r\n", + (void *) ptr, (unsigned long) sz); +#endif + return NULL; + } + + if (((unsigned long) ptr) % pagsz) { +#ifdef HARDDEBUG + fprintf(stderr,"Mapping of address %p with size %ld " + "is not page aligned\r\n", + (void *) ptr, (unsigned long) sz); +#endif + return NULL; + } + + + res = mmap(ptr, sz, + PROT_READ | PROT_WRITE, MAP_PRIVATE | + MAP_ANONYMOUS | MAP_FIXED, + -1 , 0); + + if (res == MAP_FAILED) { +#ifdef HARDDEBUG + fprintf(stderr,"Mapping of address %p with size %ld failed!\r\n", + (void *) ptr, (unsigned long) sz); +#endif + return NULL; + } + + return res; +} + +static int do_unmap(void *ptr, size_t sz) +{ + void *res; + + if (round_up_to_pagesize(sz) != sz) { +#ifdef HARDDEBUG + fprintf(stderr,"Mapping of address %p with size %ld " + "does not map complete pages\r\n", + (void *) ptr, (unsigned long) sz); +#endif + return 1; + } + + if (((unsigned long) ptr) % pagsz) { +#ifdef HARDDEBUG + fprintf(stderr,"Mapping of address %p with size %ld " + "is not page aligned\r\n", + (void *) ptr, (unsigned long) sz); +#endif + return 1; + } + + + res = mmap(ptr, sz, + PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE + | MAP_FIXED, + -1 , 0); + + if (res == MAP_FAILED) { +#ifdef HARDDEBUG + fprintf(stderr,"Mapping of address %p with size %ld failed!\r\n", + (void *) ptr, (unsigned long) sz); +#endif + return 1; + } + + return 0; +} + +#ifdef __APPLE__ +/* + * The first 4 gig's are protected on Macos X for 64bit processes :( + * The range 0x1000000000 - 0x10FFFFFFFF is selected as an arbitrary + * value of a normally unused range... Real MMAP's will avoid + * it and all 32bit compressed pointers can be in that range... + * More expensive than on Linux where expansion of compressed + * poiters involves no masking (as they are in the first 4 gig's). + * It's also very uncertain if the MAP_NORESERVE flag really has + * any effect in MacOS X. Swap space may always be allocated... + */ +#define SET_RANGE_MIN() /* nothing */ +#define RANGE_MIN 0x1000000000UL +#define RANGE_MAX 0x1100000000UL +#define RANGE_MASK (RANGE_MIN) +#define EXTRA_MAP_FLAGS (MAP_FIXED) +#else +static size_t range_min; +#define SET_RANGE_MIN() do { range_min = (size_t) sbrk(0); } while (0) +#define RANGE_MIN range_min +#define RANGE_MAX 0x100000000UL +#define RANGE_MASK 0UL +#define EXTRA_MAP_FLAGS (0) +#endif + +static int initialize_pmmap(void) +{ + char *p,*q,*rptr; + size_t rsz; + FreeBlock *initial; + + + pagsz = getpagesize(); + SET_RANGE_MIN(); + if (sizeof(void *) != 8) { + erl_exit(1,"Halfword emulator cannot be run in 32bit mode"); + } + + p = (char *) RANGE_MIN; + q = (char *) RANGE_MAX; + + rsz = round_down_to_pagesize(q - p); + + rptr = mmap((void *) p, rsz, + PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | + MAP_NORESERVE | EXTRA_MAP_FLAGS, + -1 , 0); +#ifdef HARDDEBUG + printf("rsz = %ld, pages = %ld, rptr = %p\r\n", + (unsigned long) rsz, (unsigned long) (rsz / pagsz), + (void *) rptr); +#endif + if (!do_map(rptr,pagsz)) { + erl_exit(1,"Could not actually mmap first page for halfword emulator...\n"); + } + initial = (FreeBlock *) rptr; + initial->num = (rsz / pagsz); + initial->next = NULL; + first = initial; + INIT_LOCK(); + return 0; +} + +#ifdef HARDDEBUG +static void dump_freelist(void) +{ + FreeBlock *p = first; + + while (p) { + printf("p = %p\r\np->num = %ld\r\np->next = %p\r\n\r\n", + (void *) p, (unsigned long) p->num, (void *) p->next); + p = p->next; + } +} +#endif + + +static void *pmmap(size_t size) +{ + size_t real_size = round_up_to_pagesize(size); + size_t num_pages = real_size / pagsz; + FreeBlock **block; + FreeBlock *tail; + FreeBlock *res; + TAKE_LOCK(); + for (block = &first; + *block != NULL && (*block)->num < num_pages; + block = &((*block)->next)) + ; + if (!(*block)) { + RELEASE_LOCK(); + return NULL; + } + if ((*block)->num == num_pages) { + /* nice, perfect fit */ + res = *block; + *block = (*block)->next; + } else { + tail = (FreeBlock *) (((char *) ((void *) (*block))) + real_size); + if (!do_map(tail,pagsz)) { +#ifdef HARDDEBUG + fprintf(stderr, "Could not actually allocate page at %p...\r\n", + (void *) tail); +#endif + RELEASE_LOCK(); + return NULL; + } + tail->num = (*block)->num - num_pages; + tail->next = (*block)->next; + res = *block; + *block = tail; + } + RELEASE_LOCK(); + if (!do_map(res,real_size)) { +#ifdef HARDDEBUG + fprintf(stderr, "Could not actually allocate %ld at %p...\r\n", + (unsigned long) real_size, (void *) res); +#endif + return NULL; + } + + return (void *) res; +} + +static int pmunmap(void *p, size_t size) +{ + size_t real_size = round_up_to_pagesize(size); + size_t num_pages = real_size / pagsz; + FreeBlock *block; + FreeBlock *last; + FreeBlock *nb = (FreeBlock *) p; + + if (real_size > pagsz) { + if (do_unmap(((char *) p) + pagsz,real_size - pagsz)) { + return 1; + } + } + + TAKE_LOCK(); + + last = NULL; + block = first; + while(block != NULL && ((void *) block) < p) { + last = block; + block = block->next; + } + + if (block != NULL && + ((void *) block) == ((void *) (((char *) p) + real_size))) { + /* Merge new free block with following */ + nb->num = block->num + num_pages; + nb->next = block->next; + if (do_unmap(block,pagsz)) { + RELEASE_LOCK(); + return 1; + } + } else { + /* just link in */ + nb->num = num_pages; + nb->next = block; + } + if (last != NULL) { + if (p == ((void *) (((char *) last) + (last->num * pagsz)))) { + /* Merge with previous */ + last->num += nb->num; + last->next = nb->next; + if (do_unmap(nb,pagsz)) { + RELEASE_LOCK(); + return 1; + } + } else { + last->next = nb; + } + } else { + first = nb; + } + RELEASE_LOCK(); + return 0; +} + +static void *pmremap(void *old_address, size_t old_size, + size_t new_size) +{ + size_t new_real_size = round_up_to_pagesize(new_size); + size_t new_num_pages = new_real_size / pagsz; + size_t old_real_size = round_up_to_pagesize(old_size); + size_t old_num_pages = old_real_size / pagsz; + if (new_num_pages == old_num_pages) { + return old_address; + } else if (new_num_pages < old_num_pages) { /* Shrink */ + size_t nfb_pages = old_num_pages - new_num_pages; + size_t nfb_real_size = old_real_size - new_real_size; + void *vnfb = (void *) (((char *)old_address) + new_real_size); + FreeBlock *nfb = (FreeBlock *) vnfb; + FreeBlock **block; + TAKE_LOCK(); + for (block = &first; + *block != NULL && (*block) < nfb; + block = &((*block)->next)) + ; + if (!(*block) || + (*block) > ((FreeBlock *)(((char *) vnfb) + nfb_real_size))) { + /* Normal link in */ + if (nfb_pages > 1) { + if (do_unmap((void *)(((char *) vnfb) + pagsz), + (nfb_pages - 1)*pagsz)) { + return NULL; + } + } + nfb->next = (*block); + nfb->num = nfb_pages; + (*block) = nfb; + } else { /* block merge */ + nfb->next = (*block)->next; + nfb->num = nfb_pages + (*block)->num; + /* unmap also the first page of the next freeblock */ + (*block) = nfb; + if (do_unmap((void *)(((char *) vnfb) + pagsz), + nfb_pages*pagsz)) { + return NULL; + } + } + RELEASE_LOCK(); + return old_address; + } else { /* Enlarge */ + FreeBlock **block; + void *old_end = (void *) (((char *)old_address) + old_real_size); + TAKE_LOCK(); + for (block = &first; + *block != NULL && (*block) < (FreeBlock *) old_address; + block = &((*block)->next)) + ; + if ((*block) == NULL || old_end > ((void *) RANGE_MAX) || + (*block) != old_end || + (*block)->num < (new_num_pages - old_num_pages)) { + /* cannot extend */ + void *result; + RELEASE_LOCK(); + result = pmmap(new_size); + if (result == NULL) { + return NULL; + } + memcpy(result,old_address,old_size); + if (pmunmap(old_address,old_size)) { + /* Oups... */ + pmunmap(result,new_size); + return NULL; + } + return result; + } else { /* extend */ + size_t remaining_pages = (*block)->num - + (new_num_pages - old_num_pages); + if (!remaining_pages) { + void *p = (void *) (((char *) (*block)) + pagsz); + void *n = (*block)->next; + size_t x = ((*block)->num - 1) * pagsz; + if (x > 0) { + if (do_map(p,x) == NULL) { + RELEASE_LOCK(); + return NULL; + } + } + (*block) = n; + } else { + FreeBlock *nfb = (FreeBlock *) ((void *) + (((char *) old_address) + + new_real_size)); + void *p = (void *) (((char *) (*block)) + pagsz); + if (do_map(p,new_real_size - old_real_size) == NULL) { + RELEASE_LOCK(); + return NULL; + } + nfb->num = remaining_pages; + nfb->next = (*block)->next; + (*block) = nfb; + } + RELEASE_LOCK(); + return old_address; + } + } +} + +#endif /* HALFWORD_HEAP */ diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h index 1c5aa63e90..d8053eb0d9 100644 --- a/erts/emulator/sys/common/erl_mseg.h +++ b/erts/emulator/sys/common/erl_mseg.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2002-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2002-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -58,8 +58,8 @@ typedef struct { typedef struct { int cache; int preserv; - Uint abs_shrink_th; - Uint rel_shrink_th; + UWord abs_shrink_th; + UWord rel_shrink_th; } ErtsMsegOpt_t; #define ERTS_MSEG_DEFAULT_OPT_INITIALIZER \ diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c index 5cca33d7eb..c17806d96c 100644 --- a/erts/emulator/sys/common/erl_poll.c +++ b/erts/emulator/sys/common/erl_poll.c @@ -130,13 +130,18 @@ #define ERTS_POLLSET_IS_POLLED(PS) \ ((int) erts_smp_atomic_read(&(PS)->polled)) -#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) \ - ((int) erts_smp_atomic_xchg(&(PS)->woken, (long) 1)) -#define ERTS_POLLSET_SET_POLLER_WOKEN(PS) \ - erts_smp_atomic_set(&(PS)->woken, (long) 1) -#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS) \ - erts_smp_atomic_set(&(PS)->woken, (long) 0) -#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) \ +#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) set_poller_woken_chk((PS)) +#define ERTS_POLLSET_SET_POLLER_WOKEN(PS) \ +do { \ + ERTS_THR_MEMORY_BARRIER; \ + erts_smp_atomic_set(&(PS)->woken, (long) 1); \ +} while (0) +#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS) \ +do { \ + erts_smp_atomic_set(&(PS)->woken, (long) 0); \ + ERTS_THR_MEMORY_BARRIER; \ +} while (0) +#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) \ ((int) erts_smp_atomic_read(&(PS)->woken)) #else @@ -194,13 +199,18 @@ #else -#define ERTS_POLLSET_UNSET_INTERRUPTED_CHK(PS) \ - ((int) erts_smp_atomic_xchg(&(PS)->interrupt, (long) 0)) -#define ERTS_POLLSET_UNSET_INTERRUPTED(PS) \ - erts_smp_atomic_set(&(PS)->interrupt, (long) 0) -#define ERTS_POLLSET_SET_INTERRUPTED(PS) \ - erts_smp_atomic_set(&(PS)->interrupt, (long) 1) -#define ERTS_POLLSET_IS_INTERRUPTED(PS) \ +#define ERTS_POLLSET_UNSET_INTERRUPTED_CHK(PS) unset_interrupted_chk((PS)) +#define ERTS_POLLSET_UNSET_INTERRUPTED(PS) \ +do { \ + erts_smp_atomic_set(&(PS)->interrupt, (long) 0); \ + ERTS_THR_MEMORY_BARRIER; \ +} while (0) +#define ERTS_POLLSET_SET_INTERRUPTED(PS) \ +do { \ + ERTS_THR_MEMORY_BARRIER; \ + erts_smp_atomic_set(&(PS)->interrupt, (long) 1); \ +} while (0) +#define ERTS_POLLSET_IS_INTERRUPTED(PS) \ ((int) erts_smp_atomic_read(&(PS)->interrupt)) #endif @@ -276,7 +286,7 @@ struct ErtsPollSet_ { ErtsPollSet next; int internal_fd_limit; ErtsFdStatus *fds_status; - int no_of_user_fds; + erts_smp_atomic_t no_of_user_fds; int fds_status_len; #if ERTS_POLL_USE_KERNEL_POLL int kp_fd; @@ -336,16 +346,30 @@ struct ErtsPollSet_ { #endif }; -#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT && !defined(ERTS_SMP) - static ERTS_INLINE int unset_interrupted_chk(ErtsPollSet ps) { + int res; +#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT && !defined(ERTS_SMP) /* This operation isn't atomic, but we have no need at all for an atomic operation here... */ - int res = ps->interrupt; + res = ps->interrupt; ps->interrupt = 0; +#else + res = (int) erts_smp_atomic_xchg(&ps->interrupt, (long) 0); + ERTS_THR_MEMORY_BARRIER; +#endif return res; + +} + +#ifdef ERTS_SMP + +static ERTS_INLINE int +set_poller_woken_chk(ErtsPollSet ps) +{ + ERTS_THR_MEMORY_BARRIER; + return (int) erts_smp_atomic_xchg(&ps->woken, (long) 1); } #endif @@ -828,7 +852,7 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp) ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK; ASSERT(ps->fds_status[fd].used_events); ps->fds_status[fd].used_events = 0; - ps->no_of_user_fds--; + erts_smp_atomic_dec(&ps->no_of_user_fds); update_fallback_pollset(ps, fd); ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK); break; @@ -878,11 +902,11 @@ batch_update_pollset(ErtsPollSet ps, int fd, ErtsPollBatchBuf *bbp) events = ERTS_POLL_EV_E2N(ps->fds_status[fd].events); if (!events) { buf[buf_len].events = POLLREMOVE; - ps->no_of_user_fds--; + erts_smp_atomic_dec(&ps->no_of_user_fds); } else if (!ps->fds_status[fd].used_events) { buf[buf_len].events = events; - ps->no_of_user_fds++; + erts_smp_atomic_inc(&ps->no_of_user_fds); } else { if ((ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST) @@ -972,12 +996,12 @@ batch_update_pollset(ErtsPollSet ps, int fd, ErtsPollBatchBuf *bbp) } if (used_events) { if (!events) { - ps->no_of_user_fds--; + erts_smp_atomic_dec(&ps->no_of_user_fds); } } else { if (events) - ps->no_of_user_fds++; + erts_smp_atomic_inc(&ps->no_of_user_fds); } ASSERT((events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0); ASSERT((used_events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0); @@ -1051,7 +1075,7 @@ update_pollset(ErtsPollSet ps, int fd) epe.data.fd = epe_templ.data.fd; res = epoll_ctl(ps->kp_fd, EPOLL_CTL_DEL, fd, &epe); } while (res != 0 && errno == EINTR); - ps->no_of_user_fds--; + erts_smp_atomic_dec(&ps->no_of_user_fds); ps->fds_status[fd].used_events = 0; } @@ -1059,11 +1083,11 @@ update_pollset(ErtsPollSet ps, int fd) /* A note on EPOLL_CTL_DEL: linux kernel versions before 2.6.9 need a non-NULL event pointer even though it is ignored... */ op = EPOLL_CTL_DEL; - ps->no_of_user_fds--; + erts_smp_atomic_dec(&ps->no_of_user_fds); } else if (!ps->fds_status[fd].used_events) { op = EPOLL_CTL_ADD; - ps->no_of_user_fds++; + erts_smp_atomic_inc(&ps->no_of_user_fds); } else { op = EPOLL_CTL_MOD; @@ -1113,7 +1137,7 @@ update_pollset(ErtsPollSet ps, int fd) /* Fall through ... */ case EPOLL_CTL_ADD: { ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK; - ps->no_of_user_fds--; + erts_smp_atomic_dec(&ps->no_of_user_fds); #if ERTS_POLL_USE_CONCURRENT_UPDATE if (!*update_fallback) { *update_fallback = 1; @@ -1201,7 +1225,7 @@ static int update_pollset(ErtsPollSet ps, int fd) #if ERTS_POLL_USE_FALLBACK ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK); #endif - ps->no_of_user_fds--; + erts_smp_atomic_dec(&ps->no_of_user_fds); last_pix = --ps->no_poll_fds; if (pix != last_pix) { /* Move last pix to this pix */ @@ -1228,7 +1252,7 @@ static int update_pollset(ErtsPollSet ps, int fd) ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK) || fd == ps->kp_fd); #endif - ps->no_of_user_fds++; + erts_smp_atomic_inc(&ps->no_of_user_fds); ps->fds_status[fd].pix = pix = ps->no_poll_fds++; if (pix >= ps->poll_fds_len) grow_poll_fds(ps, pix); @@ -1279,7 +1303,7 @@ static int update_pollset(ErtsPollSet ps, int fd) if (!ps->fds_status[fd].used_events) { ASSERT(events); - ps->no_of_user_fds++; + erts_smp_atomic_inc(&ps->no_of_user_fds); #if ERTS_POLL_USE_FALLBACK ps->no_select_fds++; ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_INFLBCK; @@ -1287,7 +1311,7 @@ static int update_pollset(ErtsPollSet ps, int fd) } else if (!events) { ASSERT(ps->fds_status[fd].used_events); - ps->no_of_user_fds--; + erts_smp_atomic_dec(&ps->no_of_user_fds); ps->fds_status[fd].events = events; #if ERTS_POLL_USE_FALLBACK ps->no_select_fds--; @@ -1888,7 +1912,8 @@ static ERTS_INLINE int check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked) { ASSERT(!*ps_locked); - if (ps->no_of_user_fds == 0 && tv->tv_usec == 0 && tv->tv_sec == 0) { + if (erts_smp_atomic_read(&ps->no_of_user_fds) == 0 + && tv->tv_usec == 0 && tv->tv_sec == 0) { /* Nothing to poll and zero timeout; done... */ return 0; } @@ -1926,7 +1951,7 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked) * the maximum number of file descriptors in the poll set. */ struct dvpoll poll_res; - int nfds = ps->no_of_user_fds; + int nfds = (int) erts_smp_atomic_read(&ps->no_of_user_fds); #ifdef ERTS_SMP nfds++; /* Wakeup pipe */ #endif @@ -2204,7 +2229,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) ps->internal_fd_limit = 0; ps->fds_status = NULL; ps->fds_status_len = 0; - ps->no_of_user_fds = 0; + erts_smp_atomic_init(&ps->no_of_user_fds, 0); #if ERTS_POLL_USE_KERNEL_POLL ps->kp_fd = -1; #if ERTS_POLL_USE_EPOLL @@ -2302,7 +2327,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) #if ERTS_POLL_USE_FALLBACK ps->fallback_used = 0; #endif - ps->no_of_user_fds = 0; /* Don't count wakeup pipe and fallback fd */ + erts_smp_atomic_set(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */ erts_smp_spin_lock(&pollsets_lock); ps->next = pollsets; @@ -2405,6 +2430,7 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) while (urqbp) { size += sizeof(ErtsPollSetUpdateRequestsBlock); pending_updates += urqbp->len; + urqbp = urqbp->next; } } #endif @@ -2447,7 +2473,7 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) pip->memory_size = size; - pip->poll_set_size = ps->no_of_user_fds; + pip->poll_set_size = (int) erts_smp_atomic_read(&ps->no_of_user_fds); #ifdef ERTS_SMP pip->poll_set_size++; /* Wakeup pipe */ #endif diff --git a/erts/emulator/sys/unix/erl9_start.c b/erts/emulator/sys/unix/erl9_start.c deleted file mode 100644 index 578062d7e2..0000000000 --- a/erts/emulator/sys/unix/erl9_start.c +++ /dev/null @@ -1,130 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2002-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif -#include "sys.h" -#include "erl_vm.h" -#include "global.h" -#include <stdio.h> -#include <string.h> -#include <stdlib.h> - -/* - * XXX This is a temporary dummy to make sys.c happy until we'll rewrite it. - */ -unsigned preloaded_size_ring0 = 1; -unsigned char preloaded_ring0[1] = {0}; - -Preload pre_loaded[] = { - {"ring0", 1, preloaded_ring0}, - {0, 0, 0} -}; - -int -main(int argc, char** argv) -{ - char sbuf[1024]; - struct { - void* p; - int sz; - } bins[2]; - int bin_num = 0; - FILE* fp; - char* progname = argv[0]; - char* eq; - - argv++, argc--; - - if (argc > 0 && argv[0][0] == '-') { - argv++, argc--; - } - if (argc < 1) { - abort(); - } - if ((fp = fopen(argv[0], "r")) == NULL) { - abort(); - } - - /* Needs to be called before any memory allocation */ - erts_short_init(); - - while (fgets(sbuf, sizeof sbuf, fp)) { - if (sbuf[0] == '#') { - continue; /* Comment */ - } else if (sbuf[0] == 'e' && strncmp("exec", sbuf, 4) == 0) { - continue; /* Comment ;-) */ - } else if ((eq = strchr(sbuf, '=')) != NULL) { - char* val; - char* p = strchr(sbuf, '\n'); - if (p) { - *p = '\0'; - } - *eq = '\0'; - val = erts_read_env(sbuf); - if (val == NULL) { - *eq = '='; - erts_sys_putenv(sbuf, eq - &sbuf[0]); - } - erts_free_read_env(val); - } else if (sbuf[0] == ':' && '0' <= sbuf[1] && sbuf[1] <= '9') { - int load_size = atoi(sbuf+1); - void* bin; - - bin = malloc(load_size); - if (fread(bin, 1, load_size, fp) != load_size) { - abort(); - } - bins[bin_num].p = bin; - bins[bin_num].sz = load_size; - bin_num++; - } else if (strcmp(sbuf, "--end--\n") == 0) { - int rval; - Eterm mod = NIL; - char *val; - - fclose(fp); - - if (bin_num != 2) { - abort(); - } - - val = erts_read_env("ERLBREAKHANDLER"); - if (val) { - init_break_handler(); - } - erts_free_read_env(val); - - if ((rval = erts_load_module(NULL, 0, NIL, &mod, bins[0].p, bins[0].sz)) < 0) { - fprintf(stderr, "%s: Load of initial module failed: %d\n", - progname, rval); - abort(); - } - erts_first_process(mod, bins[1].p, bins[1].sz, argc, argv); - free(bins[0].p); - free(bins[1].p); - process_main(); - abort(); - } else { - fprintf(stderr, "%s: bad line: %s\n", progname, sbuf); - abort(); - } - } - abort(); -} diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index 31ab5d03de..af4ab693dc 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -123,8 +123,6 @@ static ErtsSysReportExit *report_exit_transit_list; extern int check_async_ready(void); extern int driver_interrupt(int, int); -/*EXTERN_FUNCTION(void, increment_time, (int));*/ -/*EXTERN_FUNCTION(int, next_time, (_VOID_));*/ extern void do_break(void); extern void erl_sys_args(int*, char**); @@ -221,10 +219,10 @@ static struct fd_data { } *fd_data; /* indexed by fd */ /* static FUNCTION(int, write_fill, (int, char*, int)); unused? */ -static FUNCTION(void, note_child_death, (int, int)); +static void note_child_death(int, int); #if CHLDWTHR -static FUNCTION(void *, child_waiter, (void *)); +static void* child_waiter(void *); #endif /********************* General functions ****************************/ @@ -367,7 +365,7 @@ erts_sys_misc_mem_sz(void) /* * reset the terminal to the original settings on exit */ -void sys_tty_reset(void) +void sys_tty_reset(int exit_code) { if (using_oldshell && !replace_intr) { SET_BLOCKING(0); @@ -384,18 +382,6 @@ MALLOC_USE_HASH(1); #endif #ifdef USE_THREADS -static void *ethr_internal_alloc(size_t size) -{ - return erts_alloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, (Uint) size); -} -static void *ethr_internal_realloc(void *ptr, size_t size) -{ - return erts_realloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, ptr, (Uint) size); -} -static void ethr_internal_free(void *ptr) -{ - erts_free(ERTS_ALC_T_ETHR_INTERNAL, ptr); -} #ifdef ERTS_THR_HAVE_SIG_FUNCS /* @@ -457,6 +443,10 @@ thr_create_prepare_child(void *vtcdp) { erts_thr_create_data_t *tcdp = (erts_thr_create_data_t *) vtcdp; +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_thread_setup(); +#endif + #ifndef NO_FPE_SIGNALS /* * We do not want fp exeptions in other threads than the @@ -484,9 +474,6 @@ erts_sys_pre_init(void) #ifdef USE_THREADS { erts_thr_init_data_t eid = ERTS_THR_INIT_DATA_DEF_INITER; - eid.alloc = ethr_internal_alloc; - eid.realloc = ethr_internal_realloc; - eid.free = ethr_internal_free; eid.thread_create_child_func = thr_create_prepare_child; /* Before creation in parent */ @@ -534,13 +521,14 @@ erts_sys_pre_init(void) #endif #endif /* USE_THREADS */ erts_smp_atomic_init(&sys_misc_mem_sz, 0); - erts_smp_rwmtx_init(&environ_rwmtx, "environ"); } void erl_sys_init(void) { + erts_smp_rwmtx_init(&environ_rwmtx, "environ"); #if !DISABLE_VFORK + { int res; char bindir[MAXPATHLEN]; size_t bindirsz = sizeof(bindir); @@ -570,6 +558,7 @@ erl_sys_init(void) bindir, DIR_SEPARATOR_CHAR, CHILD_SETUP_PROG_NAME); + } #endif #ifdef USE_SETLINEBUF @@ -1324,9 +1313,18 @@ static char **build_unix_environment(char *block) } } - for (j = 0; j < i; j++) { - if (cpp[j][strlen(cpp[j])-1] == '=') { + for (j = 0; j < i; ) { + size_t last = strlen(cpp[j])-1; + if (cpp[j][last] == '=' && strchr(cpp[j], '=') == cpp[j]+last) { cpp[j] = cpp[--len]; + if (len < i) { + i--; + } else { + j++; + } + } + else { + j++; } } @@ -2562,7 +2560,6 @@ extern Preload pre_loaded[]; void erts_sys_alloc_init(void) { - elib_ensure_initialized(); } void *erts_sys_alloc(ErtsAlcType_t t, void *x, Uint sz) diff --git a/erts/emulator/sys/unix/sys_float.c b/erts/emulator/sys/unix/sys_float.c index c59c99f65e..5a682ab045 100644 --- a/erts/emulator/sys/unix/sys_float.c +++ b/erts/emulator/sys/unix/sys_float.c @@ -799,8 +799,17 @@ sys_chars_to_double(char* buf, double* fp) } #ifdef NO_FPE_SIGNALS - if (errno == ERANGE && (*fp == 0.0 || *fp == HUGE_VAL || *fp == -HUGE_VAL)) { - return -1; + if (errno == ERANGE) { + if (*fp == HUGE_VAL || *fp == -HUGE_VAL) { + /* overflow, should give error */ + return -1; + } else if (t == s && *fp == 0.0) { + /* This should give 0.0 - OTP-7178 */ + errno = 0; + + } else if (*fp == 0.0) { + return -1; + } } #endif return 0; @@ -810,7 +819,9 @@ int matherr(struct exception *exc) { #if !defined(NO_FPE_SIGNALS) - set_current_fp_exception((unsigned long)__builtin_return_address(0)); + volatile unsigned long *fpexnp = erts_get_current_fp_exception(); + if (fpexnp != NULL) + *fpexnp = (unsigned long)__builtin_return_address(0); #endif return 1; } diff --git a/erts/emulator/sys/vxworks/sys.c b/erts/emulator/sys/vxworks/sys.c index abddc7e107..411b4b37cf 100644 --- a/erts/emulator/sys/vxworks/sys.c +++ b/erts/emulator/sys/vxworks/sys.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1997-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ /* @@ -143,6 +143,14 @@ volatile int erts_break_requested; /********************* General functions ****************************/ +/* + * Reset the terminal to the original settings on exit + * (nothing to do for WxWorks). + */ +void sys_tty_reset(int exit_code) +{ +} + Uint erts_sys_misc_mem_sz(void) { diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c index d816cc2c07..a766fe9575 100644 --- a/erts/emulator/sys/win32/erl_poll.c +++ b/erts/emulator/sys/win32/erl_poll.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2007-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2007-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ #ifdef HAVE_CONFIG_H @@ -304,24 +304,51 @@ struct ErtsPollSet_ { erts_smp_atomic_set(&(PS)->polled, (long) 0) #define ERTS_POLLSET_IS_POLLED(PS) \ ((int) erts_smp_atomic_read(&(PS)->polled)) -#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) \ - ((int) erts_smp_atomic_xchg(&(PS)->woken, (long) 1)) -#define ERTS_POLLSET_SET_POLLER_WOKEN(PS) \ - erts_smp_atomic_set(&(PS)->woken, (long) 1) -#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS) \ - erts_smp_atomic_set(&(PS)->woken, (long) 0) -#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) \ + +#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) set_poller_woken_chk((PS)) +#define ERTS_POLLSET_SET_POLLER_WOKEN(PS) \ +do { \ + ERTS_THR_MEMORY_BARRIER; \ + erts_smp_atomic_set(&(PS)->woken, (long) 1); \ +} while (0) +#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS) \ +do { \ + erts_smp_atomic_set(&(PS)->woken, (long) 0); \ + ERTS_THR_MEMORY_BARRIER; \ +} while (0) +#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) \ ((int) erts_smp_atomic_read(&(PS)->woken)) -#define ERTS_POLLSET_UNSET_INTERRUPTED_CHK(PS) \ - ((int) erts_smp_atomic_xchg(&(PS)->interrupt, (long) 0)) -#define ERTS_POLLSET_UNSET_INTERRUPTED(PS) \ - erts_smp_atomic_set(&(PS)->interrupt, (long) 0) -#define ERTS_POLLSET_SET_INTERRUPTED(PS) \ - erts_smp_atomic_set(&(PS)->interrupt, (long) 1) -#define ERTS_POLLSET_IS_INTERRUPTED(PS) \ +#define ERTS_POLLSET_UNSET_INTERRUPTED_CHK(PS) unset_interrupted_chk((PS)) +#define ERTS_POLLSET_UNSET_INTERRUPTED(PS) \ +do { \ + erts_smp_atomic_set(&(PS)->interrupt, (long) 0); \ + ERTS_THR_MEMORY_BARRIER; \ +} while (0) +#define ERTS_POLLSET_SET_INTERRUPTED(PS) \ +do { \ + ERTS_THR_MEMORY_BARRIER; \ + erts_smp_atomic_set(&(PS)->interrupt, (long) 1); \ +} while (0) +#define ERTS_POLLSET_IS_INTERRUPTED(PS) \ ((int) erts_smp_atomic_read(&(PS)->interrupt)) +static ERTS_INLINE int +unset_interrupted_chk(ErtsPollSet ps) +{ + int res = (int) erts_smp_atomic_xchg(&ps->interrupt, (long) 0); + ERTS_THR_MEMORY_BARRIER; + return res; + +} + +static ERTS_INLINE int +set_poller_woken_chk(ErtsPollSet ps) +{ + ERTS_THR_MEMORY_BARRIER; + return (int) erts_smp_atomic_xchg(&ps->woken, (long) 1); +} + #else #define ERTS_POLLSET_LOCK(PS) diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c index 3194493ac8..15d4cd7361 100644 --- a/erts/emulator/sys/win32/sys.c +++ b/erts/emulator/sys/win32/sys.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ /* @@ -33,10 +33,11 @@ #include "../../drivers/win32/win_con.h" + void erts_sys_init_float(void); void erl_start(int, char**); -void erl_exit(int n, char*, _DOTS_); +void erl_exit(int n, char*, ...); void erl_error(char*, va_list); void erl_crash_dump(char*, int, char*, ...); @@ -66,10 +67,10 @@ static void async_read_file(struct async_io* aio, LPVOID buf, DWORD numToRead); static int async_write_file(struct async_io* aio, LPVOID buf, DWORD numToWrite); static int get_overlapped_result(struct async_io* aio, LPDWORD pBytesRead, BOOL wait); -static FUNCTION(BOOL, CreateChildProcess, (char *, HANDLE, HANDLE, - HANDLE, LPHANDLE, BOOL, - LPVOID, LPTSTR, unsigned, - char **, int *)); +static BOOL CreateChildProcess(char *, HANDLE, HANDLE, + HANDLE, LPHANDLE, BOOL, + LPVOID, LPTSTR, unsigned, + char **, int *); static int create_pipe(LPHANDLE, LPHANDLE, BOOL, BOOL); static int ApplicationType(const char* originalName, char fullPath[MAX_PATH], BOOL search_in_path, BOOL handle_quotes, @@ -92,9 +93,13 @@ static erts_smp_mtx_t sys_driver_data_lock; #define APPL_WIN3X 2 #define APPL_WIN32 3 -static FUNCTION(int, driver_write, (long, HANDLE, byte*, int)); +static int driver_write(long, HANDLE, byte*, int); static void common_stop(int); static int create_file_thread(struct async_io* aio, int mode); +#ifdef ERTS_SMP +static void close_active_handles(ErlDrvPort, const HANDLE* handles, int cnt); +static DWORD WINAPI threaded_handle_closer(LPVOID param); +#endif static DWORD WINAPI threaded_reader(LPVOID param); static DWORD WINAPI threaded_writer(LPVOID param); static DWORD WINAPI threaded_exiter(LPVOID param); @@ -132,6 +137,9 @@ static BOOL win_console = FALSE; static OSVERSIONINFO int_os_version; /* Version information for Win32. */ +#ifdef ERTS_SMP +static BOOL (WINAPI *fpCancelIoEx)(HANDLE,LPOVERLAPPED); +#endif /* This is the system's main function (which may or may not be called "main") - do general system-dependent initialization @@ -187,6 +195,17 @@ erts_sys_misc_mem_sz(void) return res; } +/* + * Reset the terminal to the original settings on exit + */ +void sys_tty_reset(int exit_code) +{ + if (exit_code > 0) + ConWaitForExit(); + else + ConNormalExit(); +} + void erl_sys_args(int* argc, char** argv) { char *event_name; @@ -665,25 +684,50 @@ release_driver_data(DriverData* dp) erts_smp_mtx_lock(&sys_driver_data_lock); #ifdef ERTS_SMP - /* This is a workaround for the fact that CancelIo cant cancel - requests issued by another thread and that we still cant use - CancelIoEx as that's only availabele in Vista etc. */ - if(dp->in.async_io_active && dp->in.fd != INVALID_HANDLE_VALUE) { - CloseHandle(dp->in.fd); - dp->in.fd = INVALID_HANDLE_VALUE; - DEBUGF(("Waiting for the in event thingie")); - WaitForSingleObject(dp->in.ov.hEvent,INFINITE); - DEBUGF(("...done\n")); - } - if(dp->out.async_io_active && dp->out.fd != INVALID_HANDLE_VALUE) { - CloseHandle(dp->out.fd); - dp->out.fd = INVALID_HANDLE_VALUE; - DEBUGF(("Waiting for the out event thingie")); - WaitForSingleObject(dp->out.ov.hEvent,INFINITE); - DEBUGF(("...done\n")); + if (fpCancelIoEx != NULL) { + if (dp->in.thread == (HANDLE) -1 && dp->in.fd != INVALID_HANDLE_VALUE) { + (*fpCancelIoEx)(dp->in.fd, NULL); + } + if (dp->out.thread == (HANDLE) -1 && dp->out.fd != INVALID_HANDLE_VALUE) { + (*fpCancelIoEx)(dp->out.fd, NULL); + } + } + else { + /* This is a workaround for the fact that CancelIo cant cancel + requests issued by another thread and that we cant use + CancelIoEx as that's only availabele in Vista etc. + R14: Avoid scheduler deadlock by only wait for 10ms, and then spawn + a thread that will keep waiting in in order to close handles. */ + HANDLE handles[2]; + int i = 0; + int timeout = 10; + if(dp->in.async_io_active && dp->in.fd != INVALID_HANDLE_VALUE) { + CloseHandle(dp->in.fd); + dp->in.fd = INVALID_HANDLE_VALUE; + DEBUGF(("Waiting for the in event thingie")); + if (WaitForSingleObject(dp->in.ov.hEvent,timeout) == WAIT_TIMEOUT) { + handles[i++] = dp->in.ov.hEvent; + dp->in.ov.hEvent = NULL; + timeout = 0; + } + DEBUGF(("...done\n")); + } + if(dp->out.async_io_active && dp->out.fd != INVALID_HANDLE_VALUE) { + CloseHandle(dp->out.fd); + dp->out.fd = INVALID_HANDLE_VALUE; + DEBUGF(("Waiting for the out event thingie")); + if (WaitForSingleObject(dp->out.ov.hEvent,timeout) == WAIT_TIMEOUT) { + handles[i++] = dp->out.ov.hEvent; + dp->out.ov.hEvent = NULL; + } + DEBUGF(("...done\n")); + } + if (i > 0) { + close_active_handles(dp->port_num, handles, i); + } } #else - if (dp->out.thread == (HANDLE) -1 && dp->in.fd != INVALID_HANDLE_VALUE) { + if (dp->in.thread == (HANDLE) -1 && dp->in.fd != INVALID_HANDLE_VALUE) { CancelIo(dp->in.fd); } if (dp->out.thread == (HANDLE) -1 && dp->out.fd != INVALID_HANDLE_VALUE) { @@ -726,6 +770,48 @@ release_driver_data(DriverData* dp) erts_smp_mtx_unlock(&sys_driver_data_lock); } +#ifdef ERTS_SMP + +struct handles_to_be_closed +{ + int cnt; + HANDLE handles[2]; +}; + +static void close_active_handles(ErlDrvPort port_num, const HANDLE* handles, int cnt) +{ + DWORD tid; + HANDLE thread; + int i; + struct handles_to_be_closed* htbc = erts_alloc(ERTS_ALC_T_DRV_TAB, + sizeof(struct handles_to_be_closed)); + htbc->cnt = cnt; + for (i=0; i < cnt; ++i) { + htbc->handles[i] = handles[i]; + (void) driver_select(port_num, (ErlDrvEvent)handles[i], + ERL_DRV_USE_NO_CALLBACK, 0); + } + thread = (HANDLE *) _beginthreadex(NULL, 0, threaded_handle_closer, htbc, 0, &tid); + CloseHandle(thread); +} + + +static DWORD WINAPI +threaded_handle_closer(LPVOID param) +{ + struct handles_to_be_closed* htbc = (struct handles_to_be_closed*) param; + int i; + DEBUGF(("threaded_handle_closer waiting for %d handles\r\n",htbc->cnt)); + WaitForMultipleObjects(htbc->cnt, htbc->handles, TRUE, INFINITE); + for (i=0; i < htbc->cnt; ++i) { + CloseHandle(htbc->handles[i]); + } + erts_free(ERTS_ALC_T_DRV_TAB, htbc); + DEBUGF(("threaded_handle_closer terminating\r\n")); + return 0; +} +#endif /* ERTS_SMP */ + /* * Stores input and output file descriptors in the DriverData structure, * and calls driver_select(). @@ -1015,12 +1101,19 @@ static int spawn_init() { int i; - +#ifdef ERTS_SMP + HMODULE module = GetModuleHandle("kernel32"); + fpCancelIoEx = (module != NULL) ? + (BOOL (WINAPI *)(HANDLE,LPOVERLAPPED)) + GetProcAddress(module,"CancelIoEx") : NULL; + DEBUGF(("fpCancelIoEx = %p\r\n", fpCancelIoEx)); +#endif driver_data = (struct driver_data *) erts_alloc(ERTS_ALC_T_DRV_TAB, max_files * sizeof(struct driver_data)); erts_smp_atomic_add(&sys_misc_mem_sz, max_files*sizeof(struct driver_data)); for (i = 0; i < max_files; i++) driver_data[i].port_num = PORT_FREE; + return 0; } @@ -2534,7 +2627,6 @@ erts_sys_main_thread(void) void erts_sys_alloc_init(void) { - elib_ensure_initialized(); } void *erts_sys_alloc(ErtsAlcType_t t, void *x, Uint sz) @@ -2881,19 +2973,14 @@ check_supported_os_version(void) } #ifdef USE_THREADS -static void *ethr_internal_alloc(size_t size) -{ - return erts_alloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, (Uint) size); -} -static void *ethr_internal_realloc(void *ptr, size_t size) -{ - return erts_realloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, ptr, (Uint) size); -} -static void ethr_internal_free(void *ptr) +#ifdef ERTS_ENABLE_LOCK_COUNT +static void +thr_create_prepare_child(void *vtcdp) { - erts_free(ERTS_ALC_T_ETHR_INTERNAL, ptr); + erts_lcnt_thread_setup(); } -#endif +#endif /* ERTS_ENABLE_LOCK_COUNT */ +#endif /* USE_THREADS */ void erts_sys_pre_init(void) @@ -2904,9 +2991,9 @@ erts_sys_pre_init(void) #ifdef USE_THREADS { erts_thr_init_data_t eid = ERTS_THR_INIT_DATA_DEF_INITER; - eid.alloc = ethr_internal_alloc; - eid.realloc = ethr_internal_realloc; - eid.free = ethr_internal_free; +#ifdef ERTS_ENABLE_LOCK_COUNT + eid.thread_create_child_func = thr_create_prepare_child; +#endif erts_thr_init(&eid); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_init(); @@ -2914,15 +3001,8 @@ erts_sys_pre_init(void) } #endif erts_smp_atomic_init(&sys_misc_mem_sz, 0); - erts_sys_env_init(); } -/* - * the last two only used for standalone erlang - * they should are used by sae_main in beam dll to - * enable standalone execution via erl_api-routines - */ - void noinherit_std_handle(DWORD type) { HANDLE h = GetStdHandle(type); @@ -2936,6 +3016,8 @@ void erl_sys_init(void) { HANDLE handle; + erts_sys_env_init(); + noinherit_std_handle(STD_OUTPUT_HANDLE); noinherit_std_handle(STD_INPUT_HANDLE); noinherit_std_handle(STD_ERROR_HANDLE); diff --git a/erts/emulator/sys/win32/sys_env.c b/erts/emulator/sys/win32/sys_env.c index ac4be3f316..02c8433a10 100644 --- a/erts/emulator/sys/win32/sys_env.c +++ b/erts/emulator/sys/win32/sys_env.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2002-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2002-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -145,15 +145,17 @@ merge_environment(char *old, char *add) for(j = 0; a_arg[j] != NULL; ++j){ char **tmp; char *current = a_arg[j]; + char *eq_p = strchr(current,'='); + int unset = (eq_p!=NULL && eq_p[1]=='\0'); if ((tmp = find_arg(c_arg, current)) != NULL) { - if (current[strlen(current)-1] != '=') { + if (!unset) { *tmp = current; } else { *tmp = c_arg[--i]; c_arg[i] = NULL; } - } else if (current[strlen(current)-1] != '=') { + } else if (!unset) { c_arg[i++] = current; c_arg[i] = NULL; } diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile index b1374950b2..a4c02da626 100644 --- a/erts/emulator/test/Makefile +++ b/erts/emulator/test/Makefile @@ -1,19 +1,19 @@ # # %CopyrightBegin% -# -# Copyright Ericsson AB 1997-2009. All Rights Reserved. -# +# +# Copyright Ericsson AB 1997-2010. All Rights Reserved. +# # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in # compliance with the License. You should have received a copy of the # Erlang Public License along with this software. If not, it can be # retrieved online at http://www.erlang.org/. -# +# # Software distributed under the License is distributed on an "AS IS" # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See # the License for the specific language governing rights and limitations # under the License. -# +# # %CopyrightEnd% # @@ -61,7 +61,7 @@ MODULES= \ exception_SUITE \ float_SUITE \ fun_SUITE \ - fun_r11_SUITE \ + fun_r12_SUITE \ gc_SUITE \ guard_SUITE \ hash_SUITE \ @@ -75,12 +75,12 @@ MODULES= \ node_container_SUITE \ nofrag_SUITE \ num_bif_SUITE \ - obsolete_SUITE \ op_SUITE \ port_SUITE \ port_bif_SUITE \ process_SUITE \ pseudoknot_SUITE \ + receive_SUITE \ ref_SUITE \ register_SUITE \ save_calls_SUITE \ @@ -100,6 +100,7 @@ MODULES= \ trace_local_SUITE \ trace_meta_SUITE \ trace_call_count_SUITE \ + trace_call_time_SUITE \ scheduler_SUITE \ old_scheduler_SUITE \ z_SUITE \ @@ -117,7 +118,8 @@ NO_OPT= bs_bincomp \ bs_match_int \ bs_match_tail \ bs_match_misc \ - bs_utf + bs_utf \ + guard NO_OPT_MODULES= $(NO_OPT:%=%_no_opt_SUITE) diff --git a/erts/emulator/test/beam_SUITE.erl b/erts/emulator/test/beam_SUITE.erl index cc1626630b..228ff15341 100644 --- a/erts/emulator/test/beam_SUITE.erl +++ b/erts/emulator/test/beam_SUITE.erl @@ -1,26 +1,26 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1998-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 1998-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% -module(beam_SUITE). -export([all/1, packed_registers/1, apply_last/1, apply_last_bif/1, - buildo_mucho/1, heap_sizes/1, big_lists/1]). + buildo_mucho/1, heap_sizes/1, big_lists/1, fconv/1]). -export([applied/2]). @@ -279,3 +279,26 @@ b() -> _} -> ok end. + +fconv(Config) when is_list(Config) -> + ?line do_fconv(atom), + ?line do_fconv(nil), + ?line do_fconv(tuple_literal), + ?line 3.0 = do_fconv(1.0, 2.0), + ok. + +do_fconv(Type) -> + try + do_fconv(Type, 1.0), + test_server:fail() + catch + error:badarith -> + ok + end. + +do_fconv(atom, Float) when is_float(Float) -> + Float + a; +do_fconv(nil, Float) when is_float(Float) -> + Float + []; +do_fconv(tuple_literal, Float) when is_float(Float) -> + Float + {a,b}. diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl index cfbc5dfe81..b4ef0e6d5a 100644 --- a/erts/emulator/test/bif_SUITE.erl +++ b/erts/emulator/test/bif_SUITE.erl @@ -1,19 +1,19 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2005-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 2005-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% @@ -22,12 +22,13 @@ -include("test_server.hrl"). -export([all/1,init_per_testcase/2,fin_per_testcase/2, + types/1, t_list_to_existing_atom/1,os_env/1,otp_7526/1, binary_to_atom/1,binary_to_existing_atom/1, atom_to_binary/1,min_max/1]). all(suite) -> - [t_list_to_existing_atom,os_env,otp_7526, + [types,t_list_to_existing_atom,os_env,otp_7526, atom_to_binary,binary_to_atom,binary_to_existing_atom, min_max]. @@ -39,6 +40,73 @@ fin_per_testcase(_Func, Config) -> Dog=?config(watchdog, Config), ?t:timetrap_cancel(Dog). +types(Config) when is_list(Config) -> + c:l(erl_bif_types), + case erlang:function_exported(erl_bif_types, module_info, 0) of + false -> + %% Fail cleanly. + ?line ?t:fail("erl_bif_types not compiled"); + true -> + types_1() + end. + +types_1() -> + ?line List0 = erlang:system_info(snifs), + + %% Ignore missing type information for hipe BIFs. + ?line List = [MFA || {M,_,_}=MFA <- List0, M =/= hipe_bifs], + + case [MFA || MFA <- List, not known_types(MFA)] of + [] -> + types_2(List); + BadTypes -> + io:put_chars("No type information:\n"), + io:format("~p\n", [lists:sort(BadTypes)]), + ?line ?t:fail({length(BadTypes),bifs_without_types}) + end. + +types_2(List) -> + BadArity = [MFA || {M,F,A}=MFA <- List, + begin + Types = erl_bif_types:arg_types(M, F, A), + length(Types) =/= A + end], + case BadArity of + [] -> + types_3(List); + [_|_] -> + io:put_chars("Bifs with bad arity\n"), + io:format("~p\n", [BadArity]), + ?line ?t:fail({length(BadArity),bad_arity}) + end. + +types_3(List) -> + BadSmokeTest = [MFA || {M,F,A}=MFA <- List, + begin + try erl_bif_types:type(M, F, A) of + Type -> + %% Test that type is returned. + not erl_types:is_erl_type(Type) + catch + Class:Error -> + io:format("~p: ~p ~p\n", + [MFA,Class,Error]), + true + end + end], + case BadSmokeTest of + [] -> + ok; + [_|_] -> + io:put_chars("Bifs with failing calls to erlang_bif_types:type/3 " + "(or with bogus return values):\n"), + io:format("~p\n", [BadSmokeTest]), + ?line ?t:fail({length(BadSmokeTest),bad_smoke_test}) + end. + +known_types({M,F,A}) -> + erl_bif_types:is_known(M, F, A). + t_list_to_existing_atom(Config) when is_list(Config) -> ?line all = list_to_existing_atom("all"), ?line ?MODULE = list_to_existing_atom(?MODULE_STRING), @@ -308,6 +376,18 @@ min_max(Config) when is_list(Config) -> ?line 42.0 = erlang:min(42.0, 42), ?line 42.0 = erlang:max(42.0, 42), + %% And now (R14) they are also autoimported! + ?line a = min(id(a), a), + ?line a = min(id(a), b), + ?line a = min(id(b), a), + ?line b = min(id(b), b), + ?line a = max(id(a), a), + ?line b = max(id(a), b), + ?line b = max(id(b), a), + ?line b = max(id(b), b), + + ?line 42.0 = min(42.0, 42), + ?line 42.0 = max(42.0, 42), ok. diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl index db2b3e10db..77d2579848 100644 --- a/erts/emulator/test/binary_SUITE.erl +++ b/erts/emulator/test/binary_SUITE.erl @@ -55,7 +55,6 @@ otp_5484/1,otp_5933/1, ordering/1,unaligned_order/1,gc_test/1, bit_sized_binary_sizes/1, - bitlevel_roundtrip/1, otp_6817/1,deep/1,obsolete_funs/1,robustness/1,otp_8117/1, otp_8180/1]). @@ -70,7 +69,7 @@ all(suite) -> bad_binary_to_term_2,safe_binary_to_term2, bad_binary_to_term, bad_terms, t_hash, bad_size, bad_term_to_binary, more_bad_terms, otp_5484, otp_5933, ordering, unaligned_order, - gc_test, bit_sized_binary_sizes, bitlevel_roundtrip, otp_6817, otp_8117, + gc_test, bit_sized_binary_sizes, otp_6817, otp_8117, deep,obsolete_funs,robustness,otp_8180]. init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> @@ -439,11 +438,11 @@ terms(Config) when is_list(Config) -> ok end, Term = binary_to_term(Bin), - Term = erlang:binary_to_term(Bin, [safe]), + Term = binary_to_term(Bin, [safe]), Unaligned = make_unaligned_sub_binary(Bin), Term = binary_to_term(Unaligned), - Term = erlang:binary_to_term(Unaligned, []), - Term = erlang:binary_to_term(Bin, [safe]), + Term = binary_to_term(Unaligned, []), + Term = binary_to_term(Bin, [safe]), BinC = erlang:term_to_binary(Term, [compressed]), Term = binary_to_term(BinC), true = size(BinC) =< size(Bin), @@ -543,7 +542,7 @@ bad_bin_to_term(BadBin) -> {'EXIT',{badarg,_}} = (catch binary_to_term(BadBin)). bad_bin_to_term(BadBin,Opts) -> - {'EXIT',{badarg,_}} = (catch erlang:binary_to_term(BadBin,Opts)). + {'EXIT',{badarg,_}} = (catch binary_to_term(BadBin,Opts)). safe_binary_to_term2(doc) -> "Test safety options for binary_to_term/2"; safe_binary_to_term2(Config) when is_list(Config) -> @@ -554,7 +553,7 @@ safe_binary_to_term2(Config) when is_list(Config) -> BadRef = <<131,114,0,3,BadHostAtom/binary,0,<<0,0,0,255>>/binary, Empty/binary,Empty/binary>>, ?line bad_bin_to_term(BadRef, [safe]), % good ref, with a bad atom - ?line fullsweep_after = erlang:binary_to_term(<<131,100,0,15,"fullsweep_after">>, [safe]), % should be a good atom + ?line fullsweep_after = binary_to_term(<<131,100,0,15,"fullsweep_after">>, [safe]), % should be a good atom BadExtFun = <<131,113,100,0,4,98,108,117,101,100,0,4,109,111,111,110,97,3>>, ?line bad_bin_to_term(BadExtFun, [safe]), ok. @@ -1150,35 +1149,6 @@ bsbs_1(A) -> Bin = binary_to_term(<<131,$M,5:32,A,0,0,0,0,0>>), BinSize = bit_size(Bin). -bitlevel_roundtrip(Config) when is_list(Config) -> - case ?t:is_release_available("r11b") of - true -> bitlevel_roundtrip_1(); - false -> {skip,"No R11B found"} - end. - -bitlevel_roundtrip_1() -> - Name = bitlevelroundtrip, - ?line N = list_to_atom(atom_to_list(Name) ++ "@" ++ hostname()), - ?line ?t:start_node(Name, slave, [{erl,[{release,"r11b"}]}]), - - ?line {<<128>>,1} = roundtrip(N, <<1:1>>), - ?line {<<64>>,2} = roundtrip(N, <<1:2>>), - ?line {<<16#E0>>,3} = roundtrip(N, <<7:3>>), - ?line {<<16#70>>,4} = roundtrip(N, <<7:4>>), - ?line {<<16#10>>,5} = roundtrip(N, <<2:5>>), - ?line {<<16#8>>,6} = roundtrip(N, <<2:6>>), - ?line {<<16#2>>,7} = roundtrip(N, <<1:7>>), - ?line {<<8,128>>,1} = roundtrip(N, <<8,1:1>>), - ?line {<<42,248>>,5} = roundtrip(N, <<42,31:5>>), - - ?line ?t:stop_node(N), - ok. - -roundtrip(Node, Term) -> - {badrpc,{'EXIT',Res}} = rpc:call(Node, erlang, exit, [Term]), - io:format("<<~p bits>> => ~w", [bit_size(Term),Res]), - Res. - deep(Config) when is_list(Config) -> ?line deep_roundtrip(lists:foldl(fun(E, A) -> [E,A] diff --git a/erts/emulator/test/busy_port_SUITE.erl b/erts/emulator/test/busy_port_SUITE.erl index 9b16170293..7350aef4ec 100644 --- a/erts/emulator/test/busy_port_SUITE.erl +++ b/erts/emulator/test/busy_port_SUITE.erl @@ -1,19 +1,19 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% @@ -182,7 +182,7 @@ system_monitor(Config) when is_list(Config) -> ?line Master ! {Owner, {command, "u"}}, ?line {Busy,beta} = rec(Void), ?line Void = rec(Void), - ?line NewMonitor = erlang:system_monitor(OldMonitor), + ?line _NewMonitor = erlang:system_monitor(OldMonitor), ?line OldMonitor = erlang:system_monitor(), ?line OldMonitor = erlang:system_monitor(OldMonitor), %% @@ -361,7 +361,6 @@ soft_busy_driver(Config) when is_list(Config) -> hs_test(Config, false). hs_test(Config, HardBusy) when is_list(Config) -> - ?line Me = self(), ?line DrvName = case HardBusy of true -> 'hard_busy_drv'; false -> 'soft_busy_drv' @@ -479,7 +478,7 @@ hs_busy_pcmd(Prt, Opts, StartFun, EndFun) -> Tester ! {self(), doing_port_command}, Start = os:timestamp(), Res = try {return, - erlang:port_command(Prt, [], Opts)} + port_command(Prt, [], Opts)} catch Exception:Error -> {Exception, Error} end, End = os:timestamp(), diff --git a/erts/emulator/test/decode_packet_SUITE.erl b/erts/emulator/test/decode_packet_SUITE.erl index 13f17e972c..d9e961be2f 100644 --- a/erts/emulator/test/decode_packet_SUITE.erl +++ b/erts/emulator/test/decode_packet_SUITE.erl @@ -1,19 +1,19 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2008-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 2008-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% @@ -24,10 +24,10 @@ -include("test_server.hrl"). -export([all/1,init_per_testcase/2,fin_per_testcase/2, - basic/1, packet_size/1, neg/1, http/1, line/1, ssl/1]). + basic/1, packet_size/1, neg/1, http/1, line/1, ssl/1, otp_8536/1]). all(suite) -> - [basic, packet_size, neg, http, line, ssl]. + [basic, packet_size, neg, http, line, ssl, otp_8536]. init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Seed = {S1,S2,S3} = now(), @@ -304,6 +304,10 @@ http(Config) when is_list(Config) -> {ok, {http_request, 'GET', ResB, {1,1}}, Rest} = decode_pkt(http_bin,Bin) end, lists:foreach(UriF, http_uri_variants()), + + %% Response with empty phrase + ?line {ok,{http_response,{1,1},200,[]},<<>>} = decode_pkt(http, <<"HTTP/1.1 200\r\n">>, []), + ?line {ok,{http_response,{1,1},200,<<>>},<<>>} = decode_pkt(http_bin, <<"HTTP/1.1 200\r\n">>, []), ok. http_with_bin(http) -> @@ -504,6 +508,27 @@ ssl(Config) when is_list(Config) -> F(v2hello), ok. +otp_8536(doc) -> ["Corrupt sub-binary-strings from httph_bin"]; +otp_8536(Config) when is_list(Config) -> + lists:foreach(fun otp_8536_do/1, lists:seq(1,50)), + ok. + +otp_8536_do(N) -> + Data = <<"some data 123">>, + Letters = <<"bcdefghijklmnopqrstuvwxyzyxwvutsrqponmlkjihgfedcba">>, + <<HdrTail:N/binary,_/binary>> = Letters, + Hdr = <<$A, HdrTail/binary>>, + Bin = <<Hdr/binary, ": ", Data/binary, "\r\n\r\n">>, + + io:format("Bin='~p'\n",[Bin]), + ?line {ok,{http_header,0,Hdr2,undefined,Data2},<<"\r\n">>} = decode_pkt(httph_bin, Bin, []), + + %% Do something to trash the C-stack, how about another decode_packet: + decode_pkt(httph_bin,<<Letters/binary, ": ", Data/binary, "\r\n\r\n">>, []), + + %% Now check that we got the expected binaries + {Hdr, Data} = {Hdr2, Data2}. + decode_pkt(Type,Bin) -> decode_pkt(Type,Bin,[]). decode_pkt(Type,Bin,Opts) -> diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl index 8f48d8a992..7c19274696 100644 --- a/erts/emulator/test/distribution_SUITE.erl +++ b/erts/emulator/test/distribution_SUITE.erl @@ -1,23 +1,24 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% -module(distribution_SUITE). +-compile(r12). %% Tests distribution and the tcp driver. diff --git a/erts/emulator/test/driver_SUITE_data/chkio_drv.c b/erts/emulator/test/driver_SUITE_data/chkio_drv.c index 9e1e5e72c2..b571cb30e6 100644 --- a/erts/emulator/test/driver_SUITE_data/chkio_drv.c +++ b/erts/emulator/test/driver_SUITE_data/chkio_drv.c @@ -17,7 +17,7 @@ */ #ifndef UNIX -#if !defined(__WIN32__) && !defined(_OSE_) && !defined(VXWORKS) +#if !defined(__WIN32__) && !defined(VXWORKS) #define UNIX 1 #endif #endif diff --git a/erts/emulator/test/driver_SUITE_data/io_ready_exit_drv.c b/erts/emulator/test/driver_SUITE_data/io_ready_exit_drv.c index 25d4b17001..6afa46b3a2 100644 --- a/erts/emulator/test/driver_SUITE_data/io_ready_exit_drv.c +++ b/erts/emulator/test/driver_SUITE_data/io_ready_exit_drv.c @@ -17,7 +17,7 @@ */ #ifndef UNIX -#if !defined(__WIN32__) && !defined(_OSE_) && !defined(VXWORKS) +#if !defined(__WIN32__) && !defined(VXWORKS) #define UNIX 1 #endif #endif diff --git a/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c b/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c index c7a42aa687..e49de388b4 100644 --- a/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c +++ b/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c @@ -29,7 +29,7 @@ */ #ifndef UNIX -#if !defined(__WIN32__) && !defined(_OSE_) && !defined(VXWORKS) +#if !defined(__WIN32__) && !defined(VXWORKS) #define UNIX 1 #endif #endif diff --git a/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c b/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c index c80e492e3f..e7d9a294fa 100644 --- a/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c +++ b/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c @@ -17,7 +17,7 @@ */ #ifndef UNIX -#if !defined(__WIN32__) && !defined(_OSE_) && !defined(VXWORKS) +#if !defined(__WIN32__) && !defined(VXWORKS) #define UNIX 1 #endif #endif diff --git a/erts/emulator/test/driver_SUITE_data/peek_non_existing_queue_drv.c b/erts/emulator/test/driver_SUITE_data/peek_non_existing_queue_drv.c index f429a5b51e..3a5b5af13a 100644 --- a/erts/emulator/test/driver_SUITE_data/peek_non_existing_queue_drv.c +++ b/erts/emulator/test/driver_SUITE_data/peek_non_existing_queue_drv.c @@ -28,7 +28,7 @@ */ #ifndef UNIX -#if !defined(__WIN32__) && !defined(_OSE_) && !defined(VXWORKS) +#if !defined(__WIN32__) && !defined(VXWORKS) #define UNIX 1 #endif #endif diff --git a/erts/emulator/test/float_SUITE.erl b/erts/emulator/test/float_SUITE.erl index 102e472ea6..99e9457985 100644 --- a/erts/emulator/test/float_SUITE.erl +++ b/erts/emulator/test/float_SUITE.erl @@ -22,7 +22,10 @@ -include("test_server.hrl"). -export([all/1,init_per_testcase/2,fin_per_testcase/2, - fpe/1,fp_drv/1,fp_drv_thread/1,denormalized/1,match/1,bad_float_unpack/1]). + fpe/1,fp_drv/1,fp_drv_thread/1,denormalized/1,match/1, + bad_float_unpack/1]). +-export([otp_7178/1]). + init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog = ?t:timetrap(?t:minutes(3)), @@ -33,7 +36,29 @@ fin_per_testcase(_Func, Config) -> ?t:timetrap_cancel(Dog). all(suite) -> - [fpe,fp_drv,fp_drv_thread,denormalized,match,bad_float_unpack]. + [fpe, + fp_drv, + fp_drv_thread, + otp_7178, + denormalized, + match, + bad_float_unpack]. + +%% +%% OTP-7178, list_to_float on very small numbers should give 0.0 +%% instead of exception, i.e. ignore underflow. +%% +otp_7178(suite) -> + []; +otp_7178(doc) -> + ["test that list_to_float on very small numbers give 0.0"]; +otp_7178(Config) when is_list(Config) -> + ?line X = list_to_float("1.0e-325"), + ?line true = (X < 0.00000001) and (X > -0.00000001), + ?line Y = list_to_float("1.0e-325325325"), + ?line true = (Y < 0.00000001) and (Y > -0.00000001), + ?line {'EXIT', {badarg,_}} = (catch list_to_float("1.0e83291083210")), + ok. %% Forces floating point exceptions and tests that subsequent, legal, %% operations are calculated correctly. Original version by Sebastian diff --git a/erts/emulator/test/fun_r11_SUITE.erl b/erts/emulator/test/fun_r12_SUITE.erl index 61ba816cc8..9262731dcb 100644 --- a/erts/emulator/test/fun_r11_SUITE.erl +++ b/erts/emulator/test/fun_r12_SUITE.erl @@ -1,24 +1,24 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2007-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 2007-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% --module(fun_r11_SUITE). --compile(r11). +-module(fun_r12_SUITE). +-compile(r12). -export([all/1,init_per_testcase/2,fin_per_testcase/2,dist_old_release/1]). @@ -37,36 +37,36 @@ fin_per_testcase(_Case, Config) -> ok. dist_old_release(Config) when is_list(Config) -> - case ?t:is_release_available("r11b") of + case ?t:is_release_available("r12b") of true -> do_dist_old(Config); - false -> {skip,"No R11B found"} + false -> {skip,"No R12B found"} end. do_dist_old(Config) when is_list(Config) -> ?line Pa = filename:dirname(code:which(?MODULE)), - Name = fun_dist_r11, + Name = fun_dist_r12, ?line {ok,Node} = ?t:start_node(Name, peer, [{args,"-pa "++Pa}, - {erl,[{release,"r11b"}]}]), + {erl,[{release,"r12b"}]}]), ?line Pid = spawn_link(Node, fun() -> receive Fun when is_function(Fun) -> - R11BFun = fun(H) -> cons(H, [b,c]) end, - Fun(Fun, R11BFun) + R12BFun = fun(H) -> cons(H, [b,c]) end, + Fun(Fun, R12BFun) end end), Self = self(), - Fun = fun(F, R11BFun) -> + Fun = fun(F, R12BFun) -> {pid,Self} = erlang:fun_info(F, pid), {module,?MODULE} = erlang:fun_info(F, module), - Self ! {ok,F,R11BFun} + Self ! {ok,F,R12BFun} end, ?line Pid ! Fun, ?line receive - {ok,Fun,R11BFun} -> - ?line [a,b,c] = R11BFun(a); + {ok,Fun,R12BFun} -> + ?line [a,b,c] = R12BFun(a); Other -> ?line ?t:fail({bad_message,Other}) end, diff --git a/erts/emulator/test/guard_SUITE.erl b/erts/emulator/test/guard_SUITE.erl index 23482a20d7..8fef36dfaf 100644 --- a/erts/emulator/test/guard_SUITE.erl +++ b/erts/emulator/test/guard_SUITE.erl @@ -1,33 +1,34 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% -module(guard_SUITE). -export([all/1, bad_arith/1, bad_tuple/1, test_heap_guards/1, guard_bifs/1, - type_tests/1]). + type_tests/1,guard_bif_binary_part/1]). -include("test_server.hrl"). -export([init/3]). -import(lists, [member/2]). -all(suite) -> [bad_arith, bad_tuple, test_heap_guards, guard_bifs, type_tests]. +all(suite) -> [bad_arith, bad_tuple, test_heap_guards, guard_bifs, + type_tests, guard_bif_binary_part]. bad_arith(doc) -> "Test that a bad arithmetic operation in a guard works correctly."; bad_arith(Config) when is_list(Config) -> @@ -136,6 +137,170 @@ init(Fun, Args, Filler) -> dummy(_) -> ok. +-define(MASK_ERROR(EXPR),mask_error((catch (EXPR)))). +mask_error({'EXIT',{Err,_}}) -> + Err; +mask_error(Else) -> + Else. + +guard_bif_binary_part(doc) -> + ["Test the binary_part/2,3 guard BIF's extensively"]; +guard_bif_binary_part(Config) when is_list(Config) -> + %% Overflow tests that need to be unoptimized + ?line badarg = + ?MASK_ERROR( + binary_part(<<1,2,3>>,{16#FFFFFFFFFFFFFFFF, + -16#7FFFFFFFFFFFFFFF-1})), + ?line badarg = + ?MASK_ERROR( + binary_part(<<1,2,3>>,{16#FFFFFFFFFFFFFFFF, + 16#7FFFFFFFFFFFFFFF})), + F = fun(X) -> + Master = self(), + {Pid,Ref} = spawn_monitor( fun() -> + A = lists:duplicate(X,a), + B = [do_binary_part_guard() | A], + Master ! {self(),hd(B)}, + ok + end), + receive + {Pid,ok} -> + erlang:demonitor(Ref,[flush]), + ok; + Error -> + Error + end + end, + [ ok = F(N) || N <- lists:seq(1,10000) ], + ok. + + +do_binary_part_guard() -> + ?line 1 = bptest(<<1,2,3>>), + ?line 2 = bptest(<<2,1,3>>), + ?line error = bptest(<<1>>), + ?line error = bptest(<<>>), + ?line error = bptest(apa), + ?line 3 = bptest(<<2,3,3>>), + % With one variable (pos) + ?line 1 = bptest(<<1,2,3>>,1), + ?line 2 = bptest(<<2,1,3>>,1), + ?line error = bptest(<<1>>,1), + ?line error = bptest(<<>>,1), + ?line error = bptest(apa,1), + ?line 3 = bptest(<<2,3,3>>,1), + % With one variable (length) + ?line 1 = bptesty(<<1,2,3>>,1), + ?line 2 = bptesty(<<2,1,3>>,1), + ?line error = bptesty(<<1>>,1), + ?line error = bptesty(<<>>,1), + ?line error = bptesty(apa,1), + ?line 3 = bptesty(<<2,3,3>>,2), + % With one variable (whole tuple) + ?line 1 = bptestx(<<1,2,3>>,{1,1}), + ?line 2 = bptestx(<<2,1,3>>,{1,1}), + ?line error = bptestx(<<1>>,{1,1}), + ?line error = bptestx(<<>>,{1,1}), + ?line error = bptestx(apa,{1,1}), + ?line 3 = bptestx(<<2,3,3>>,{1,2}), + % With two variables + ?line 1 = bptest(<<1,2,3>>,1,1), + ?line 2 = bptest(<<2,1,3>>,1,1), + ?line error = bptest(<<1>>,1,1), + ?line error = bptest(<<>>,1,1), + ?line error = bptest(apa,1,1), + ?line 3 = bptest(<<2,3,3>>,1,2), + % Direct (autoimported) call, these will be evaluated by the compiler... + ?line <<2>> = binary_part(<<1,2,3>>,1,1), + ?line <<1>> = binary_part(<<2,1,3>>,1,1), + % Compiler warnings due to constant evaluation expected (3) + ?line badarg = ?MASK_ERROR(binary_part(<<1>>,1,1)), + ?line badarg = ?MASK_ERROR(binary_part(<<>>,1,1)), + ?line badarg = ?MASK_ERROR(binary_part(apa,1,1)), + ?line <<3,3>> = binary_part(<<2,3,3>>,1,2), + % Direct call through apply + ?line <<2>> = apply(erlang,binary_part,[<<1,2,3>>,1,1]), + ?line <<1>> = apply(erlang,binary_part,[<<2,1,3>>,1,1]), + % Compiler warnings due to constant evaluation expected (3) + ?line badarg = ?MASK_ERROR(apply(erlang,binary_part,[<<1>>,1,1])), + ?line badarg = ?MASK_ERROR(apply(erlang,binary_part,[<<>>,1,1])), + ?line badarg = ?MASK_ERROR(apply(erlang,binary_part,[apa,1,1])), + ?line <<3,3>> = apply(erlang,binary_part,[<<2,3,3>>,1,2]), + % Constant propagation + ?line Bin = <<1,2,3>>, + ?line ok = if + binary_part(Bin,1,1) =:= <<2>> -> + ok; + %% Compiler warning, clause cannot match (expected) + true -> + error + end, + ?line ok = if + binary_part(Bin,{1,1}) =:= <<2>> -> + ok; + %% Compiler warning, clause cannot match (expected) + true -> + error + end, + ok. + + +bptest(B) when length(B) =:= 1337 -> + 1; +bptest(B) when binary_part(B,{1,1}) =:= <<2>> -> + 1; +bptest(B) when erlang:binary_part(B,1,1) =:= <<1>> -> + 2; +bptest(B) when erlang:binary_part(B,{1,2}) =:= <<3,3>> -> + 3; +bptest(_) -> + error. + +bptest(B,A) when length(B) =:= A -> + 1; +bptest(B,A) when binary_part(B,{A,1}) =:= <<2>> -> + 1; +bptest(B,A) when erlang:binary_part(B,A,1) =:= <<1>> -> + 2; +bptest(B,A) when erlang:binary_part(B,{A,2}) =:= <<3,3>> -> + 3; +bptest(_,_) -> + error. + +bptestx(B,A) when length(B) =:= A -> + 1; +bptestx(B,A) when binary_part(B,A) =:= <<2>> -> + 1; +bptestx(B,A) when erlang:binary_part(B,A) =:= <<1>> -> + 2; +bptestx(B,A) when erlang:binary_part(B,A) =:= <<3,3>> -> + 3; +bptestx(_,_) -> + error. + +bptesty(B,A) when length(B) =:= A -> + 1; +bptesty(B,A) when binary_part(B,{1,A}) =:= <<2>> -> + 1; +bptesty(B,A) when erlang:binary_part(B,1,A) =:= <<1>> -> + 2; +bptesty(B,A) when erlang:binary_part(B,{1,A}) =:= <<3,3>> -> + 3; +bptesty(_,_) -> + error. + +bptest(B,A,_C) when length(B) =:= A -> + 1; +bptest(B,A,C) when binary_part(B,{A,C}) =:= <<2>> -> + 1; +bptest(B,A,C) when erlang:binary_part(B,A,C) =:= <<1>> -> + 2; +bptest(B,A,C) when erlang:binary_part(B,{A,C}) =:= <<3,3>> -> + 3; +bptest(_,_,_) -> + error. + + guard_bifs(doc) -> "Test all guard bifs with nasty (but legal arguments)."; guard_bifs(Config) when is_list(Config) -> ?line Big = -237849247829874297658726487367328971246284736473821617265433, diff --git a/erts/emulator/test/hash_SUITE.erl b/erts/emulator/test/hash_SUITE.erl index 85bdb8bff8..f5d1871bfb 100644 --- a/erts/emulator/test/hash_SUITE.erl +++ b/erts/emulator/test/hash_SUITE.erl @@ -480,14 +480,14 @@ otp_5292_test() -> S2 = md5([md5(hash_int(S, E, PH)) || {Start, N, Sz} <- d(), {S, E} <- int(Start, N, Sz)]), ?line Comment = case S1 of - <<43,186,76,102,87,4,110,245,203,177,206,6,130,69,43,99>> -> + <<4,248,208,156,200,131,7,1,173,13,239,173,112,81,16,174>> -> ?line big = erlang:system_info(endian), "Big endian machine"; - <<21,206,139,15,149,28,167,81,98,225,132,254,49,125,174,195>> -> + <<180,28,33,231,239,184,71,125,76,47,227,241,78,184,176,233>> -> ?line little = erlang:system_info(endian), "Little endian machine" end, - ?line <<140,37,79,80,26,242,130,22,20,229,123,240,223,244,43,99>> = S2, + ?line <<124,81,198,121,174,233,19,137,10,83,33,80,226,111,238,99>> = S2, ?line 2 = erlang:hash(1, (1 bsl 27) -1), ?line {'EXIT', _} = (catch erlang:hash(1, (1 bsl 27))), {comment, Comment}. @@ -507,7 +507,7 @@ hash_int(Start, End, F) -> {Start, End, md5(HL)}. md5(T) -> - erlang:md5(term_to_binary(T)). + erlang:md5(term_to_binary(T)). bit_level_binaries() -> ?line [3511317,7022633,14044578,28087749,56173436,112344123,90467083|_] = diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index 522caec8f1..f45cfa3e4a 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -20,23 +20,39 @@ -module(nif_SUITE). %%-define(line_trace,true). -%%-define(CHECK(Exp,Got), ?line check(Exp,Got,?LINE)). --define(CHECK(Exp,Got), ?line Exp = Got). +-define(CHECK(Exp,Got), check(Exp,Got,?LINE)). +%%-define(CHECK(Exp,Got), ?line Exp = Got). -include("test_server.hrl"). --export([all/1, fin_per_testcase/2, basic/1, reload/1, upgrade/1, heap_frag/1, +-export([all/1, + %%init_per_testcase/2, + fin_per_testcase/2, basic/1, reload/1, upgrade/1, heap_frag/1, types/1, many_args/1, binaries/1, get_string/1, get_atom/1, api_macros/1, - from_array/1, iolist_as_binary/1, resource/1, resource_takeover/1, - threading/1, neg/1]). + from_array/1, iolist_as_binary/1, resource/1, resource_binary/1, resource_takeover/1, + threading/1, send/1, send2/1, send3/1, send_threaded/1, neg/1, is_checks/1, + get_length/1, make_atom/1, make_string/1]). -export([many_args_100/100]). + + +%% -export([lib_version/0,call_history/0,hold_nif_mod_priv_data/1,nif_mod_call_history/0, +%% list_seq/1,type_test/0,tuple_2_list/1,is_identical/2,compare/2, +%% clone_bin/1,make_sub_bin/3,string_to_bin/2,atom_to_bin/2,macros/1, +%% tuple_2_list_and_tuple/1,iolist_2_bin/1,get_resource_type/1,alloc_resource/2, +%% make_resource/1,get_resource/2,release_resource/1,last_resource_dtor_call/0, +%% make_new_resource/2,make_new_resource_binary/1,send_list_seq/2,send_new_blob/2, +%% alloc_msgenv/0,clear_msgenv/1,grow_blob/2,send_blob/2,send_blob_thread/3, +%% join_send_thread/1]). + + -define(nif_stub,nif_stub_error(?LINE)). all(suite) -> [basic, reload, upgrade, heap_frag, types, many_args, binaries, get_string, - get_atom, api_macros, from_array, iolist_as_binary, resource, - resource_takeover, threading, neg]. + get_atom, api_macros, from_array, iolist_as_binary, resource, resource_binary, + resource_takeover, threading, send, send2, send3, send_threaded, neg, is_checks, + get_length, make_atom, make_string]. %%init_per_testcase(_Case, Config) -> %% ?line Dog = ?t:timetrap(?t:seconds(60*60*24)), @@ -473,12 +489,51 @@ resource_new_do2(Type) -> {{PtrA,BinA}, {ResB,PtrB,BinB}}. resource_neg(TypeA) -> + resource_neg_do(TypeA), + + catch exit(42), % dummy exception to purge saved stacktraces from earlier exception + erlang:garbage_collect(), + ?line {_,_,2} = last_resource_dtor_call(), + ok. + +resource_neg_do(TypeA) -> TypeB = get_resource_type(1), - Aptr = alloc_resource(TypeA, <<"Arnold">>), - Bptr = alloc_resource(TypeB, <<"Bobo">>), - ?line {'EXIT',{badarg,_}} = (catch get_resource(TypeA, Bptr)), - ?line {'EXIT',{badarg,_}} = (catch get_resource(TypeB, Aptr)), + ResA = make_new_resource(TypeA, <<"Arnold">>), + ResB= make_new_resource(TypeB, <<"Bobo">>), + ?line {'EXIT',{badarg,_}} = (catch get_resource(TypeA, ResB)), + ?line {'EXIT',{badarg,_}} = (catch get_resource(TypeB, ResA)), ok. + +resource_binary(doc) -> ["Test enif_make_resource_binary"]; +resource_binary(suite) -> []; +resource_binary(Config) when is_list(Config) -> + ?line ensure_lib_loaded(Config, 1), + ?line {Ptr,Bin} = resource_binary_do(), + erlang:garbage_collect(), + Last = last_resource_dtor_call(), + ?CHECK({Ptr,Bin,1}, Last), + ok. + +resource_binary_do() -> + Bin = <<"Hej Hopp i lingonskogen">>, + ?line {Ptr,ResBin1} = make_new_resource_binary(Bin), + ?line ResBin1 = Bin, + ?line ResInfo = {Ptr,_} = get_resource(binary_resource_type,ResBin1), + + Papa = self(), + Forwarder = spawn_link(fun() -> forwarder(Papa) end), + io:format("sending to forwarder pid=~p\n",[Forwarder]), + Forwarder ! ResBin1, + ResBin2 = receive_any(), + ?line ResBin2 = ResBin1, + ?line ResInfo = get_resource(binary_resource_type,ResBin2), + Forwarder ! terminate, + ?line {Forwarder, 1} = receive_any(), + erlang:garbage_collect(), + ?line ResInfo = get_resource(binary_resource_type,ResBin1), + ?line ResInfo = get_resource(binary_resource_type,ResBin2), + ResInfo. + -define(RT_CREATE,1). -define(RT_TAKEOVER,2). @@ -743,7 +798,282 @@ threading(Config) when is_list(Config) -> ?line ok = tester:load_nif_lib(Config, "tsd"), ?line ok = tester:run(). + +send(doc) -> ["Test NIF message sending"]; +send(Config) when is_list(Config) -> + ensure_lib_loaded(Config), + + N = 1500, + List = lists:seq(1,N), + ?line {ok,1} = send_list_seq(N, self), + ?line {ok,1} = send_list_seq(N, self()), + ?line List = receive_any(), + ?line List = receive_any(), + Papa = self(), + spawn_link(fun() -> ?line {ok,1} = send_list_seq(N, Papa) end), + ?line List = receive_any(), + + ?line {ok, 1, BlobS} = send_new_blob(self(), other_term()), + ?line BlobR = receive_any(), + io:format("Sent ~p\nGot ~p\n", [BlobS, BlobR]), + ?line BlobR = BlobS, + + %% send to dead pid + {DeadPid, DeadMon} = spawn_monitor(fun() -> void end), + ?line {'DOWN', DeadMon, process, DeadPid, normal} = receive_any(), + {ok,0} = send_list_seq(7, DeadPid), + ok. + +send2(doc) -> ["More NIF message sending"]; +send2(Config) when is_list(Config) -> + ensure_lib_loaded(Config), + + send2_do1(fun send_blob_dbg/2), + ok. + +send_threaded(doc) -> ["Send msg from user thread"]; +send_threaded(Config) when is_list(Config) -> + case erlang:system_info(smp_support) of + true -> + send2_do1(fun(ME,To) -> send_blob_thread_dbg(ME,To,join) end), + send2_do1(fun(ME,To) -> send_blob_thread_and_join(ME,To) end), + ok; + false -> + {skipped,"No threaded send on non-SMP"} + end. + + +send2_do1(SendBlobF) -> + io:format("sending to self=~p\n",[self()]), + send2_do2(SendBlobF, self()), + + Papa = self(), + Forwarder = spawn_link(fun() -> forwarder(Papa) end), + io:format("sending to forwarder pid=~p\n",[Forwarder]), + send2_do2(SendBlobF, Forwarder), + Forwarder ! terminate, + ?line {Forwarder, 4} = receive_any(), + ok. + +send2_do2(SendBlobF, To) -> + MsgEnv = alloc_msgenv(), + repeat(50, fun(_) -> grow_blob(MsgEnv,other_term()) end, []), + ?line {ok,1,Blob0} = SendBlobF(MsgEnv, To), + ?line Blob1 = receive_any(), + ?line Blob1 = Blob0, + clear_msgenv(MsgEnv), + repeat(50, fun(_) -> grow_blob(MsgEnv,other_term()) end, []), + ?line {ok,1,Blob2} = SendBlobF(MsgEnv, To), + ?line Blob3 = receive_any(), + ?line Blob3 = Blob2, + + clear_msgenv(MsgEnv), + repeat(50, fun(_) -> grow_blob(MsgEnv,other_term()) end, []), + + clear_msgenv(MsgEnv), + repeat(50, fun(_) -> grow_blob(MsgEnv,other_term()) end, []), + ?line {ok,1,Blob4} = SendBlobF(MsgEnv, To), + ?line Blob5 = receive_any(), + ?line Blob5 = Blob4, + + clear_msgenv(MsgEnv), + clear_msgenv(MsgEnv), + repeat(50, fun(_) -> grow_blob(MsgEnv,other_term()) end, []), + ?line {ok,1,Blob6} = SendBlobF(MsgEnv, To), + ?line Blob7 = receive_any(), + ?line Blob7 = Blob6, + + ok. + + +send_blob_thread_and_join(MsgEnv, To) -> + ?line {ok,Blob} = send_blob_thread_dbg(MsgEnv, To, no_join), + ?line {ok,SendRes} = join_send_thread(MsgEnv), + {ok,SendRes,Blob}. + +send_blob_dbg(MsgEnv, To) -> + Ret = send_blob(MsgEnv, To), + %%io:format("send_blob to ~p returned ~p\n",[To,Ret]), + Ret. + +send_blob_thread_dbg(MsgEnv, To, Join) -> + Ret = send_blob_thread(MsgEnv, To, Join), + %%io:format("send_blob_thread to ~p Join=~p returned ~p\n",[To,Join,Ret]), + Ret. + + +forwarder(To) -> + forwarder(To, 0). +forwarder(To, N) -> + case receive_any() of + terminate -> + To ! {self(), N}; + Msg -> + To ! Msg, + forwarder(To, N+1) + end. + +other_term() -> + {fun(X,Y) -> X*Y end, make_ref()}. + +send3(doc) -> ["Message sending stress test"]; +send3(Config) when is_list(Config) -> + %% Let a number of processes send random message blobs between each other + %% using enif_send. Kill and spawn new ones randomly to keep a ~constant + %% number of workers running. + Seed = now(), + io:format("seed: ~p\n",[Seed]), + random:seed(Seed), + ets:new(nif_SUITE,[named_table,public]), + ?line true = ets:insert(nif_SUITE,{send3,0,0,0,0}), + timer:send_after(10000, timeout), % Run for 10 seconds + SpawnCnt = send3_controller(0, [], [], 20), + ?line [{_,Rcv,SndOk,SndFail,Balance}] = ets:lookup(nif_SUITE,send3), + io:format("spawns=~p received=~p, sent=~p send-failure=~p balance=~p\n", + [SpawnCnt,Rcv,SndOk,SndFail,Balance]), + ets:delete(nif_SUITE). + +send3_controller(SpawnCnt, [], _, infinity) -> + SpawnCnt; +send3_controller(SpawnCnt0, Mons0, Pids0, Tick) -> + receive + timeout -> + io:format("Timeout. Sending 'halt' to ~p\n",[Pids0]), + lists:foreach(fun(P) -> P ! {halt,self()} end, Pids0), + lists:foreach(fun(P) -> receive {halted,P} -> ok end end, Pids0), + QTot = lists:foldl(fun(P,QSum) -> + {message_queue_len,QLen} = + erlang:process_info(P,message_queue_len), + QSum + QLen + end, 0, Pids0), + io:format("Total queue length ~p\n",[QTot]), + lists:foreach(fun(P) -> P ! die end, Pids0), + send3_controller(SpawnCnt0, Mons0, [], infinity); + {'DOWN', MonRef, process, _Pid, _} -> + Mons1 = lists:delete(MonRef, Mons0), + %%io:format("Got DOWN from ~p. Monitors left: ~p\n",[Pid,Mons1]), + send3_controller(SpawnCnt0, Mons1, Pids0, Tick) + after Tick -> + Max = 20, + N = length(Pids0), + PidN = random:uniform(Max), + %%io:format("N=~p PidN=~p Pids0=~p\n", [N,PidN,Pids0]), + case PidN > N of + true -> + {NewPid,Mon} = spawn_opt(fun send3_proc/0, [link,monitor]), + lists:foreach(fun(P) -> P ! {is_born,NewPid} end, Pids0), + ?line Balance = ets:lookup_element(nif_SUITE,send3,5), + Inject = (Balance =< 0), + case Inject of + true -> ok; + false -> ets:update_element(nif_SUITE,send3,{5,-1}) + end, + NewPid ! {pids,Pids0,Inject}, + send3_controller(SpawnCnt0+1, [Mon|Mons0], [NewPid|Pids0], Tick); + false -> + KillPid = lists:nth(PidN,Pids0), + KillPid ! die, + Pids1 = lists:delete(KillPid, Pids0), + lists:foreach(fun(P) -> P ! {is_dead,KillPid} end, Pids1), + send3_controller(SpawnCnt0, Mons0, Pids1, Tick) + end + end. + +send3_proc() -> + %%io:format("Process ~p spawned\n",[self()]), + send3_proc([self()], {0,0,0}, {1,2,3,4,5}). +send3_proc(Pids0, Counters={Rcv,SndOk,SndFail}, State0) -> + %%io:format("~p: Pids0=~p", [self(), Pids0]), + %%timer:sleep(10), + receive + {pids, Pids1, Inject} -> + %%io:format("~p: got ~p Inject=~p\n", [self(), Pids1, Inject]), + ?line Pids0 = [self()], + Pids2 = [self() | Pids1], + case Inject of + true -> send3_proc_send(Pids2, Counters, State0); + false -> send3_proc(Pids2, Counters, State0) + end; + {is_born, Pid} -> + %%io:format("~p: is_born ~p, got ~p\n", [self(), Pid, Pids0]), + send3_proc([Pid | Pids0], Counters, State0); + {is_dead, Pid} -> + Pids1 = lists:delete(Pid,Pids0), + %%io:format("~p: is_dead ~p, got ~p\n", [self(), Pid, Pids1]), + send3_proc(Pids1, Counters, State0); + {blob, Blob0} -> + %%io:format("~p: blob ~p\n", [self(), Blob0]), + State1 = send3_new_state(State0, Blob0), + send3_proc_send(Pids0, {Rcv+1,SndOk,SndFail}, State1); + die -> + %%io:format("Process ~p terminating, stats = ~p\n",[self(),Counters]), + {message_queue_len,Dropped} = erlang:process_info(self(),message_queue_len), + _R = ets:update_counter(nif_SUITE,send3, + [{2,Rcv},{3,SndOk},{4,SndFail},{5,1-Dropped}]), + %%io:format("~p: dies R=~p\n", [self(), R]), + ok; + {halt,Papa} -> + Papa ! {halted,self()}, + io:format("~p halted\n",[self()]), + receive die -> ok end, + io:format("~p dying\n",[self()]) + end. + +send3_proc_send(Pids, {Rcv,SndOk,SndFail}, State0) -> + To = lists:nth(random:uniform(length(Pids)),Pids), + Blob = send3_make_blob(), + State1 = send3_new_state(State0,Blob), + case send3_send(To, Blob) of + true -> + send3_proc(Pids, {Rcv,SndOk+1,SndFail}, State1); + false -> + send3_proc(Pids, {Rcv,SndOk,SndFail+1}, State1) + end. + + +send3_make_blob() -> + case random:uniform(20)-1 of + 0 -> {term,[]}; + N -> + MsgEnv = alloc_msgenv(), + repeat(N bsr 1, + fun(_) -> grow_blob(MsgEnv,other_term(),random:uniform(1 bsl 20)) + end, void), + case (N band 1) of + 0 -> {term,copy_blob(MsgEnv)}; + 1 -> {msgenv,MsgEnv} + end + end. + +send3_send(Pid, Msg) -> + %% 90% enif_send and 10% normal bang + case random:uniform(10) of + 1 -> send3_send_bang(Pid,Msg); + _ -> send3_send_nif(Pid,Msg) + end. +send3_send_nif(Pid, {term,Blob}) -> + %%io:format("~p send term nif\n",[self()]), + send_term(Pid, {blob, Blob}) =:= 1; +send3_send_nif(Pid, {msgenv,MsgEnv}) -> + %%io:format("~p send blob nif\n",[self()]), + send3_blob(MsgEnv, Pid, blob) =:= 1. + +send3_send_bang(Pid, {term,Blob}) -> + %%io:format("~p send term bang\n",[self()]), + Pid ! {blob, Blob}, + true; +send3_send_bang(Pid, {msgenv,MsgEnv}) -> + %%io:format("~p send blob bang\n",[self()]), + Pid ! {blob, copy_blob(MsgEnv)}, + true. + +send3_new_state(State, Blob) -> + case random:uniform(5+2) of + N when N =< 5-> setelement(N, State, Blob); + _ -> State % Don't store blob + end. + neg(doc) -> ["Negative testing of load_nif"]; neg(Config) when is_list(Config) -> TmpMem = tmpmem(), @@ -759,7 +1089,17 @@ neg(Config) when is_list(Config) -> ?line verify_tmpmem(TmpMem), ?line ok. +is_checks(doc) -> ["Test all enif_is functions"]; +is_checks(Config) when is_list(Config) -> + ?line ensure_lib_loaded(Config, 1), + ?line ok = check_is(hejsan, <<19,98>>, make_ref(), ok, fun() -> ok end, + self(), hd(erlang:ports()), [], [1,9,9,8], + {hejsan, "hejsan", [$h,"ejs",<<"an">>]}). +get_length(doc) -> ["Test all enif_get_length functions"]; +get_length(Config) when is_list(Config) -> + ?line ensure_lib_loaded(Config, 1), + ?line ok = length_test(hejsan, "hejsan", [], [], not_a_list). ensure_lib_loaded(Config) -> ensure_lib_loaded(Config, 1). @@ -773,6 +1113,23 @@ ensure_lib_loaded(Config, Ver) -> ok end. +make_atom(Config) when is_list(Config) -> + ?line ensure_lib_loaded(Config, 1), + An0Atom = an0atom, + An0Atom0 = 'an\000atom\000', + ?line Atoms = make_atoms(), + ?line 7 = size(Atoms), + ?line Atoms = {An0Atom,An0Atom,An0Atom,An0Atom0,An0Atom,An0Atom,An0Atom0}. + +make_string(Config) when is_list(Config) -> + ?line ensure_lib_loaded(Config, 1), + ?line Strings = make_strings(), + ?line 5 = size(Strings), + A0String = "a0string", + A0String0 = [$a,0,$s,$t,$r,$i,$n,$g,0], + AStringWithAccents = [$E,$r,$l,$a,$n,$g,$ ,16#e4,$r,$ ,$e,$t,$t,$ ,$g,$e,$n,$e,$r,$e,$l,$l,$t,$ ,$p,$r,$o,$g,$r,$a,$m,$s,$p,$r,16#e5,$k], + ?line Strings = {A0String,A0String,A0String,A0String0, AStringWithAccents}. + tmpmem() -> case erlang:system_info({allocator,temp_alloc}) of false -> undefined; @@ -821,13 +1178,18 @@ call(Pid,Cmd) -> receive_any() -> receive M -> M end. -%% check(Exp,Got,Line) -> -%% case Got of -%% Exp -> Exp; -%% _ -> -%% io:format("CHECK at ~p: Expected ~p but got ~p\n",[Line,Exp,Got]), -%% Got -%% end. +repeat(0, _, Arg) -> + Arg; +repeat(N, Fun, Arg0) -> + repeat(N-1, Fun, Fun(Arg0)). + +check(Exp,Got,Line) -> + case Got of + Exp -> Exp; + _ -> + io:format("CHECK at ~p: Expected ~p but got ~p\n",[Line,Exp,Got]), + Got + end. %% The NIFs: @@ -855,6 +1217,23 @@ get_resource(_,_) -> ?nif_stub. release_resource(_) -> ?nif_stub. last_resource_dtor_call() -> ?nif_stub. make_new_resource(_,_) -> ?nif_stub. +check_is(_,_,_,_,_,_,_,_,_,_) -> ?nif_stub. +length_test(_,_,_,_,_) -> ?nif_stub. +make_atoms() -> ?nif_stub. +make_strings() -> ?nif_stub. +make_new_resource_binary(_) -> ?nif_stub. +send_list_seq(_,_) -> ?nif_stub. +send_new_blob(_,_) -> ?nif_stub. +alloc_msgenv() -> ?nif_stub. +clear_msgenv(_) -> ?nif_stub. +grow_blob(_,_) -> ?nif_stub. +grow_blob(_,_,_) -> ?nif_stub. +send_blob(_,_) -> ?nif_stub. +send3_blob(_,_,_) -> ?nif_stub. +send_blob_thread(_,_,_) -> ?nif_stub. +join_send_thread(_) -> ?nif_stub. +copy_blob(_) -> ?nif_stub. +send_term(_,_) -> ?nif_stub. nif_stub_error(Line) -> exit({nif_not_loaded,module,?MODULE,line,Line}). diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c index 7d05a9a880..5384a32f21 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c +++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c @@ -1,3 +1,21 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ #include "erl_nif.h" #include <stdio.h> @@ -10,6 +28,12 @@ static int static_cntA; /* zero by default */ static int static_cntB = NIF_SUITE_LIB_VER * 100; +static ERL_NIF_TERM atom_self; +static ERL_NIF_TERM atom_ok; +static ERL_NIF_TERM atom_join; +static ERL_NIF_TERM atom_binary_resource_type; + + typedef struct { int ref_cnt; @@ -20,7 +44,7 @@ typedef struct void add_call(ErlNifEnv* env, PrivData* data, const char* func_name) { - CallInfo* call = enif_alloc(env, sizeof(CallInfo)+strlen(func_name)); + CallInfo* call = enif_alloc(sizeof(CallInfo)+strlen(func_name)); strcpy(call->func_name, func_name); call->lib_ver = NIF_SUITE_LIB_VER; call->next = data->call_history; @@ -31,7 +55,7 @@ void add_call(ErlNifEnv* env, PrivData* data, const char* func_name) call->arg_sz = 0; } -#define ADD_CALL(FUNC_NAME) add_call(env, enif_get_data(env),FUNC_NAME) +#define ADD_CALL(FUNC_NAME) add_call(env, enif_priv_data(env),FUNC_NAME) static void* resource_dtor_last = NULL; static unsigned resource_dtor_last_sz = 0; @@ -42,15 +66,24 @@ static void resource_dtor(ErlNifEnv* env, void* obj) { resource_dtor_last = obj; resource_dtor_cnt++; - resource_dtor_last_sz = enif_sizeof_resource(env, obj); + resource_dtor_last_sz = enif_sizeof_resource(obj); assert(resource_dtor_last_sz <= sizeof(resource_dtor_last_data)); memcpy(resource_dtor_last_data, obj, resource_dtor_last_sz); } +static ErlNifResourceType* msgenv_resource_type; +static void msgenv_dtor(ErlNifEnv* env, void* obj); + +static ErlNifResourceType* binary_resource_type; +static void binary_resource_dtor(ErlNifEnv* env, void* obj); +struct binary_resource { + unsigned char* data; + unsigned size; +}; + static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) { - /*ERL_NIF_TERM head, tail;*/ - PrivData* data = enif_alloc(env, sizeof(PrivData)); + PrivData* data = enif_alloc(sizeof(PrivData)); assert(data != NULL); data->ref_cnt = 1; data->call_history = NULL; @@ -58,41 +91,71 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) add_call(env, data, "load"); - /* - head = load_info; - data->rt_cnt = 0; - for (head=load_info; enif_get_list_cell(env,load_info,&head,&tail); - head=tail) { - char buf[20]; - int n = enif_get_string(env,head,buf,sizeof(buf)); - assert(n > 0); - assert(i < sizeof(data->rt_arr)/sizeof(*data->rt_arr)); - data->rt_arr[data->rt_cnt++].t = enif_create_resource_type(env,buf,resource_dtor, - ERL_NIF_RT_CREATE,NULL); - } - assert(enif_is_empty_list(env,head)); - */ - data->rt_arr[0].t = enif_open_resource_type(env,"Gold",resource_dtor, + data->rt_arr[0].t = enif_open_resource_type(env,NULL,"Gold",resource_dtor, ERL_NIF_RT_CREATE,NULL); - data->rt_arr[1].t = enif_open_resource_type(env,"Silver",resource_dtor, + data->rt_arr[1].t = enif_open_resource_type(env,NULL,"Silver",resource_dtor, ERL_NIF_RT_CREATE,NULL); + binary_resource_type = enif_open_resource_type(env,NULL,"nif_SUITE.binary", + binary_resource_dtor, + ERL_NIF_RT_CREATE, NULL); + + msgenv_resource_type = enif_open_resource_type(env,NULL,"nif_SUITE.msgenv", + msgenv_dtor, + ERL_NIF_RT_CREATE, NULL); + + atom_self = enif_make_atom(env,"self"); + atom_ok = enif_make_atom(env,"ok"); + atom_join = enif_make_atom(env,"join"); + atom_binary_resource_type = enif_make_atom(env,"binary_resource_type"); + *priv_data = data; return 0; } +static void resource_takeover(ErlNifEnv* env, PrivData* priv) +{ + ErlNifResourceFlags tried; + ErlNifResourceType* rt; + rt = enif_open_resource_type(env, NULL, "Gold", resource_dtor, + ERL_NIF_RT_TAKEOVER, &tried); + assert(rt == priv->rt_arr[0].t); + assert(tried == ERL_NIF_RT_TAKEOVER); + rt = enif_open_resource_type(env, NULL, "Silver", resource_dtor, + ERL_NIF_RT_TAKEOVER, &tried); + assert(rt == priv->rt_arr[1].t); + assert(tried == ERL_NIF_RT_TAKEOVER); + + rt = enif_open_resource_type(env, NULL, "nif_SUITE.binary", binary_resource_dtor, + ERL_NIF_RT_TAKEOVER, &tried); + assert(rt != NULL); + assert(tried == ERL_NIF_RT_TAKEOVER); + assert(binary_resource_type==NULL || binary_resource_type == rt); + binary_resource_type = rt; + + rt = enif_open_resource_type(env, NULL, "nif_SUITE.msgenv", msgenv_dtor, + ERL_NIF_RT_TAKEOVER, &tried); + assert(rt != NULL); + assert(tried == ERL_NIF_RT_TAKEOVER); + assert(msgenv_resource_type==NULL || msgenv_resource_type == rt); + msgenv_resource_type = rt; +} + static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) { - add_call(env, *priv_data, "reload"); + PrivData* priv = (PrivData*) *priv_data; + add_call(env, priv, "reload"); + resource_takeover(env,priv); return 0; } static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) { - PrivData* data = *old_priv_data; - add_call(env, data, "upgrade"); - data->ref_cnt++; - *priv_data = *old_priv_data; + PrivData* priv = (PrivData*) *old_priv_data; + add_call(env, priv, "upgrade"); + priv->ref_cnt++; + *priv_data = *old_priv_data; + resource_takeover(env,priv); return 0; } @@ -101,7 +164,10 @@ static void unload(ErlNifEnv* env, void* priv_data) PrivData* data = priv_data; add_call(env, data, "unload"); if (--data->ref_cnt == 0) { - enif_free(env, priv_data); + if (data->nif_mod != NULL) { + NifModPrivData_release(data->nif_mod); + } + enif_free(priv_data); } } @@ -120,11 +186,10 @@ static ERL_NIF_TERM make_call_history(ErlNifEnv* env, CallInfo** headp) ERL_NIF_TERM func_term = enif_make_atom(env,call->func_name); ERL_NIF_TERM tpl; if (call->arg != NULL) { - ErlNifBinary arg_bin; - enif_alloc_binary(env, call->arg_sz, &arg_bin); - memcpy(arg_bin.data, call->arg, call->arg_sz); - func_term = enif_make_tuple2(env, func_term, - enif_make_binary(env, &arg_bin)); + ERL_NIF_TERM arg_bin; + memcpy(enif_make_new_binary(env, call->arg_sz, &arg_bin), + call->arg, call->arg_sz); + func_term = enif_make_tuple2(env, func_term, arg_bin); } tpl = enif_make_tuple4(env, func_term, enif_make_int(env,call->lib_ver), @@ -132,28 +197,28 @@ static ERL_NIF_TERM make_call_history(ErlNifEnv* env, CallInfo** headp) enif_make_int(env,call->static_cntB)); list = enif_make_list_cell(env, tpl, list); *headp = call->next; - enif_free(env,call); + enif_free(call); } return list; } static ERL_NIF_TERM call_history(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - PrivData* data = (PrivData*) enif_get_data(env); + PrivData* data = (PrivData*) enif_priv_data(env); return make_call_history(env,&data->call_history); } static ERL_NIF_TERM hold_nif_mod_priv_data(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - PrivData* data = (PrivData*) enif_get_data(env); + PrivData* data = (PrivData*) enif_priv_data(env); unsigned long ptr_as_ulong; if (!enif_get_ulong(env,argv[0],&ptr_as_ulong)) { return enif_make_badarg(env); } - if (data->nif_mod != NULL && --(data->nif_mod->ref_cnt) == 0) { - enif_free(env,data->nif_mod); + if (data->nif_mod != NULL) { + NifModPrivData_release(data->nif_mod); } data->nif_mod = (NifModPrivData*) ptr_as_ulong; return enif_make_int(env,++(data->nif_mod->ref_cnt)); @@ -161,7 +226,7 @@ static ERL_NIF_TERM hold_nif_mod_priv_data(ErlNifEnv* env, int argc, const ERL_N static ERL_NIF_TERM nif_mod_call_history(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - PrivData* data = (PrivData*) enif_get_data(env); + PrivData* data = (PrivData*) enif_priv_data(env); ERL_NIF_TERM ret; if (data->nif_mod == NULL) { return enif_make_string(env,"nif_mod pointer is NULL", ERL_NIF_LATIN1); @@ -231,6 +296,30 @@ static int test_ulong(ErlNifEnv* env, unsigned long i1) return 1; } +static int test_int64(ErlNifEnv* env, ErlNifSInt64 i1) +{ + ErlNifSInt64 i2 = 0; + ERL_NIF_TERM int_term = enif_make_int64(env, i1); + if (!enif_get_int64(env,int_term, &i2) || i1 != i2) { + fprintf(stderr, "test_int64(%ld) ...FAILED i2=%ld\r\n", + (long)i1, (long)i2); + return 0; + } + return 1; +} + +static int test_uint64(ErlNifEnv* env, ErlNifUInt64 i1) +{ + ErlNifUInt64 i2 = 0; + ERL_NIF_TERM int_term = enif_make_uint64(env, i1); + if (!enif_get_uint64(env,int_term, &i2) || i1 != i2) { + fprintf(stderr, "test_ulong(%lu) ...FAILED i2=%lu\r\n", + (unsigned long)i1, (unsigned long)i2); + return 0; + } + return 1; +} + static int test_double(ErlNifEnv* env, double d1) { double d2 = 0; @@ -254,6 +343,8 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ unsigned uint; long slong; unsigned long ulong; + ErlNifSInt64 sint64; + ErlNifUInt64 uint64; double d; ERL_NIF_TERM atom, ref1, ref2; @@ -287,11 +378,25 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ slong -= slong / 3 + 1; } while (slong >= 0); + sint64 = ((ErlNifSInt64)1 << 63); /* INT64_MIN */ + do { + if (!test_int64(env,sint64)) { + goto error; + } + sint64 += ~sint64 / 3 + 1; + } while (sint64 < 0); + sint64 = ((ErlNifUInt64)1 << 63) - 1; /* INT64_MAX */ + do { + if (!test_int64(env,sint64)) { + goto error; + } + sint64 -= sint64 / 3 + 1; + } while (sint64 >= 0); uint = UINT_MAX; for (;;) { if (!test_uint(env,uint)) { - + goto error; } if (uint == 0) break; uint -= uint / 3 + 1; @@ -299,11 +404,19 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ ulong = ULONG_MAX; for (;;) { if (!test_ulong(env,ulong)) { - + goto error; } if (ulong == 0) break; ulong -= ulong / 3 + 1; } + uint64 = (ErlNifUInt64)-1; /* UINT64_MAX */ + for (;;) { + if (!test_uint64(env,uint64)) { + goto error; + } + if (uint64 == 0) break; + uint64 -= uint64 / 3 + 1; + } if (MAX_SMALL < INT_MAX) { /* 32-bit */ for (i=-10 ; i <= 10; i++) { @@ -326,24 +439,31 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ for (i=-10 ; i < 10; i++) { if (!test_long(env,MAX_SMALL+i) || !test_ulong(env,MAX_SMALL+i) || - !test_long(env,MIN_SMALL+i)) { + !test_long(env,MIN_SMALL+i) || + !test_int64(env,MAX_SMALL+i) || !test_uint64(env,MAX_SMALL+i) || + !test_int64(env,MIN_SMALL+i)) { goto error; } + if (MAX_SMALL < INT_MAX) { + if (!test_int(env,MAX_SMALL+i) || !test_uint(env,MAX_SMALL+i) || + !test_int(env,MIN_SMALL+i)) { + goto error; + } + } } - for (d=3.141592e-100 ; d < 1e100 ; d *= 9.97) { if (!test_double(env,d) || !test_double(env,-d)) { goto error; } } - if (!enif_make_existing_atom(env,"nif_SUITE", &atom) - || !enif_is_identical(env,atom,enif_make_atom(env,"nif_SUITE"))) { + if (!enif_make_existing_atom(env,"nif_SUITE", &atom, ERL_NIF_LATIN1) + || !enif_is_identical(atom,enif_make_atom(env,"nif_SUITE"))) { fprintf(stderr, "nif_SUITE not an atom?\r\n"); goto error; } for (i=2; i; i--) { - if (enif_make_existing_atom(env,"nif_SUITE_pink_unicorn", &atom)) { + if (enif_make_existing_atom(env,"nif_SUITE_pink_unicorn", &atom, ERL_NIF_LATIN1)) { fprintf(stderr, "pink unicorn exist?\r\n"); goto error; } @@ -351,7 +471,7 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ ref1 = enif_make_ref(env); ref2 = enif_make_ref(env); if (!enif_is_ref(env,ref1) || !enif_is_ref(env,ref2) - || enif_is_identical(env,ref1,ref2) || enif_compare(env,ref1,ref2)==0) { + || enif_is_identical(ref1,ref2) || enif_compare(ref1,ref2)==0) { fprintf(stderr, "strange refs?\r\n"); goto error; } @@ -381,7 +501,7 @@ static ERL_NIF_TERM is_identical(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar if (argc != 2) { return enif_make_badarg(env); } - return enif_make_atom(env, (enif_is_identical(env,argv[0],argv[1]) ? + return enif_make_atom(env, (enif_is_identical(argv[0],argv[1]) ? "true" : "false")); } @@ -390,7 +510,7 @@ static ERL_NIF_TERM compare(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) if (argc != 2) { return enif_make_badarg(env); } - return enif_make_int(env, enif_compare(env,argv[0],argv[1])); + return enif_make_int(env, enif_compare(argv[0],argv[1])); } static ERL_NIF_TERM many_args_100(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) @@ -412,11 +532,10 @@ static ERL_NIF_TERM clone_bin(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ { ErlNifBinary ibin; if (enif_inspect_binary(env,argv[0],&ibin)) { - ErlNifBinary obin; - enif_alloc_binary(env,ibin.size,&obin); - memcpy(obin.data,ibin.data,ibin.size); - /*enif_release_binary(env,&ibin);*/ - return enif_make_binary(env,&obin); + ERL_NIF_TERM obin; + memcpy(enif_make_new_binary(env, ibin.size, &obin), + ibin.data, ibin.size); + return obin; } else { return enif_make_badarg(env); @@ -438,7 +557,7 @@ static ERL_NIF_TERM string_to_bin(ErlNifEnv* env, int argc, const ERL_NIF_TERM a unsigned size; int n; if (!enif_get_int(env,argv[1],(int*)&size) - || !enif_alloc_binary(env,size,&obin)) { + || !enif_alloc_binary(size,&obin)) { return enif_make_badarg(env); } n = enif_get_string(env, argv[0], (char*)obin.data, size, ERL_NIF_LATIN1); @@ -452,10 +571,10 @@ static ERL_NIF_TERM atom_to_bin(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg unsigned size; int n; if (!enif_get_int(env,argv[1],(int*)&size) - || !enif_alloc_binary(env,size,&obin)) { + || !enif_alloc_binary(size,&obin)) { return enif_make_badarg(env); } - n = enif_get_atom(env, argv[0], (char*)obin.data, size); + n = enif_get_atom(env, argv[0], (char*)obin.data, size, ERL_NIF_LATIN1); return enif_make_tuple(env, 2, enif_make_int(env,n), enif_make_binary(env,&obin)); } @@ -515,14 +634,14 @@ static ERL_NIF_TERM iolist_2_bin(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar static ERL_NIF_TERM last_resource_dtor_call(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - ErlNifBinary bin; ERL_NIF_TERM ret; if (resource_dtor_last != NULL) { - enif_alloc_binary(env, resource_dtor_last_sz, &bin); - memcpy(bin.data, resource_dtor_last_data, resource_dtor_last_sz); + ERL_NIF_TERM bin; + memcpy(enif_make_new_binary(env, resource_dtor_last_sz, &bin), + resource_dtor_last_data, resource_dtor_last_sz); ret = enif_make_tuple3(env, enif_make_long(env, (long)resource_dtor_last), - enif_make_binary(env, &bin), + bin, enif_make_int(env, resource_dtor_cnt)); } else { @@ -536,7 +655,7 @@ static ERL_NIF_TERM last_resource_dtor_call(ErlNifEnv* env, int argc, const ERL_ static ERL_NIF_TERM get_resource_type(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - PrivData* data = (PrivData*) enif_get_data(env); + PrivData* data = (PrivData*) enif_priv_data(env); int ix; if (!enif_get_int(env, argv[0], &ix) || ix >= 2) { @@ -552,7 +671,7 @@ static ERL_NIF_TERM alloc_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM union { void* p; long l;} data; if (!enif_get_long(env, argv[0], &type.l) || !enif_inspect_binary(env, argv[1], &data_bin) - || (data.p = enif_alloc_resource(env, type.t, data_bin.size))==NULL) { + || (data.p = enif_alloc_resource(type.t, data_bin.size))==NULL) { return enif_make_badarg(env); } @@ -577,28 +696,65 @@ static ERL_NIF_TERM make_new_resource(ErlNifEnv* env, int argc, const ERL_NIF_TE ERL_NIF_TERM ret; if (!enif_get_long(env, argv[0], &type.l) || !enif_inspect_binary(env, argv[1], &data_bin) - || (data = enif_alloc_resource(env, type.t, data_bin.size))==NULL) { + || (data = enif_alloc_resource(type.t, data_bin.size))==NULL) { return enif_make_badarg(env); } ret = enif_make_resource(env, data); memcpy(data, data_bin.data, data_bin.size); - enif_release_resource(env, data); + enif_release_resource(data); return ret; } +static ERL_NIF_TERM make_new_resource_binary(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ErlNifBinary data_bin; + union { struct binary_resource* p; void* vp; long l;} br; + void* buf; + ERL_NIF_TERM ret; + if (!enif_inspect_binary(env, argv[0], &data_bin) + || (br.vp = enif_alloc_resource(binary_resource_type, + sizeof(struct binary_resource)))==NULL + || (buf = enif_alloc(data_bin.size)) == NULL) { + + return enif_make_badarg(env); + } + memset(br.vp,0xba,sizeof(struct binary_resource)); /* avoid valgrind warning */ + br.p->data = buf; + br.p->size = data_bin.size; + memcpy(br.p->data, data_bin.data, data_bin.size); + ret = enif_make_resource_binary(env, br.vp, br.p->data, br.p->size); + enif_release_resource(br.p); + return enif_make_tuple2(env, enif_make_long(env,br.l), ret); +} + +static void binary_resource_dtor(ErlNifEnv* env, void* obj) +{ + struct binary_resource* br = (struct binary_resource*) obj; + resource_dtor(env,obj); + assert(br->data != NULL); + enif_free(br->data); + br->data = NULL; +} + static ERL_NIF_TERM get_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { ErlNifBinary data_bin; union { ErlNifResourceType* t; long l; } type; union { void* p; long l; } data; - if (!enif_get_long(env, argv[0], &type.l) + type.t = NULL; + if (enif_is_identical(argv[0], atom_binary_resource_type)) { + type.t = binary_resource_type; + } + else { + enif_get_long(env, argv[0], &type.l); + } + if (type.t == NULL || !enif_get_resource(env, argv[1], type.t, &data.p)) { return enif_make_badarg(env); } - - enif_alloc_binary(env, enif_sizeof_resource(env,data.p), &data_bin); + enif_alloc_binary(enif_sizeof_resource(data.p), &data_bin); memcpy(data_bin.data, data.p, data_bin.size); return enif_make_tuple2(env, enif_make_long(env,data.l), enif_make_binary(env, &data_bin)); @@ -610,10 +766,592 @@ static ERL_NIF_TERM release_resource(ErlNifEnv* env, int argc, const ERL_NIF_TER if (!enif_get_long(env, argv[0], &data.l)) { return enif_make_badarg(env); } - enif_release_resource(env, data.p); + enif_release_resource(data.p); return enif_make_atom(env,"ok"); } +/* + * argv[0] an atom + * argv[1] a binary + * argv[2] a ref + * argv[3] 'ok' + * argv[4] a fun + * argv[5] a pid + * argv[6] a port + * argv[7] an empty list + * argv[8] a non-empty list + * argv[9] a tuple + */ +static ERL_NIF_TERM check_is(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ERL_NIF_TERM ok_atom = enif_make_atom(env, "ok"); + + if (!enif_is_atom(env, argv[0])) return enif_make_badarg(env); + if (!enif_is_binary(env, argv[1])) return enif_make_badarg(env); + if (!enif_is_ref(env, argv[2])) return enif_make_badarg(env); + if (!enif_is_identical(argv[3], ok_atom)) return enif_make_badarg(env); + if (!enif_is_fun(env, argv[4])) return enif_make_badarg(env); + if (!enif_is_pid(env, argv[5])) return enif_make_badarg(env); + if (!enif_is_port(env, argv[6])) return enif_make_badarg(env); + if (!enif_is_empty_list(env, argv[7])) return enif_make_badarg(env); + if (!enif_is_list(env, argv[7])) return enif_make_badarg(env); + if (!enif_is_list(env, argv[8])) return enif_make_badarg(env); + if (!enif_is_tuple(env, argv[9])) return enif_make_badarg(env); + + return ok_atom; +} + +/* + * argv[0] atom with length of 6 + * argv[1] list with length of 6 + * argv[2] empty list + * argv[3] not an atom + * argv[4] not a list + */ +static ERL_NIF_TERM length_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + unsigned len; + + if (!enif_get_atom_length(env, argv[0], &len, ERL_NIF_LATIN1) || len != 6) + return enif_make_badarg(env); + + if (!enif_get_list_length(env, argv[1], &len) || len != 6) + return enif_make_badarg(env); + + if (!enif_get_list_length(env, argv[2], &len) || len != 0) + return enif_make_badarg(env); + + if (enif_get_atom_length(env, argv[3], &len, ERL_NIF_LATIN1)) + return enif_make_badarg(env); + + if (enif_get_list_length(env, argv[4], &len)) + return enif_make_badarg(env); + + return enif_make_atom(env, "ok"); +} + +static ERL_NIF_TERM make_atoms(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ERL_NIF_TERM arr[7]; + ERL_NIF_TERM existingatom0a, existingatom0b; + ERL_NIF_TERM existing0atom0; + const char * const an0atom = "an0atom"; + const char an0atom0[8] = {'a','n','\0','a','t','o','m',0}; + + arr[0] = enif_make_atom(env, "an0atom"); + arr[1] = enif_make_atom_len(env, "an0atom", 7); + arr[2] = enif_make_atom_len(env, an0atom, 7); + arr[3] = enif_make_atom_len(env, an0atom0, 8); + + if (!enif_make_existing_atom(env, "an0atom", &existingatom0a, ERL_NIF_LATIN1)) + return enif_make_atom(env, "error"); + arr[4] = existingatom0a; + + if (!enif_make_existing_atom_len(env, an0atom, 7, &existingatom0b, ERL_NIF_LATIN1)) + return enif_make_atom(env, "error"); + arr[5] = existingatom0b; + + if (!enif_make_existing_atom_len(env, an0atom0, 8, &existing0atom0, ERL_NIF_LATIN1)) + return enif_make_atom(env, "error"); + arr[6] = existing0atom0; + + return enif_make_tuple7(env, + arr[0],arr[1],arr[2],arr[3],arr[4],arr[5],arr[6]); +} + +static ERL_NIF_TERM make_strings(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + const char a0string[8] = {'a','0','s','t','r','i','n','g'}; + const char a0string0[9] = {'a','\0','s','t','r','i','n','g',0}; + const char astringwith8bits[37] = {'E','r','l','a','n','g',' ',0xE4 /* 'ä' */,'r',' ','e','t','t',' ','g','e','n','e','r','e','l','l','t',' ','p','r','o','g','r','a','m','s','p','r', 0xE5 /* 'å' */,'k',0}; + + return enif_make_tuple5(env, + enif_make_string(env, "a0string", ERL_NIF_LATIN1), + enif_make_string_len(env, "a0string", 8, ERL_NIF_LATIN1), + enif_make_string_len(env, a0string, 8, ERL_NIF_LATIN1), + enif_make_string_len(env, a0string0, 9, ERL_NIF_LATIN1), + enif_make_string(env, astringwith8bits, ERL_NIF_LATIN1)); +} +static ERL_NIF_TERM send_list_seq(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ErlNifPid to; + ERL_NIF_TERM msg; + ErlNifEnv* msg_env; + int i, res; + + if (!enif_get_int(env, argv[0], &i)) { + return enif_make_badarg(env); + } + if (argv[1] == atom_self) { + enif_self(env, &to); + } + else if (!enif_get_local_pid(env, argv[1], &to)) { + return enif_make_badarg(env); + } + msg_env = enif_alloc_env(); + msg = enif_make_list(msg_env,0); + for ( ; i>0 ; i--) { + msg = enif_make_list_cell(msg_env, enif_make_int(msg_env, i), msg); + } + res = enif_send(env, &to, msg_env, msg); + enif_free_env(msg_env); + return enif_make_tuple2(env, atom_ok, enif_make_int(env,res)); +} + +static void fill(void* dst, unsigned bytes, int seed) +{ + unsigned char* ptr = dst; + int i; + for (i=bytes; i>0; i--) { + *ptr++ = seed; + seed += 7; + } +} + +#define MAKE_TERM_REUSE_LEN 16 +struct make_term_info +{ + ErlNifEnv* caller_env; + ErlNifEnv* dst_env; + ERL_NIF_TERM reuse[MAKE_TERM_REUSE_LEN]; + unsigned reuse_push; + unsigned reuse_pull; + ErlNifResourceType* resource_type; + ERL_NIF_TERM other_term; + ERL_NIF_TERM blob; + ErlNifPid to_pid; + ErlNifTid tid; + ErlNifCond* cond; + ErlNifMutex* mtx; + int send_it; + int send_res; + unsigned n; +}; + + +static void push_term(struct make_term_info* mti, ERL_NIF_TERM term) +{ + unsigned ix = (mti->reuse_push++) % MAKE_TERM_REUSE_LEN; + mti->reuse[ix] = term; + //enif_fprintf(stderr, "push at %u: %T\r\n", ix, term); +} +static ERL_NIF_TERM pull_term(struct make_term_info* mti) +{ + unsigned ix; + if (mti->reuse_pull >= mti->reuse_push && + mti->reuse_push < MAKE_TERM_REUSE_LEN) { + mti->reuse_pull = 0; + if (mti->reuse_push == 0) { + mti->reuse[0] = enif_make_list(mti->dst_env, 0); + } + } + ix = (mti->reuse_pull++) % MAKE_TERM_REUSE_LEN; + //enif_fprintf(stderr, "pull from %u: %T\r\n", ix, mti->reuse[ix]); + return mti->reuse[ix]; +} + +static int make_term_n(struct make_term_info* mti, int n, ERL_NIF_TERM* res); + +static ERL_NIF_TERM make_term_binary(struct make_term_info* mti, int n) +{ + ErlNifBinary bin; + enif_alloc_binary(100, &bin); + fill(bin.data, bin.size, n); + return enif_make_binary(mti->dst_env, &bin); +} + +static ERL_NIF_TERM make_term_int(struct make_term_info* mti, int n) +{ + int i; + fill(&i, sizeof(i), n); + return enif_make_int(mti->dst_env, i); +} + +static ERL_NIF_TERM make_term_ulong(struct make_term_info* mti, int n) +{ + unsigned long ul; + fill(&ul, sizeof(ul), n); + return enif_make_ulong(mti->dst_env, ul); +} + +static ERL_NIF_TERM make_term_double(struct make_term_info* mti, int n) +{ + double d = 3.141592; + return enif_make_double(mti->dst_env, d); +} +static ERL_NIF_TERM make_term_atom(struct make_term_info* mti, int n) +{ + return enif_make_atom(mti->dst_env, "make_term_n"); +} +static ERL_NIF_TERM make_term_existing_atom(struct make_term_info* mti, int n) +{ + ERL_NIF_TERM res; + int exist = enif_make_existing_atom(mti->dst_env, "nif_SUITE", &res, + ERL_NIF_LATIN1); + assert(exist); + return res; +} +static ERL_NIF_TERM make_term_string(struct make_term_info* mti, int n) +{ + return enif_make_string(mti->dst_env, "Hello!", ERL_NIF_LATIN1); +} +static ERL_NIF_TERM make_term_ref(struct make_term_info* mti, int n) +{ + return enif_make_ref(mti->dst_env); +} +static ERL_NIF_TERM make_term_sub_binary(struct make_term_info* mti, int n) +{ + ERL_NIF_TERM orig; + unsigned char* ptr = enif_make_new_binary(mti->dst_env, 10, &orig); + fill(ptr, 10, n); + return enif_make_sub_binary(mti->dst_env, orig, 3, 5); +} +static ERL_NIF_TERM make_term_uint(struct make_term_info* mti, int n) +{ + unsigned int ui; + fill(&ui, sizeof(ui), n); + return enif_make_uint(mti->dst_env, ui); +} +static ERL_NIF_TERM make_term_long(struct make_term_info* mti, int n) +{ + long l; + fill(&l, sizeof(l), n); + return enif_make_long(mti->dst_env, l); +} +static ERL_NIF_TERM make_term_tuple0(struct make_term_info* mti, int n) +{ + return enif_make_tuple(mti->dst_env, 0); +} +static ERL_NIF_TERM make_term_list0(struct make_term_info* mti, int n) +{ + return enif_make_list(mti->dst_env, 0); +} +static ERL_NIF_TERM make_term_resource(struct make_term_info* mti, int n) +{ + void* resource = enif_alloc_resource(mti->resource_type, 10); + fill(resource, 10, n); + return enif_make_resource(mti->dst_env, resource); +} +static ERL_NIF_TERM make_term_new_binary(struct make_term_info* mti, int n) +{ + ERL_NIF_TERM res; + unsigned char* ptr = enif_make_new_binary(mti->dst_env,20,&res); + fill(ptr, 20, n); + return res; +} +static ERL_NIF_TERM make_term_caller_pid(struct make_term_info* mti, int n) +{ + ErlNifPid pid; + return enif_make_pid(mti->dst_env, enif_self(mti->caller_env, &pid)); +} + +static ERL_NIF_TERM make_term_tuple(struct make_term_info* mti, int n) +{ + ERL_NIF_TERM t[3]; + t[0] = pull_term(mti); + t[1] = pull_term(mti); + t[2] = pull_term(mti); + return enif_make_tuple3(mti->dst_env, t[0], t[1], t[2]); +} +static ERL_NIF_TERM make_term_list(struct make_term_info* mti, int n) +{ + ERL_NIF_TERM t[3]; + t[0] = pull_term(mti); + t[1] = pull_term(mti); + t[2] = pull_term(mti); + return enif_make_list3(mti->dst_env, t[0], t[1], t[2]); +} +static ERL_NIF_TERM make_term_list_cell(struct make_term_info* mti, int n) +{ + ERL_NIF_TERM t[2]; + t[0] = pull_term(mti); + t[1] = pull_term(mti); + return enif_make_list_cell(mti->dst_env, t[0], t[1]); +} +static ERL_NIF_TERM make_term_tuple_from_array(struct make_term_info* mti, int n) +{ + ERL_NIF_TERM t[3]; + t[0] = pull_term(mti); + t[1] = pull_term(mti); + t[2] = pull_term(mti); + return enif_make_tuple_from_array(mti->dst_env, t, 3); +} +static ERL_NIF_TERM make_term_list_from_array(struct make_term_info* mti, int n) +{ + ERL_NIF_TERM t[3]; + t[0] = pull_term(mti); + t[1] = pull_term(mti); + t[2] = pull_term(mti); + return enif_make_list_from_array(mti->dst_env, t, 3); +} +static ERL_NIF_TERM make_term_garbage(struct make_term_info* mti, int n) +{ + (void) enif_make_string(mti->dst_env, "garbage string", ERL_NIF_LATIN1); + return pull_term(mti); +} +static ERL_NIF_TERM make_term_copy(struct make_term_info* mti, int n) +{ + return enif_make_copy(mti->dst_env, mti->other_term); +} + +typedef ERL_NIF_TERM Make_term_Func(struct make_term_info*, int); +static Make_term_Func* make_funcs[] = { + make_term_binary, + make_term_int, + make_term_ulong, + make_term_double, + make_term_atom, + make_term_existing_atom, + make_term_string, + //make_term_ref, + make_term_sub_binary, + make_term_uint, + make_term_long, + make_term_tuple0, + make_term_list0, + make_term_resource, + make_term_new_binary, + make_term_caller_pid, + make_term_tuple, + make_term_list, + make_term_list_cell, + make_term_tuple_from_array, + make_term_list_from_array, + make_term_garbage, + make_term_copy +}; +static unsigned num_of_make_funcs() +{ + return sizeof(make_funcs)/sizeof(*make_funcs); +} +static int make_term_n(struct make_term_info* mti, int n, ERL_NIF_TERM* res) +{ + if (n < num_of_make_funcs()) { + *res = make_funcs[n](mti, n); + push_term(mti, *res); + return 1; + } + return 0; +} + +static ERL_NIF_TERM make_blob(ErlNifEnv* caller_env, ErlNifEnv* dst_env, + ERL_NIF_TERM other_term) +{ + PrivData* priv = (PrivData*) enif_priv_data(caller_env); + ERL_NIF_TERM term, list; + int n = 0; + struct make_term_info mti; + mti.caller_env = caller_env; + mti.dst_env = dst_env; + mti.reuse_push = 0; + mti.reuse_pull = 0; + mti.resource_type = priv->rt_arr[0].t; + mti.other_term = other_term; + + list = enif_make_list(dst_env, 0); + while (make_term_n(&mti, n++, &term)) { + list = enif_make_list_cell(dst_env, term, list); + } + return list; +} + +static ERL_NIF_TERM send_new_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ErlNifPid to; + ERL_NIF_TERM msg, copy; + ErlNifEnv* msg_env; + int res; + + if (!enif_get_local_pid(env, argv[0], &to)) { + return enif_make_badarg(env); + } + msg_env = enif_alloc_env(); + msg = make_blob(env,msg_env, argv[1]); + copy = make_blob(env,env, argv[1]); + res = enif_send(env, &to, msg_env, msg); + enif_free_env(msg_env); + return enif_make_tuple3(env, atom_ok, enif_make_int(env,res), copy); +} + +static ERL_NIF_TERM alloc_msgenv(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + PrivData* priv = (PrivData*) enif_priv_data(env); + struct make_term_info* mti; + ERL_NIF_TERM ret; + + mti = (struct make_term_info*) enif_alloc_resource(msgenv_resource_type, + sizeof(*mti)); + mti->caller_env = NULL; + mti->dst_env = enif_alloc_env(); + mti->reuse_push = 0; + mti->reuse_pull = 0; + mti->resource_type = priv->rt_arr[0].t; + mti->other_term = enif_make_list(mti->dst_env, 0); + mti->blob = enif_make_list(mti->dst_env, 0); + mti->mtx = enif_mutex_create("nif_SUITE:mtx"); + mti->cond = enif_cond_create("nif_SUITE:cond"); + mti->send_res = 0xcafebabe; + mti->n = 0; + ret = enif_make_resource(env, mti); + enif_release_resource(mti); + return ret; +} + +static void msgenv_dtor(ErlNifEnv* env, void* obj) +{ + struct make_term_info* mti = (struct make_term_info*) obj; + if (mti->dst_env != NULL) { + enif_free_env(mti->dst_env); + } + enif_mutex_destroy(mti->mtx); + enif_cond_destroy(mti->cond); +} + +static ERL_NIF_TERM clear_msgenv(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + union { + void* vp; + struct make_term_info* p; + }mti; + if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)) { + return enif_make_badarg(env); + } + enif_clear_env(mti.p->dst_env); + mti.p->reuse_pull = 0; + mti.p->reuse_push = 0; + mti.p->blob = enif_make_list(mti.p->dst_env, 0); + return atom_ok; +} + +static ERL_NIF_TERM grow_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + union { void* vp; struct make_term_info* p; }mti; + ERL_NIF_TERM term; + if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp) + || (argc>2 && !enif_get_uint(env,argv[2], &mti.p->n))) { + return enif_make_badarg(env); + } + mti.p->caller_env = env; + mti.p->other_term = argv[1]; + mti.p->n %= num_of_make_funcs(); + make_term_n(mti.p, mti.p->n++, &term); + mti.p->blob = enif_make_list_cell(mti.p->dst_env, term, mti.p->blob); + return atom_ok; +} + +static ERL_NIF_TERM send_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + union { void* vp; struct make_term_info* p; }mti; + ErlNifPid to; + ERL_NIF_TERM copy; + int res; + if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp) + || !enif_get_local_pid(env, argv[1], &to)) { + return enif_make_badarg(env); + } + copy = enif_make_copy(env, mti.p->blob); + res = enif_send(env, &to, mti.p->dst_env, mti.p->blob); + return enif_make_tuple3(env, atom_ok, enif_make_int(env,res), copy); +} + +static ERL_NIF_TERM send3_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + union { void* vp; struct make_term_info* p; }mti; + ErlNifPid to; + ERL_NIF_TERM copy; + int res; + if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp) + || !enif_get_local_pid(env, argv[1], &to)) { + return enif_make_badarg(env); + } + mti.p->blob = enif_make_tuple2(mti.p->dst_env, + enif_make_copy(mti.p->dst_env, argv[2]), + mti.p->blob); + res = enif_send(env, &to, mti.p->dst_env, mti.p->blob); + return enif_make_int(env,res); +} + +void* threaded_sender(void *arg) +{ + + union { void* vp; struct make_term_info* p; }mti; + mti.vp = arg; + + enif_mutex_lock(mti.p->mtx); + while (!mti.p->send_it) { + enif_cond_wait(mti.p->cond, mti.p->mtx); + } + mti.p->send_it = 0; + enif_mutex_unlock(mti.p->mtx); + mti.p->send_res = enif_send(NULL, &mti.p->to_pid, mti.p->dst_env, mti.p->blob); + return NULL; +} + +static ERL_NIF_TERM send_blob_thread(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + union { void* vp; struct make_term_info* p; }mti; + ERL_NIF_TERM copy; + if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp) + || !enif_get_local_pid(env,argv[1], &mti.p->to_pid)) { + return enif_make_badarg(env); + } + copy = enif_make_copy(env, mti.p->blob); + + mti.p->send_it = enif_is_identical(argv[2],atom_join); + if (enif_thread_create("nif_SUITE:send_from_thread", &mti.p->tid, + threaded_sender, mti.p, NULL) != 0) { + return enif_make_badarg(env); + } + if (enif_is_identical(argv[2],atom_join)) { + int err = enif_thread_join(mti.p->tid, NULL); + assert(err == 0); + return enif_make_tuple3(env, atom_ok, enif_make_int(env, mti.p->send_res), copy); + } + else { + enif_keep_resource(mti.vp); + return enif_make_tuple2(env, atom_ok, copy); + } +} + +static ERL_NIF_TERM join_send_thread(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + union { void* vp; struct make_term_info* p; }mti; + int err; + if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)) { + return enif_make_badarg(env); + } + enif_mutex_lock(mti.p->mtx); + mti.p->send_it = 1; + enif_cond_signal(mti.p->cond); + enif_mutex_unlock(mti.p->mtx); + err = enif_thread_join(mti.p->tid, NULL); + assert(err == 0); + enif_release_resource(mti.vp); + return enif_make_tuple2(env, atom_ok, enif_make_int(env, mti.p->send_res)); +} + +static ERL_NIF_TERM copy_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + union { void* vp; struct make_term_info* p; }mti; + if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)) { + return enif_make_badarg(env); + } + return enif_make_copy(env, mti.p->blob); +} + +static ERL_NIF_TERM send_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ErlNifEnv* menv; + ErlNifPid pid; + int ret; + if (!enif_get_local_pid(env, argv[0], &pid)) { + return enif_make_badarg(env); + } + menv = enif_alloc_env(); + ret = enif_send(env, &pid, menv, enif_make_copy(menv, argv[1])); + enif_free_env(menv); + return enif_make_int(env, ret); +} static ErlNifFunc nif_funcs[] = { @@ -640,8 +1378,25 @@ static ErlNifFunc nif_funcs[] = {"get_resource", 2, get_resource}, {"release_resource", 1, release_resource}, {"last_resource_dtor_call", 0, last_resource_dtor_call}, - {"make_new_resource", 2, make_new_resource} - + {"make_new_resource", 2, make_new_resource}, + {"check_is", 10, check_is}, + {"length_test", 5, length_test}, + {"make_atoms", 0, make_atoms}, + {"make_strings", 0, make_strings}, + {"make_new_resource", 2, make_new_resource}, + {"make_new_resource_binary", 1, make_new_resource_binary}, + {"send_list_seq", 2, send_list_seq}, + {"send_new_blob", 2, send_new_blob}, + {"alloc_msgenv", 0, alloc_msgenv}, + {"clear_msgenv", 1, clear_msgenv}, + {"grow_blob", 2, grow_blob}, + {"grow_blob", 3, grow_blob}, + {"send_blob", 2, send_blob}, + {"send3_blob", 3, send3_blob}, + {"send_blob_thread", 3, send_blob_thread}, + {"join_send_thread", 1, join_send_thread}, + {"copy_blob", 1, copy_blob}, + {"send_term", 2, send_term} }; ERL_NIF_INIT(nif_SUITE,nif_funcs,load,reload,upgrade,unload) diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.c b/erts/emulator/test/nif_SUITE_data/nif_mod.c index c075b74c57..e32d10057c 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_mod.c +++ b/erts/emulator/test/nif_SUITE_data/nif_mod.c @@ -1,3 +1,21 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ #include "erl_nif.h" #include <string.h> #include <stdio.h> @@ -24,6 +42,11 @@ static ERL_NIF_TERM am_resource_type; static ERL_NIF_TERM am_resource_dtor_A; static ERL_NIF_TERM am_resource_dtor_B; +static NifModPrivData* priv_data(ErlNifEnv* env) +{ + return (NifModPrivData*) enif_priv_data(env); +} + static void init(ErlNifEnv* env) { am_true = enif_make_atom(env, "true"); @@ -36,7 +59,7 @@ static void init(ErlNifEnv* env) static void add_call_with_arg(ErlNifEnv* env, NifModPrivData* data, const char* func_name, const char* arg, int arg_sz) { - CallInfo* call = enif_alloc(env, sizeof(CallInfo)+strlen(func_name) + arg_sz); + CallInfo* call = (CallInfo*)enif_alloc(sizeof(CallInfo)+strlen(func_name) + arg_sz); strcpy(call->func_name, func_name); call->lib_ver = NIF_LIB_VER; call->static_cntA = ++static_cntA; @@ -60,7 +83,7 @@ static void add_call(ErlNifEnv* env, NifModPrivData* data,const char* func_name) add_call_with_arg(env, data, func_name, NULL, 0); } -#define ADD_CALL(FUNC_NAME) add_call(env, enif_priv_data(env),FUNC_NAME) +#define ADD_CALL(FUNC_NAME) add_call(env, priv_data(env),FUNC_NAME) #define STRINGIFY_(X) #X #define STRINGIFY(X) STRINGIFY_(X) @@ -69,56 +92,56 @@ static void resource_dtor_A(ErlNifEnv* env, void* a) { const char dtor_name[] = "resource_dtor_A_v" STRINGIFY(NIF_LIB_VER); - add_call_with_arg(env, enif_priv_data(env), dtor_name, - a, enif_sizeof_resource(env, a)); + add_call_with_arg(env, priv_data(env), dtor_name, (const char*)a, + enif_sizeof_resource(a)); } static void resource_dtor_B(ErlNifEnv* env, void* a) { - const char dtor_name[] = "resource_dtor_B_v" STRINGIFY(NIF_LIB_VER); + const char dtor_name[] = "resource_dtor_B_v" STRINGIFY(NIF_LIB_VER); - add_call_with_arg(env, enif_priv_data(env), dtor_name, - a, enif_sizeof_resource(env, a)); + add_call_with_arg(env, priv_data(env), dtor_name, (const char*)a, + enif_sizeof_resource(a)); } /* {resource_type, Ix|null, ErlNifResourceFlags in, "TypeName", dtor(A|B|null), ErlNifResourceFlags out}*/ static void open_resource_type(ErlNifEnv* env, ERL_NIF_TERM op_tpl) { - NifModPrivData* data = enif_priv_data(env); + NifModPrivData* data = priv_data(env); const ERL_NIF_TERM* arr; int arity; char rt_name[30]; - union { enum ErlNifResourceFlags e; int i; } flags, exp_res, got_res; + union { ErlNifResourceFlags e; int i; } flags, exp_res, got_res; unsigned ix; ErlNifResourceDtor* dtor; ErlNifResourceType* got_ptr; CHECK(enif_get_tuple(env, op_tpl, &arity, &arr)); CHECK(arity == 6); - CHECK(enif_is_identical(env, arr[0], am_resource_type)); + CHECK(enif_is_identical(arr[0], am_resource_type)); CHECK(enif_get_int(env, arr[2], &flags.i)); CHECK(enif_get_string(env, arr[3], rt_name, sizeof(rt_name), ERL_NIF_LATIN1) > 0); CHECK(enif_get_int(env, arr[5], &exp_res.i)); - if (enif_is_identical(env, arr[4], am_null)) { + if (enif_is_identical(arr[4], am_null)) { dtor = NULL; } - else if (enif_is_identical(env, arr[4], am_resource_dtor_A)) { + else if (enif_is_identical(arr[4], am_resource_dtor_A)) { dtor = resource_dtor_A; } else { - CHECK(enif_is_identical(env, arr[4], am_resource_dtor_B)); + CHECK(enif_is_identical(arr[4], am_resource_dtor_B)); dtor = resource_dtor_B; } - got_ptr = enif_open_resource_type(env, rt_name, dtor, + got_ptr = enif_open_resource_type(env, NULL, rt_name, dtor, flags.e, &got_res.e); if (enif_get_uint(env, arr[1], &ix) && ix < RT_MAX && got_ptr != NULL) { data->rt_arr[ix] = got_ptr; } else { - CHECK(enif_is_identical(env, arr[1], am_null)); + CHECK(enif_is_identical(arr[1], am_null)); CHECK(got_ptr == NULL); } CHECK(got_res.e == exp_res.e); @@ -126,7 +149,7 @@ static void open_resource_type(ErlNifEnv* env, ERL_NIF_TERM op_tpl) static void do_load_info(ErlNifEnv* env, ERL_NIF_TERM load_info) { - NifModPrivData* data = enif_priv_data(env); + NifModPrivData* data = priv_data(env); ERL_NIF_TERM head, tail; unsigned ix; for (ix=0; ix<RT_MAX; ix++) { @@ -140,17 +163,18 @@ static void do_load_info(ErlNifEnv* env, ERL_NIF_TERM load_info) CHECK(enif_is_empty_list(env, head)); } -static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +static int load(ErlNifEnv* env, void** priv, ERL_NIF_TERM load_info) { NifModPrivData* data; init(env); - data = enif_alloc(env, sizeof(NifModPrivData)); + data = (NifModPrivData*) enif_alloc(sizeof(NifModPrivData)); CHECK(data != NULL); - *priv_data = data; + *priv = data; data->mtx = enif_mutex_create("nif_mod_priv_data"); data->ref_cnt = 1; data->call_history = NULL; + add_call(env, data, "load"); do_load_info(env, load_info); @@ -158,39 +182,35 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) return 0; } -static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +static int reload(ErlNifEnv* env, void** priv, ERL_NIF_TERM load_info) { + NifModPrivData* data = (NifModPrivData*) *priv; init(env); - add_call(env, *priv_data, "reload"); + add_call(env, data, "reload"); do_load_info(env, load_info); return 0; } -static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) +static int upgrade(ErlNifEnv* env, void** priv, void** old_priv_data, ERL_NIF_TERM load_info) { - NifModPrivData* data = *old_priv_data; + NifModPrivData* data = (NifModPrivData*) *old_priv_data; init(env); add_call(env, data, "upgrade"); data->ref_cnt++; - *priv_data = *old_priv_data; + *priv = *old_priv_data; do_load_info(env, load_info); return 0; } -static void unload(ErlNifEnv* env, void* priv_data) +static void unload(ErlNifEnv* env, void* priv) { - NifModPrivData* data = priv_data; + NifModPrivData* data = (NifModPrivData*) priv; + int is_last; add_call(env, data, "unload"); - enif_mutex_lock(data->mtx); - if (--data->ref_cnt == 0) { - enif_mutex_unlock(data->mtx); - enif_mutex_destroy(data->mtx); - enif_free(env, data); - } - enif_mutex_unlock(data->mtx); + NifModPrivData_release(data); } static ERL_NIF_TERM lib_version(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) @@ -202,12 +222,12 @@ static ERL_NIF_TERM lib_version(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg static ERL_NIF_TERM get_priv_data_ptr(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { ADD_CALL("get_priv_data_ptr"); - return enif_make_ulong(env, (unsigned long)enif_priv_data(env)); + return enif_make_ulong(env, (unsigned long)priv_data(env)); } static ERL_NIF_TERM make_new_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - NifModPrivData* data = (NifModPrivData*) enif_priv_data(env); + NifModPrivData* data = priv_data(env); ErlNifBinary ibin; char* a; ERL_NIF_TERM ret; @@ -216,22 +236,22 @@ static ERL_NIF_TERM make_new_resource(ErlNifEnv* env, int argc, const ERL_NIF_TE || !enif_inspect_binary(env, argv[1], &ibin)) { return enif_make_badarg(env); } - a = enif_alloc_resource(env, data->rt_arr[ix], ibin.size); + a = (char*) enif_alloc_resource(data->rt_arr[ix], ibin.size); memcpy(a, ibin.data, ibin.size); ret = enif_make_resource(env, a); - enif_release_resource(env, a); + enif_release_resource(a); return ret; } static ERL_NIF_TERM get_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - NifModPrivData* data = (NifModPrivData*) enif_priv_data(env); + NifModPrivData* data = priv_data(env); ErlNifBinary obin; unsigned ix; void* a; if (!enif_get_uint(env, argv[0], &ix) || ix >= RT_MAX || !enif_get_resource(env, argv[1], data->rt_arr[ix], &a) - || !enif_alloc_binary(env, enif_sizeof_resource(env, a), &obin)) { + || !enif_alloc_binary(enif_sizeof_resource(a), &obin)) { return enif_make_badarg(env); } memcpy(obin.data, a, obin.size); diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.h b/erts/emulator/test/nif_SUITE_data/nif_mod.h index 0eaf91d6e1..cd0ecf4b54 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_mod.h +++ b/erts/emulator/test/nif_SUITE_data/nif_mod.h @@ -20,3 +20,15 @@ typedef struct ErlNifResourceType* rt_arr[RT_MAX]; }NifModPrivData; +#define NifModPrivData_release(NMPD) \ + do { \ + int is_last; \ + enif_mutex_lock((NMPD)->mtx); \ + is_last = (--(NMPD)->ref_cnt == 0); \ + enif_mutex_unlock((NMPD)->mtx); \ + if (is_last) { \ + enif_mutex_destroy((NMPD)->mtx); \ + enif_free((NMPD)); \ + } \ + }while (0) + diff --git a/erts/emulator/test/obsolete_SUITE.erl b/erts/emulator/test/obsolete_SUITE.erl deleted file mode 100644 index 33c4726699..0000000000 --- a/erts/emulator/test/obsolete_SUITE.erl +++ /dev/null @@ -1,123 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2004-2009. All Rights Reserved. -%% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. -%% -%% %CopyrightEnd% -%% - - --module(obsolete_SUITE). --author('[email protected]'). --compile(nowarn_obsolete_guard). - --export([all/1]). - --export([erl_threads/1]). - --include("test_server.hrl"). - --define(DEFAULT_TIMETRAP_SECS, 240). - -all(doc) -> []; -all(suite) -> - case catch erlang:system_info(wordsize) of - 4 -> [erl_threads]; - _ -> {skip, "Only expected to work on 32-bit architectures"} - end. - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% %% -%% Testcases %% -%% %% - -erl_threads(suite) -> []; -erl_threads(doc) -> []; -erl_threads(Cfg) -> - ?line case erlang:system_info(threads) of - true -> - ?line drv_case(Cfg, erl_threads); - false -> - ?line {skip, "Emulator not compiled with threads support"} - end. - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% %% -%% Internal functions %% -%% %% - -drv_case(Config, CaseName) -> - drv_case(Config, CaseName, ""). - -drv_case(Config, CaseName, TimeTrap) when integer(TimeTrap) -> - drv_case(Config, CaseName, "", TimeTrap); -drv_case(Config, CaseName, Command) when list(Command) -> - drv_case(Config, CaseName, Command, ?DEFAULT_TIMETRAP_SECS). - -drv_case(Config, CaseName, TimeTrap, Command) when list(Command), - integer(TimeTrap) -> - drv_case(Config, CaseName, Command, TimeTrap); -drv_case(Config, CaseName, Command, TimeTrap) when list(Config), - atom(CaseName), - list(Command), - integer(TimeTrap) -> - case ?t:os_type() of - {Family, _} when Family == unix; Family == win32 -> - ?line run_drv_case(Config, CaseName, Command, TimeTrap); - SkipOs -> - ?line {skipped, - lists:flatten(["Not run on " - | io_lib:format("~p",[SkipOs])])} - end. - -run_drv_case(Config, CaseName, Command, TimeTrap) -> - ?line Dog = test_server:timetrap(test_server:seconds(TimeTrap)), - ?line DataDir = ?config(data_dir,Config), - case erl_ddll:load_driver(DataDir, CaseName) of - ok -> ok; - {error, Error} -> - io:format("~s\n", [erl_ddll:format_error(Error)]), - ?line ?t:fail() - end, - ?line Port = open_port({spawn, atom_to_list(CaseName)}, []), - ?line true = is_port(Port), - ?line Port ! {self(), {command, Command}}, - ?line Result = receive_drv_result(Port, CaseName), - ?line Port ! {self(), close}, - ?line receive - {Port, closed} -> - ok - end, - ?line ok = erl_ddll:unload_driver(CaseName), - ?line test_server:timetrap_cancel(Dog), - ?line Result. - -receive_drv_result(Port, CaseName) -> - ?line receive - {print, Port, CaseName, Str} -> - ?line ?t:format("~s", [Str]), - ?line receive_drv_result(Port, CaseName); - {'EXIT', Port, Error} -> - ?line ?t:fail(Error); - {'EXIT', error, Error} -> - ?line ?t:fail(Error); - {failed, Port, CaseName, Comment} -> - ?line ?t:fail(Comment); - {skipped, Port, CaseName, Comment} -> - ?line {skipped, Comment}; - {succeeded, Port, CaseName, ""} -> - ?line succeeded; - {succeeded, Port, CaseName, Comment} -> - ?line {comment, Comment} - end. diff --git a/erts/emulator/test/obsolete_SUITE_data/Makefile.src b/erts/emulator/test/obsolete_SUITE_data/Makefile.src deleted file mode 100644 index d8e2b861c0..0000000000 --- a/erts/emulator/test/obsolete_SUITE_data/Makefile.src +++ /dev/null @@ -1,33 +0,0 @@ -# ``The contents of this file are subject to the Erlang Public License, -# Version 1.1, (the "License"); you may not use this file except in -# compliance with the License. You should have received a copy of the -# Erlang Public License along with this software. If not, it can be -# retrieved via the world wide web at http://www.erlang.org/. -# -# Software distributed under the License is distributed on an "AS IS" -# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -# the License for the specific language governing rights and limitations -# under the License. -# -# The Initial Developer of the Original Code is Ericsson Utvecklings AB. -# Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings -# AB. All Rights Reserved.'' -# -# $Id$ -# - -TEST_DRVS = erl_threads@dll@ -CC = @CC@ -LD = @LD@ -CFLAGS = @SHLIB_CFLAGS@ -I@erl_include@ @DEFS@ -SHLIB_EXTRA_LDLIBS = testcase_driver@obj@ - -all: $(TEST_DRVS) - -@SHLIB_RULES@ - -testcase_driver@obj@: testcase_driver.c testcase_driver.h -$(TEST_DRVS): testcase_driver@obj@ - - - diff --git a/erts/emulator/test/obsolete_SUITE_data/erl_threads.c b/erts/emulator/test/obsolete_SUITE_data/erl_threads.c deleted file mode 100644 index 27a5163121..0000000000 --- a/erts/emulator/test/obsolete_SUITE_data/erl_threads.c +++ /dev/null @@ -1,302 +0,0 @@ -/* ``The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved via the world wide web at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * The Initial Developer of the Original Code is Ericsson Utvecklings AB. - * Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings - * AB. All Rights Reserved.'' - * - * $Id$ - */ - -#include "testcase_driver.h" - -#ifndef __WIN32__ - -#define NO_OF_THREADS 2 - -#include <unistd.h> -#include <errno.h> - -static int die; -static int cw_passed; -static int res_tf0; -static int res_tf1; -static erl_mutex_t mtx; -static erl_cond_t cnd; -static erl_thread_t tid[NO_OF_THREADS]; -static int need_join[NO_OF_THREADS]; - -typedef struct { - int n; -} thr_arg_t; - - -static void *tf0(void *vta) -{ - int r; - - if (((thr_arg_t *) vta)->n != 0) - goto fail; - - r = erts_mutex_lock(mtx); - if (r != 0) { - erts_mutex_unlock(mtx); - goto fail; - } - - r = erts_cond_wait(cnd, mtx); - if (r != 0 || die) { - erts_mutex_unlock(mtx); - goto fail; - } - - cw_passed++; - - r = erts_cond_wait(cnd, mtx); - if (r != 0 || die) { - erts_mutex_unlock(mtx); - goto fail; - } - - cw_passed++; - - r = erts_mutex_unlock(mtx); - if (r != 0) - goto fail; - - res_tf0 = 0; - - return (void *) &res_tf0; - - fail: - return NULL; -} - - -static void *tf1(void *vta) -{ - int r; - - if (((thr_arg_t *) vta)->n != 1) - goto fail; - - r = erts_mutex_lock(mtx); - if (r != 0) { - erts_mutex_unlock(mtx); - goto fail; - } - - r = erts_cond_wait(cnd, mtx); - if (r != 0 || die) { - erts_mutex_unlock(mtx); - goto fail; - } - - cw_passed++; - - r = erts_cond_wait(cnd, mtx); - if (r != 0 || die) { - erts_mutex_unlock(mtx); - goto fail; - } - - cw_passed++; - - r = erts_mutex_unlock(mtx); - if (r != 0) - goto fail; - - res_tf1 = 1; - - erts_thread_exit((void *) &res_tf1); - - res_tf1 = 4711; - - fail: - return NULL; -} - -#endif /* #ifndef __WIN32__ */ - -void -testcase_run(TestCaseState_t *tcs) -{ -#ifdef __WIN32__ - testcase_skipped(tcs, "Nothing to test; not supported on windows."); -#else - int i, r; - void *tres[NO_OF_THREADS]; - thr_arg_t ta[NO_OF_THREADS]; - erl_thread_t t1; - - die = 0; - cw_passed = 0; - - for (i = 0; i < NO_OF_THREADS; i++) - need_join[i] = 0; - - res_tf0 = 17; - res_tf1 = 17; - - cnd = mtx = NULL; - - /* Create mutex and cond */ - mtx = erts_mutex_create(); - ASSERT(tcs, mtx); - cnd = erts_cond_create(); - ASSERT(tcs, cnd); - - /* Create the threads */ - ta[0].n = 0; - r = erts_thread_create(&tid[0], tf0, (void *) &ta[0], 0); - ASSERT(tcs, r == 0); - need_join[0] = 1; - - ta[1].n = 1; - r = erts_thread_create(&tid[1], tf1, (void *) &ta[1], 0); - ASSERT(tcs, r == 0); - need_join[1] = 1; - - /* Make sure the threads waits on cond wait */ - sleep(1); - - r = erts_mutex_lock(mtx); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - - ASSERT_CLNUP(tcs, cw_passed == 0, (void) erts_mutex_unlock(mtx)); - - - /* Let one thread pass one cond wait */ - r = erts_cond_signal(cnd); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - - r = erts_mutex_unlock(mtx); - ASSERT(tcs, r == 0); - - sleep(1); - - r = erts_mutex_lock(mtx); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - - ASSERT_CLNUP(tcs, cw_passed == 1, (void) erts_mutex_unlock(mtx)); - - - /* Let both threads pass one cond wait */ - r = erts_cond_broadcast(cnd); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - - r = erts_mutex_unlock(mtx); - ASSERT(tcs, r == 0); - - sleep(1); - - r = erts_mutex_lock(mtx); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - - ASSERT_CLNUP(tcs, cw_passed == 3, (void) erts_mutex_unlock(mtx)); - - - /* Let the thread that only have passed one cond wait pass the other one */ - r = erts_cond_signal(cnd); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - - r = erts_mutex_unlock(mtx); - ASSERT(tcs, r == 0); - - sleep(1); - - r = erts_mutex_lock(mtx); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - - ASSERT_CLNUP(tcs, cw_passed == 4, (void) erts_mutex_unlock(mtx)); - - /* Both threads should have passed both cond waits and exited; - join them and check returned values */ - - r = erts_thread_join(tid[0], &tres[0]); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - need_join[0] = 0; - - ASSERT_CLNUP(tcs, tres[0] == &res_tf0, (void) erts_mutex_unlock(mtx)); - ASSERT_CLNUP(tcs, res_tf0 == 0, (void) erts_mutex_unlock(mtx)); - - r = erts_thread_join(tid[1], &tres[1]); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - need_join[1] = 0; - - ASSERT_CLNUP(tcs, tres[1] == &res_tf1, (void) erts_mutex_unlock(mtx)); - ASSERT_CLNUP(tcs, res_tf1 == 1, (void) erts_mutex_unlock(mtx)); - - /* Test signaling when noone waits */ - - r = erts_cond_signal(cnd); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - - /* Test broadcasting when noone waits */ - - r = erts_cond_broadcast(cnd); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - - /* erts_cond_timedwait() not supported anymore */ - r = erts_cond_timedwait(cnd, mtx, 1000); - ASSERT_CLNUP(tcs, r != 0, (void) erts_mutex_unlock(mtx)); - ASSERT_CLNUP(tcs, - strcmp(erl_errno_id(r), "enotsup") == 0, - (void) erts_mutex_unlock(mtx)); - - r = erts_mutex_unlock(mtx); - ASSERT(tcs, r == 0); - - r = erts_mutex_destroy(mtx); - ASSERT(tcs, r == 0); - mtx = NULL; - - r = erts_cond_destroy(cnd); - ASSERT(tcs, r == 0); - cnd = NULL; - - /* ... */ - t1 = erts_thread_self(); - - if (cw_passed == 4711) { - /* We don't want to execute this just check that the - symbol/symbols is/are defined */ - erts_thread_kill(t1); - } - -#endif /* #ifndef __WIN32__ */ -} - -char * -testcase_name(void) -{ - return "erl_threads"; -} - -void -testcase_cleanup(TestCaseState_t *tcs) -{ - int i; - for (i = 0; i < NO_OF_THREADS; i++) { - if (need_join[i]) { - erts_mutex_lock(mtx); - die = 1; - erts_cond_broadcast(cnd); - erts_mutex_unlock(mtx); - erts_thread_join(tid[1], NULL); - } - } - if (mtx) - erts_mutex_destroy(mtx); - if (cnd) - erts_cond_destroy(cnd); -} - diff --git a/erts/emulator/test/obsolete_SUITE_data/testcase_driver.c b/erts/emulator/test/obsolete_SUITE_data/testcase_driver.c deleted file mode 100644 index 99d5adb041..0000000000 --- a/erts/emulator/test/obsolete_SUITE_data/testcase_driver.c +++ /dev/null @@ -1,262 +0,0 @@ -/* ``The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved via the world wide web at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * The Initial Developer of the Original Code is Ericsson Utvecklings AB. - * Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings - * AB. All Rights Reserved.'' - * - * $Id$ - */ - -#include "testcase_driver.h" -#include <stdio.h> -#include <stdlib.h> -#include <stdarg.h> -#include <setjmp.h> -#include <string.h> - -#ifdef __WIN32__ -#undef HAVE_VSNPRINTF -#define HAVE_VSNPRINTF 1 -#define vsnprintf _vsnprintf -#endif - -#ifndef HAVE_VSNPRINTF -#define HAVE_VSNPRINTF 0 -#endif - -#define COMMENT_BUF_SZ 4096 - -#define TESTCASE_FAILED 0 -#define TESTCASE_SKIPPED 1 -#define TESTCASE_SUCCEEDED 2 - -typedef struct { - TestCaseState_t visible; - int port; - int result; - jmp_buf done_jmp_buf; - char *comment; - char comment_buf[COMMENT_BUF_SZ]; -} InternalTestCaseState_t; - -long testcase_drv_start(int port, char *command); -int testcase_drv_stop(long drv_data); -int testcase_drv_run(long drv_data, char *buf, int len); - -static DriverEntry testcase_drv_entry = { - NULL, - testcase_drv_start, - testcase_drv_stop, - testcase_drv_run -}; - - -int DRIVER_INIT(testcase_drv)(void *arg) -{ - testcase_drv_entry.driver_name = testcase_name(); - return (int) &testcase_drv_entry; -} - -long -testcase_drv_start(int port, char *command) -{ - InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) - driver_alloc(sizeof(InternalTestCaseState_t)); - if (!itcs) { - return -1; - } - - itcs->visible.testcase_name = testcase_name(); - itcs->visible.extra = NULL; - itcs->port = port; - itcs->result = TESTCASE_FAILED; - itcs->comment = ""; - - return (long) itcs; -} - -int -testcase_drv_stop(long drv_data) -{ - testcase_cleanup((TestCaseState_t *) drv_data); - driver_free((void *) drv_data); - return 0; -} - -int -testcase_drv_run(long drv_data, char *buf, int len) -{ - InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) drv_data; - DriverTermData result_atom; - DriverTermData msg[12]; - - itcs->visible.command = buf; - itcs->visible.command_len = len; - - if (setjmp(itcs->done_jmp_buf) == 0) { - testcase_run((TestCaseState_t *) itcs); - itcs->result = TESTCASE_SUCCEEDED; - } - - switch (itcs->result) { - case TESTCASE_SUCCEEDED: - result_atom = driver_mk_atom("succeeded"); - break; - case TESTCASE_SKIPPED: - result_atom = driver_mk_atom("skipped"); - break; - case TESTCASE_FAILED: - default: - result_atom = driver_mk_atom("failed"); - break; - } - - msg[0] = ERL_DRV_ATOM; - msg[1] = (DriverTermData) result_atom; - - msg[2] = ERL_DRV_PORT; - msg[3] = driver_mk_port(itcs->port); - - msg[4] = ERL_DRV_ATOM; - msg[5] = driver_mk_atom(itcs->visible.testcase_name); - - msg[6] = ERL_DRV_STRING; - msg[7] = (DriverTermData) itcs->comment; - msg[8] = (DriverTermData) strlen(itcs->comment); - - msg[9] = ERL_DRV_TUPLE; - msg[10] = (DriverTermData) 4; - - driver_output_term(itcs->port, msg, 11); - return 0; -} - -int -testcase_assertion_failed(TestCaseState_t *tcs, - char *file, int line, char *assertion) -{ - testcase_failed(tcs, "%s:%d: Assertion failed: \"%s\"", - file, line, assertion); - return 0; -} - -void -testcase_printf(TestCaseState_t *tcs, char *frmt, ...) -{ - InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) tcs; - DriverTermData msg[12]; - va_list va; - va_start(va, frmt); -#if HAVE_VSNPRINTF - vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va); -#else - vsprintf(itcs->comment_buf, frmt, va); -#endif - va_end(va); - - msg[0] = ERL_DRV_ATOM; - msg[1] = (DriverTermData) driver_mk_atom("print"); - - msg[2] = ERL_DRV_PORT; - msg[3] = driver_mk_port(itcs->port); - - msg[4] = ERL_DRV_ATOM; - msg[5] = driver_mk_atom(itcs->visible.testcase_name); - - msg[6] = ERL_DRV_STRING; - msg[7] = (DriverTermData) itcs->comment_buf; - msg[8] = (DriverTermData) strlen(itcs->comment_buf); - - msg[9] = ERL_DRV_TUPLE; - msg[10] = (DriverTermData) 4; - - driver_output_term(itcs->port, msg, 11); -} - - -void testcase_succeeded(TestCaseState_t *tcs, char *frmt, ...) -{ - InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) tcs; - va_list va; - va_start(va, frmt); -#if HAVE_VSNPRINTF - vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va); -#else - vsprintf(itcs->comment_buf, frmt, va); -#endif - va_end(va); - - itcs->result = TESTCASE_SUCCEEDED; - itcs->comment = itcs->comment_buf; - - longjmp(itcs->done_jmp_buf, 1); -} - -void testcase_skipped(TestCaseState_t *tcs, char *frmt, ...) -{ - InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) tcs; - va_list va; - va_start(va, frmt); -#if HAVE_VSNPRINTF - vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va); -#else - vsprintf(itcs->comment_buf, frmt, va); -#endif - va_end(va); - - itcs->result = TESTCASE_SKIPPED; - itcs->comment = itcs->comment_buf; - - longjmp(itcs->done_jmp_buf, 1); -} - -void testcase_failed(TestCaseState_t *tcs, char *frmt, ...) -{ - InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) tcs; - char buf[10]; - size_t bufsz = sizeof(buf); - va_list va; - va_start(va, frmt); -#if HAVE_VSNPRINTF - vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va); -#else - vsprintf(itcs->comment_buf, frmt, va); -#endif - va_end(va); - - itcs->result = TESTCASE_FAILED; - itcs->comment = itcs->comment_buf; - - if (erl_drv_getenv("ERL_ABORT_ON_FAILURE", buf, &bufsz) == 0 - && strcmp("true", buf) == 0) { - fprintf(stderr, "Testcase \"%s\" failed: %s\n", - itcs->visible.testcase_name, itcs->comment); - abort(); - } - - longjmp(itcs->done_jmp_buf, 1); -} - -void *testcase_alloc(size_t size) -{ - return driver_alloc(size); -} - -void *testcase_realloc(void *ptr, size_t size) -{ - return driver_realloc(ptr, size); -} - -void testcase_free(void *ptr) -{ - driver_free(ptr); -} diff --git a/erts/emulator/test/obsolete_SUITE_data/testcase_driver.h b/erts/emulator/test/obsolete_SUITE_data/testcase_driver.h deleted file mode 100644 index 3d85ca6df0..0000000000 --- a/erts/emulator/test/obsolete_SUITE_data/testcase_driver.h +++ /dev/null @@ -1,57 +0,0 @@ -/* ``The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved via the world wide web at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * The Initial Developer of the Original Code is Ericsson Utvecklings AB. - * Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings - * AB. All Rights Reserved.'' - * - * $Id$ - */ - -#ifndef TESTCASE_DRIVER_H__ -#define TESTCASE_DRIVER_H__ - -#include "obsolete/driver.h" -#include <stdlib.h> - -typedef struct { - char *testcase_name; - char *command; - int command_len; - void *extra; -} TestCaseState_t; - -#define ASSERT_CLNUP(TCS, B, CLN) \ -do { \ - if (!(B)) { \ - CLN; \ - testcase_assertion_failed((TCS), __FILE__, __LINE__, #B); \ - } \ -} while (0) - -#define ASSERT(TCS, B) ASSERT_CLNUP(TCS, B, (void) 0) - -void testcase_printf(TestCaseState_t *tcs, char *frmt, ...); -void testcase_succeeded(TestCaseState_t *tcs, char *frmt, ...); -void testcase_skipped(TestCaseState_t *tcs, char *frmt, ...); -void testcase_failed(TestCaseState_t *tcs, char *frmt, ...); -int testcase_assertion_failed(TestCaseState_t *tcs, char *file, int line, - char *assertion); -void *testcase_alloc(size_t size); -void *testcase_realloc(void *ptr, size_t size); -void testcase_free(void *ptr); - - -char *testcase_name(void); -void testcase_run(TestCaseState_t *tcs); -void testcase_cleanup(TestCaseState_t *tcs); - -#endif diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl index b9100738e4..77fa75b78f 100644 --- a/erts/emulator/test/port_SUITE.erl +++ b/erts/emulator/test/port_SUITE.erl @@ -88,7 +88,7 @@ otp_3906/1, otp_4389/1, win_massive/1, win_massive_client/1, mix_up_ports/1, otp_5112/1, otp_5119/1, otp_6224/1, exit_status_multi_scheduling_block/1, ports/1, - spawn_driver/1,spawn_executable/1, + spawn_driver/1, spawn_executable/1, close_deaf_port/1, unregister_name/1]). -export([]). @@ -113,7 +113,7 @@ all(suite) -> otp_3906, otp_4389, win_massive, mix_up_ports, otp_5112, otp_5119, exit_status_multi_scheduling_block, - ports, spawn_driver, spawn_executable, + ports, spawn_driver, spawn_executable, close_deaf_port, unregister_name ]. @@ -878,12 +878,20 @@ env2(Config) -> "nisse" = os:getenv(Long) end), - + ?line env_slave(Temp, [{"must_define_something","some_value"}, - {"certainly_not_existing",false}, + {"certainly_not_existing",false}, + {"ends_with_equal", "value="}, {Long,false}, {"glurf","a glorfy string"}]), + %% A lot of non existing variables (mingled with existing) + NotExistingList = [{lists:flatten(io_lib:format("V~p_not_existing",[X])),false} + || X <- lists:seq(1,150)], + ExistingList = [{lists:flatten(io_lib:format("V~p_existing",[X])),"a_value"} + || X <- lists:seq(1,150)], + ?line env_slave(Temp, lists:sort(ExistingList ++ NotExistingList)), + ?line test_server:timetrap_cancel(Dog), ok. @@ -2292,3 +2300,12 @@ load_driver(Dir, Driver) -> io:format("~s\n", [erl_ddll:format_error(Error)]), Res end. + + +close_deaf_port(doc) -> ["Send data to port program that does not read it, then close port."]; +close_deaf_port(suite) -> []; +close_deaf_port(Config) when is_list(Config) -> + Port = open_port({spawn,"sleep 999999"},[]), + erlang:port_command(Port,"Hello, can you hear me!?!?"), + port_close(Port), + ok. diff --git a/erts/emulator/test/receive_SUITE.erl b/erts/emulator/test/receive_SUITE.erl new file mode 100644 index 0000000000..40ebf2bd21 --- /dev/null +++ b/erts/emulator/test/receive_SUITE.erl @@ -0,0 +1,113 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(receive_SUITE). + +%% Tests receive after. + +-include("test_server.hrl"). + +-export([all/1, + call_with_huge_message_queue/1,receive_in_between/1]). + +-export([init_per_testcase/2,fin_per_testcase/2]). + +all(suite) -> + [call_with_huge_message_queue,receive_in_between]. + +init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> + Dog=?t:timetrap(?t:minutes(3)), + [{watchdog, Dog}|Config]. + +fin_per_testcase(_Func, Config) -> + Dog=?config(watchdog, Config), + ?t:timetrap_cancel(Dog). + +call_with_huge_message_queue(Config) when is_list(Config) -> + ?line Pid = spawn_link(fun echo_loop/0), + + ?line {Time,ok} = tc(fun() -> calls(10, Pid) end), + + ?line [self() ! {msg,N} || N <- lists:seq(1, 500000)], + erlang:garbage_collect(), + ?line {NewTime,ok} = tc(fun() -> calls(10, Pid) end), + io:format("Time for empty message queue: ~p", [Time]), + io:format("Time for huge message queue: ~p", [NewTime]), + + case (NewTime+1) / (Time+1) of + Q when Q < 10 -> + ok; + Q -> + io:format("Q = ~p", [Q]), + ?line ?t:fail() + end, + ok. + +calls(0, _) -> ok; +calls(N, Pid) -> + {ok,{ultimate_answer,42}} = call(Pid, {ultimate_answer,42}), + calls(N-1, Pid). + +call(Pid, Msg) -> + Mref = erlang:monitor(process, Pid), + Pid ! {Mref,{self(),Msg}}, + receive + {Mref, Reply} -> + erlang:demonitor(Mref, [flush]), + {ok, Reply}; + {'DOWN', Mref, _, _, Reason} -> + exit(Reason) + end. + +receive_in_between(Config) when is_list(Config) -> + ?line Pid = spawn_link(fun echo_loop/0), + ?line [{ok,{a,b}} = call2(Pid, {a,b}) || _ <- lists:seq(1, 100000)], + ok. + +call2(Pid, Msg) -> + self() ! dummy, + Mref = erlang:monitor(process, Pid), + Pid ! {Mref,{self(),Msg}}, + receive_one(), + receive + {Mref,Reply} -> + erlang:demonitor(Mref, [flush]), + {ok,Reply}; + {'DOWN',Mref,_,_,Reason} -> + exit(Reason) + end. + +receive_one() -> + receive + dummy -> ok + end. + +%%% +%%% Common helpers. +%%% + +echo_loop() -> + receive + {Ref,{Pid,Msg}} -> + Pid ! {Ref,Msg}, + echo_loop() + end. + +tc(Fun) -> + timer:tc(erlang, apply, [Fun,[]]). diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl index c9101b77c2..4f4458802c 100644 --- a/erts/emulator/test/scheduler_SUITE.erl +++ b/erts/emulator/test/scheduler_SUITE.erl @@ -33,7 +33,7 @@ -include("test_server.hrl"). %-compile(export_all). --export([all/1, init_per_testcase/2, fin_per_testcase/2]). +-export([all/1, init_per_testcase/2, fin_per_testcase/2, end_per_suite/1]). -export([equal/1, few_low/1, @@ -49,7 +49,8 @@ cpu_topology/1, sct_cmd/1, sbt_cmd/1, - scheduler_suspend/1]). + scheduler_suspend/1, + reader_groups/1]). -define(DEFAULT_TIMEOUT, ?t:minutes(10)). @@ -67,7 +68,8 @@ all(suite) -> equal_with_high_max, bound_process, scheduler_bind, - scheduler_suspend]. + scheduler_suspend, + reader_groups]. init_per_testcase(Case, Config) when is_list(Config) -> Dog = ?t:timetrap(?DEFAULT_TIMEOUT), @@ -81,6 +83,10 @@ fin_per_testcase(_Case, Config) when is_list(Config) -> ?t:timetrap_cancel(Dog), ok. +end_per_suite(Config) -> + catch erts_debug:set_internal_state(available_internal_state, false), + Config. + -define(ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED, (2000*2000)). -define(DEFAULT_TEST_REDS_PER_SCHED, 200000000). @@ -902,7 +908,8 @@ scheduler_suspend_test(Config, Schedulers) -> ?line [SState] = mcall(Node, [fun () -> erlang:system_info(schedulers_state) end]), - ?line {Sched, _, _} = SState, + ?line ?t:format("SState=~p~n", [SState]), + ?line {Sched, SchedOnln, _SchedAvail} = SState, ?line true = is_integer(Sched), ?line [ok] = mcall(Node, [fun () -> sst0_loop(300) end]), ?line [ok] = mcall(Node, [fun () -> sst1_loop(300) end]), @@ -914,6 +921,14 @@ scheduler_suspend_test(Config, Schedulers) -> fun () -> sst2_loop(200) end, fun () -> sst3_loop(Sched, 200) end]), ?line [SState] = mcall(Node, [fun () -> + case Sched == SchedOnln of + false -> + Sched = erlang:system_flag( + schedulers_online, + SchedOnln); + true -> + ok + end, erlang:system_info(schedulers_state) end]), ?line stop_node(Node), @@ -956,12 +971,361 @@ sst3_loop(S, N) -> erlang:system_flag(schedulers_online, 1), erlang:system_flag(schedulers_online, S), sst3_loop(S, N-1). + +reader_groups(Config) when is_list(Config) -> + %% White box testing. These results are correct, but other results + %% could be too... + + %% The actual tilepro64 topology + CPUT0 = [{processor,[{node,[{core,{logical,0}}, + {core,{logical,1}}, + {core,{logical,2}}, + {core,{logical,8}}, + {core,{logical,9}}, + {core,{logical,10}}, + {core,{logical,11}}, + {core,{logical,16}}, + {core,{logical,17}}, + {core,{logical,18}}, + {core,{logical,19}}, + {core,{logical,24}}, + {core,{logical,25}}, + {core,{logical,27}}, + {core,{logical,29}}]}, + {node,[{core,{logical,3}}, + {core,{logical,4}}, + {core,{logical,5}}, + {core,{logical,6}}, + {core,{logical,7}}, + {core,{logical,12}}, + {core,{logical,13}}, + {core,{logical,14}}, + {core,{logical,15}}, + {core,{logical,20}}, + {core,{logical,21}}, + {core,{logical,22}}, + {core,{logical,23}}, + {core,{logical,28}}, + {core,{logical,30}}]}, + {node,[{core,{logical,31}}, + {core,{logical,36}}, + {core,{logical,37}}, + {core,{logical,38}}, + {core,{logical,44}}, + {core,{logical,45}}, + {core,{logical,46}}, + {core,{logical,47}}, + {core,{logical,51}}, + {core,{logical,52}}, + {core,{logical,53}}, + {core,{logical,54}}, + {core,{logical,55}}, + {core,{logical,60}}, + {core,{logical,61}}]}, + {node,[{core,{logical,26}}, + {core,{logical,32}}, + {core,{logical,33}}, + {core,{logical,34}}, + {core,{logical,35}}, + {core,{logical,39}}, + {core,{logical,40}}, + {core,{logical,41}}, + {core,{logical,42}}, + {core,{logical,43}}, + {core,{logical,48}}, + {core,{logical,49}}, + {core,{logical,50}}, + {core,{logical,58}}]}]}], + + ?line [{0,1},{1,1},{2,1},{3,3},{4,3},{5,3},{6,3},{7,3},{8,1},{9,1},{10,1}, + {11,1},{12,3},{13,3},{14,4},{15,4},{16,2},{17,2},{18,2},{19,2}, + {20,4},{21,4},{22,4},{23,4},{24,2},{25,2},{26,7},{27,2},{28,4}, + {29,2},{30,4},{31,5},{32,7},{33,7},{34,7},{35,7},{36,5},{37,5}, + {38,5},{39,7},{40,7},{41,8},{42,8},{43,8},{44,5},{45,5},{46,5}, + {47,6},{48,8},{49,8},{50,8},{51,6},{52,6},{53,6},{54,6},{55,6}, + {58,8},{60,6},{61,6}] + = reader_groups_map(CPUT0, 8), + + CPUT1 = [n([p([c([t(l(0)),t(l(1)),t(l(2)),t(l(3))]), + c([t(l(4)),t(l(5)),t(l(6)),t(l(7))]), + c([t(l(8)),t(l(9)),t(l(10)),t(l(11))]), + c([t(l(12)),t(l(13)),t(l(14)),t(l(15))])]), + p([c([t(l(16)),t(l(17)),t(l(18)),t(l(19))]), + c([t(l(20)),t(l(21)),t(l(22)),t(l(23))]), + c([t(l(24)),t(l(25)),t(l(26)),t(l(27))]), + c([t(l(28)),t(l(29)),t(l(30)),t(l(31))])])]), + n([p([c([t(l(32)),t(l(33)),t(l(34)),t(l(35))]), + c([t(l(36)),t(l(37)),t(l(38)),t(l(39))]), + c([t(l(40)),t(l(41)),t(l(42)),t(l(43))]), + c([t(l(44)),t(l(45)),t(l(46)),t(l(47))])]), + p([c([t(l(48)),t(l(49)),t(l(50)),t(l(51))]), + c([t(l(52)),t(l(53)),t(l(54)),t(l(55))]), + c([t(l(56)),t(l(57)),t(l(58)),t(l(59))]), + c([t(l(60)),t(l(61)),t(l(62)),t(l(63))])])]), + n([p([c([t(l(64)),t(l(65)),t(l(66)),t(l(67))]), + c([t(l(68)),t(l(69)),t(l(70)),t(l(71))]), + c([t(l(72)),t(l(73)),t(l(74)),t(l(75))]), + c([t(l(76)),t(l(77)),t(l(78)),t(l(79))])]), + p([c([t(l(80)),t(l(81)),t(l(82)),t(l(83))]), + c([t(l(84)),t(l(85)),t(l(86)),t(l(87))]), + c([t(l(88)),t(l(89)),t(l(90)),t(l(91))]), + c([t(l(92)),t(l(93)),t(l(94)),t(l(95))])])]), + n([p([c([t(l(96)),t(l(97)),t(l(98)),t(l(99))]), + c([t(l(100)),t(l(101)),t(l(102)),t(l(103))]), + c([t(l(104)),t(l(105)),t(l(106)),t(l(107))]), + c([t(l(108)),t(l(109)),t(l(110)),t(l(111))])]), + p([c([t(l(112)),t(l(113)),t(l(114)),t(l(115))]), + c([t(l(116)),t(l(117)),t(l(118)),t(l(119))]), + c([t(l(120)),t(l(121)),t(l(122)),t(l(123))]), + c([t(l(124)),t(l(125)),t(l(126)),t(l(127))])])])], + + ?line [{0,1},{1,1},{2,1},{3,1},{4,2},{5,2},{6,2},{7,2},{8,3},{9,3}, + {10,3},{11,3},{12,4},{13,4},{14,4},{15,4},{16,5},{17,5},{18,5}, + {19,5},{20,6},{21,6},{22,6},{23,6},{24,7},{25,7},{26,7},{27,7}, + {28,8},{29,8},{30,8},{31,8},{32,9},{33,9},{34,9},{35,9},{36,10}, + {37,10},{38,10},{39,10},{40,11},{41,11},{42,11},{43,11},{44,12}, + {45,12},{46,12},{47,12},{48,13},{49,13},{50,13},{51,13},{52,14}, + {53,14},{54,14},{55,14},{56,15},{57,15},{58,15},{59,15},{60,16}, + {61,16},{62,16},{63,16},{64,17},{65,17},{66,17},{67,17},{68,18}, + {69,18},{70,18},{71,18},{72,19},{73,19},{74,19},{75,19},{76,20}, + {77,20},{78,20},{79,20},{80,21},{81,21},{82,21},{83,21},{84,22}, + {85,22},{86,22},{87,22},{88,23},{89,23},{90,23},{91,23},{92,24}, + {93,24},{94,24},{95,24},{96,25},{97,25},{98,25},{99,25},{100,26}, + {101,26},{102,26},{103,26},{104,27},{105,27},{106,27},{107,27}, + {108,28},{109,28},{110,28},{111,28},{112,29},{113,29},{114,29}, + {115,29},{116,30},{117,30},{118,30},{119,30},{120,31},{121,31}, + {122,31},{123,31},{124,32},{125,32},{126,32},{127,32}] + = reader_groups_map(CPUT1, 128), + + ?line [{0,1},{1,1},{2,1},{3,1},{4,1},{5,1},{6,1},{7,1},{8,1},{9,1},{10,1}, + {11,1},{12,1},{13,1},{14,1},{15,1},{16,1},{17,1},{18,1},{19,1}, + {20,1},{21,1},{22,1},{23,1},{24,1},{25,1},{26,1},{27,1},{28,1}, + {29,1},{30,1},{31,1},{32,1},{33,1},{34,1},{35,1},{36,1},{37,1}, + {38,1},{39,1},{40,1},{41,1},{42,1},{43,1},{44,1},{45,1},{46,1}, + {47,1},{48,1},{49,1},{50,1},{51,1},{52,1},{53,1},{54,1},{55,1}, + {56,1},{57,1},{58,1},{59,1},{60,1},{61,1},{62,1},{63,1},{64,2}, + {65,2},{66,2},{67,2},{68,2},{69,2},{70,2},{71,2},{72,2},{73,2}, + {74,2},{75,2},{76,2},{77,2},{78,2},{79,2},{80,2},{81,2},{82,2}, + {83,2},{84,2},{85,2},{86,2},{87,2},{88,2},{89,2},{90,2},{91,2}, + {92,2},{93,2},{94,2},{95,2},{96,2},{97,2},{98,2},{99,2},{100,2}, + {101,2},{102,2},{103,2},{104,2},{105,2},{106,2},{107,2},{108,2}, + {109,2},{110,2},{111,2},{112,2},{113,2},{114,2},{115,2},{116,2}, + {117,2},{118,2},{119,2},{120,2},{121,2},{122,2},{123,2},{124,2}, + {125,2},{126,2},{127,2}] + = reader_groups_map(CPUT1, 2), + + ?line [{0,1},{1,1},{2,1},{3,1},{4,2},{5,2},{6,2},{7,2},{8,3},{9,3},{10,3}, + {11,3},{12,3},{13,3},{14,3},{15,3},{16,4},{17,4},{18,4},{19,4}, + {20,4},{21,4},{22,4},{23,4},{24,5},{25,5},{26,5},{27,5},{28,5}, + {29,5},{30,5},{31,5},{32,6},{33,6},{34,6},{35,6},{36,6},{37,6}, + {38,6},{39,6},{40,7},{41,7},{42,7},{43,7},{44,7},{45,7},{46,7}, + {47,7},{48,8},{49,8},{50,8},{51,8},{52,8},{53,8},{54,8},{55,8}, + {56,9},{57,9},{58,9},{59,9},{60,9},{61,9},{62,9},{63,9},{64,10}, + {65,10},{66,10},{67,10},{68,10},{69,10},{70,10},{71,10},{72,11}, + {73,11},{74,11},{75,11},{76,11},{77,11},{78,11},{79,11},{80,12}, + {81,12},{82,12},{83,12},{84,12},{85,12},{86,12},{87,12},{88,13}, + {89,13},{90,13},{91,13},{92,13},{93,13},{94,13},{95,13},{96,14}, + {97,14},{98,14},{99,14},{100,14},{101,14},{102,14},{103,14}, + {104,15},{105,15},{106,15},{107,15},{108,15},{109,15},{110,15}, + {111,15},{112,16},{113,16},{114,16},{115,16},{116,16},{117,16}, + {118,16},{119,16},{120,17},{121,17},{122,17},{123,17},{124,17}, + {125,17},{126,17},{127,17}] + = reader_groups_map(CPUT1, 17), + + ?line [{0,1},{1,1},{2,1},{3,1},{4,1},{5,1},{6,1},{7,1},{8,1},{9,1},{10,1}, + {11,1},{12,1},{13,1},{14,1},{15,1},{16,2},{17,2},{18,2},{19,2}, + {20,2},{21,2},{22,2},{23,2},{24,2},{25,2},{26,2},{27,2},{28,2}, + {29,2},{30,2},{31,2},{32,3},{33,3},{34,3},{35,3},{36,3},{37,3}, + {38,3},{39,3},{40,3},{41,3},{42,3},{43,3},{44,3},{45,3},{46,3}, + {47,3},{48,4},{49,4},{50,4},{51,4},{52,4},{53,4},{54,4},{55,4}, + {56,4},{57,4},{58,4},{59,4},{60,4},{61,4},{62,4},{63,4},{64,5}, + {65,5},{66,5},{67,5},{68,5},{69,5},{70,5},{71,5},{72,5},{73,5}, + {74,5},{75,5},{76,5},{77,5},{78,5},{79,5},{80,6},{81,6},{82,6}, + {83,6},{84,6},{85,6},{86,6},{87,6},{88,6},{89,6},{90,6},{91,6}, + {92,6},{93,6},{94,6},{95,6},{96,7},{97,7},{98,7},{99,7},{100,7}, + {101,7},{102,7},{103,7},{104,7},{105,7},{106,7},{107,7},{108,7}, + {109,7},{110,7},{111,7},{112,7},{113,7},{114,7},{115,7},{116,7}, + {117,7},{118,7},{119,7},{120,7},{121,7},{122,7},{123,7},{124,7}, + {125,7},{126,7},{127,7}] + = reader_groups_map(CPUT1, 7), + + ?line CPUT2 = [p([c(l(0)),c(l(1)),c(l(2)),c(l(3)),c(l(4))]), + p([t(l(5)),t(l(6)),t(l(7)),t(l(8)),t(l(9))]), + p([t(l(10))]), + p([c(l(11)),c(l(12)),c(l(13))]), + p([c(l(14)),c(l(15))])], + + ?line [{0,1},{1,1},{2,1},{3,1},{4,1}, + {5,2},{6,2},{7,2},{8,2},{9,2}, + {10,3}, + {11,4},{12,4},{13,4}, + {14,5},{15,5}] = reader_groups_map(CPUT2, 5), + + + ?line [{0,1},{1,1},{2,2},{3,2},{4,2}, + {5,3},{6,3},{7,3},{8,3},{9,3}, + {10,4}, + {11,5},{12,5},{13,5}, + {14,6},{15,6}] = reader_groups_map(CPUT2, 6), + + ?line [{0,1},{1,1},{2,2},{3,2},{4,2}, + {5,3},{6,3},{7,3},{8,3},{9,3}, + {10,4}, + {11,5},{12,6},{13,6}, + {14,7},{15,7}] = reader_groups_map(CPUT2, 7), + + ?line [{0,1},{1,1},{2,2},{3,2},{4,2}, + {5,3},{6,3},{7,3},{8,3},{9,3}, + {10,4}, + {11,5},{12,6},{13,6}, + {14,7},{15,8}] = reader_groups_map(CPUT2, 8), + + ?line [{0,1},{1,2},{2,2},{3,3},{4,3}, + {5,4},{6,4},{7,4},{8,4},{9,4}, + {10,5}, + {11,6},{12,7},{13,7}, + {14,8},{15,9}] = reader_groups_map(CPUT2, 9), + + ?line [{0,1},{1,2},{2,2},{3,3},{4,3}, + {5,4},{6,4},{7,4},{8,4},{9,4}, + {10,5}, + {11,6},{12,7},{13,8}, + {14,9},{15,10}] = reader_groups_map(CPUT2, 10), + + ?line [{0,1},{1,2},{2,3},{3,4},{4,4}, + {5,5},{6,5},{7,5},{8,5},{9,5}, + {10,6}, + {11,7},{12,8},{13,9}, + {14,10},{15,11}] = reader_groups_map(CPUT2, 11), + + ?line [{0,1},{1,2},{2,3},{3,4},{4,5}, + {5,6},{6,6},{7,6},{8,6},{9,6}, + {10,7}, + {11,8},{12,9},{13,10}, + {14,11},{15,12}] = reader_groups_map(CPUT2, 100), + + CPUT3 = [p([t(l(5)),t(l(6)),t(l(7)),t(l(8)),t(l(9))]), + p([t(l(10))]), + p([c(l(11)),c(l(12)),c(l(13))]), + p([c(l(14)),c(l(15))]), + p([c(l(0)),c(l(1)),c(l(2)),c(l(3)),c(l(4))])], + + ?line [{0,5},{1,5},{2,6},{3,6},{4,6}, + {5,1},{6,1},{7,1},{8,1},{9,1}, + {10,2},{11,3},{12,3},{13,3}, + {14,4},{15,4}] = reader_groups_map(CPUT3, 6), + + CPUT4 = [p([t(l(0)),t(l(1)),t(l(2)),t(l(3)),t(l(4))]), + p([t(l(5))]), + p([c(l(6)),c(l(7)),c(l(8))]), + p([c(l(9)),c(l(10))]), + p([c(l(11)),c(l(12)),c(l(13)),c(l(14)),c(l(15))])], + + ?line [{0,1},{1,1},{2,1},{3,1},{4,1}, + {5,2}, + {6,3},{7,3},{8,3}, + {9,4},{10,4}, + {11,5},{12,5},{13,6},{14,6},{15,6}] = reader_groups_map(CPUT4, 6), + + ?line [{0,1},{1,1},{2,1},{3,1},{4,1}, + {5,2}, + {6,3},{7,4},{8,4}, + {9,5},{10,5}, + {11,6},{12,6},{13,7},{14,7},{15,7}] = reader_groups_map(CPUT4, 7), + + ?line [{0,1},{65535,2}] = reader_groups_map([c(l(0)),c(l(65535))], 10), + + ?line ok. +reader_groups_map(CPUT, Groups) -> + Old = erlang:system_info({cpu_topology, defined}), + erlang:system_flag(cpu_topology, CPUT), + enable_internal_state(), + Res = erts_debug:get_internal_state({reader_groups_map, Groups}), + erlang:system_flag(cpu_topology, Old), + lists:sort(Res). + %% %% Utils %% +tilera_cpu_topology() -> + [{processor,[{node,[{core,{logical,0}}, + {core,{logical,1}}, + {core,{logical,2}}, + {core,{logical,8}}, + {core,{logical,9}}, + {core,{logical,10}}, + {core,{logical,11}}, + {core,{logical,16}}, + {core,{logical,17}}, + {core,{logical,18}}, + {core,{logical,19}}, + {core,{logical,24}}, + {core,{logical,25}}, + {core,{logical,27}}, + {core,{logical,29}}]}, + {node,[{core,{logical,3}}, + {core,{logical,4}}, + {core,{logical,5}}, + {core,{logical,6}}, + {core,{logical,7}}, + {core,{logical,12}}, + {core,{logical,13}}, + {core,{logical,14}}, + {core,{logical,15}}, + {core,{logical,20}}, + {core,{logical,21}}, + {core,{logical,22}}, + {core,{logical,23}}, + {core,{logical,28}}, + {core,{logical,30}}]}, + {node,[{core,{logical,31}}, + {core,{logical,36}}, + {core,{logical,37}}, + {core,{logical,38}}, + {core,{logical,44}}, + {core,{logical,45}}, + {core,{logical,46}}, + {core,{logical,47}}, + {core,{logical,51}}, + {core,{logical,52}}, + {core,{logical,53}}, + {core,{logical,54}}, + {core,{logical,55}}, + {core,{logical,60}}, + {core,{logical,61}}]}, + {node,[{core,{logical,26}}, + {core,{logical,32}}, + {core,{logical,33}}, + {core,{logical,34}}, + {core,{logical,35}}, + {core,{logical,39}}, + {core,{logical,40}}, + {core,{logical,41}}, + {core,{logical,42}}, + {core,{logical,43}}, + {core,{logical,48}}, + {core,{logical,49}}, + {core,{logical,50}}, + {core,{logical,58}}]}]}]. + +l(Id) -> + {logical, Id}. + +t(X) -> + {thread, X}. + +c(X) -> + {core, X}. + +p(X) -> + {processor, X}. + +n(X) -> + {node, X}. + mcall(Node, Funs) -> Parent = self(), Refs = lists:map(fun (Fun) -> diff --git a/erts/emulator/test/send_term_SUITE.erl b/erts/emulator/test/send_term_SUITE.erl index 489adbd660..5fd01a9ac5 100644 --- a/erts/emulator/test/send_term_SUITE.erl +++ b/erts/emulator/test/send_term_SUITE.erl @@ -76,40 +76,43 @@ basic(Config) when is_list(Config) -> ?line ExpectedBinTup = term(P, 7), %% single terms - ?line [] = term(P, 8), % ERL_DRV_NIL - ?line '' = term(P, 9), % ERL_DRV_ATOM - ?line an_atom = term(P, 10), % ERL_DRV_ATOM - ?line -4711 = term(P, 11), % ERL_DRV_INT - ?line 4711 = term(P, 12), % ERL_DRV_UINT - ?line P = term(P, 13), % ERL_DRV_PORT - ?line <<>> = term(P, 14), % ERL_DRV_BINARY - ?line <<"hejsan">> = term(P, 15), % ERL_DRV_BINARY - ?line <<>> = term(P, 16), % ERL_DRV_BUF2BINARY - ?line <<>> = term(P, 17), % ERL_DRV_BUF2BINARY - ?line <<"hoppsan">> = term(P, 18), % ERL_DRV_BUF2BINARY - ?line "" = term(P, 19), % ERL_DRV_STRING - ?line "" = term(P, 20), % ERL_DRV_STRING - ?line "hippsan" = term(P, 21), % ERL_DRV_STRING - ?line {} = term(P, 22), % ERL_DRV_TUPLE - ?line [] = term(P, 23), % ERL_DRV_LIST - ?line Self = term(P, 24), % ERL_DRV_PID - ?line [] = term(P, 25), % ERL_DRV_STRING_CONS - ?line AFloat = term(P, 26), % ERL_DRV_FLOAT + Singles = [{[], 8}, % ERL_DRV_NIL + {'', 9}, % ERL_DRV_ATOM + {an_atom, 10}, % ERL_DRV_ATOM + {-4711, 11}, % ERL_DRV_INT + {4711, 12}, % ERL_DRV_UINT + {P, 13}, % ERL_DRV_PORT + {<<>>, 14}, % ERL_DRV_BINARY + {<<"hejsan">>, 15}, % ERL_DRV_BINARY + {<<>>, 16}, % ERL_DRV_BUF2BINARY + {<<>>, 17}, % ERL_DRV_BUF2BINARY + {<<"hoppsan">>, 18}, % ERL_DRV_BUF2BINARY + {"", 19}, % ERL_DRV_STRING + {"", 20}, % ERL_DRV_STRING + {"hippsan", 21}, % ERL_DRV_STRING + {{}, 22}, % ERL_DRV_TUPLE + {[], 23}, % ERL_DRV_LIST + {Self, 24}, % ERL_DRV_PID + {[], 25}, % ERL_DRV_STRING_CONS + {[], 27}, % ERL_DRV_EXT2TERM + {18446744073709551615, 28}, % ERL_DRV_UINT64 + {20233590931456, 29}, % ERL_DRV_UINT64 + {4711, 30}, % ERL_DRV_UINT64 + {0, 31}, % ERL_DRV_UINT64 + {9223372036854775807, 32}, % ERL_DRV_INT64 + {20233590931456, 33}, % ERL_DRV_INT64 + {4711, 34}, % ERL_DRV_INT64 + {0, 35}, % ERL_DRV_INT64 + {-1, 36}, % ERL_DRV_INT64 + {-4711, 37}, % ERL_DRV_INT64 + {-20233590931456, 38}, % ERL_DRV_INT64 + {-9223372036854775808, 39}], % ERL_DRV_INT64 + ?line {Terms, Ops} = lists:unzip(Singles), + ?line Terms = term(P,Ops), + + AFloat = term(P, 26), % ERL_DRV_FLOAT ?line true = AFloat < 0.001, ?line true = AFloat > -0.001, - ?line [] = term(P, 27), % ERL_DRV_EXT2TERM - ?line 18446744073709551615 = term(P, 28), % ERL_DRV_UINT64 - ?line 20233590931456 = term(P, 29), % ERL_DRV_UINT64 - ?line 4711 = term(P, 30), % ERL_DRV_UINT64 - ?line 0 = term(P, 31), % ERL_DRV_UINT64 - ?line 9223372036854775807 = term(P, 32), % ERL_DRV_INT64 - ?line 20233590931456 = term(P, 33), % ERL_DRV_INT64 - ?line 4711 = term(P, 34), % ERL_DRV_INT64 - ?line 0 = term(P, 35), % ERL_DRV_INT64 - ?line -1 = term(P, 36), % ERL_DRV_INT64 - ?line -4711 = term(P, 37), % ERL_DRV_INT64 - ?line -20233590931456 = term(P, 38), % ERL_DRV_INT64 - ?line -9223372036854775808 = term(P, 39), % ERL_DRV_INT64 %% Failure cases. ?line [] = term(P, 127), diff --git a/erts/emulator/test/send_term_SUITE_data/send_term_drv.c b/erts/emulator/test/send_term_SUITE_data/send_term_drv.c index 6638de0560..165cce2e9d 100644 --- a/erts/emulator/test/send_term_SUITE_data/send_term_drv.c +++ b/erts/emulator/test/send_term_SUITE_data/send_term_drv.c @@ -17,6 +17,7 @@ */ #include "erl_driver.h" +#include <stdio.h> #include <errno.h> #include <string.h> @@ -65,12 +66,21 @@ static void fail_term(ErlDrvTermData* msg, int len, int line); static void send_term_drv_run(ErlDrvData port, char *buf, int count) { - ErlDrvTermData msg[1024]; - - switch (*buf) { + char buf7[1024]; + ErlDrvTermData spec[1024]; + ErlDrvTermData* msg = spec; + ErlDrvBinary* bins[15]; + int bin_ix = 0; + ErlDrvSInt64 s64[15]; + int s64_ix = 0; + ErlDrvUInt64 u64[15]; + int u64_ix = 0; + int i = 0; + + for (i=0; i<count; i++) switch (buf[i]) { case 0: msg[0] = ERL_DRV_NIL; - output_term(msg, 1); + msg += 1; break; case 1: /* Most term types inside a tuple. */ @@ -102,7 +112,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) msg[22] = driver_connected(erlang_port); msg[23] = ERL_DRV_TUPLE; msg[24] = (ErlDrvTermData) 7; - output_term(msg, 25); + msg += 25; } break; @@ -117,7 +127,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) msg[i] = ERL_DRV_NIL; msg[i+1] = ERL_DRV_LIST; msg[i+2] = (ErlDrvTermData) 201; - output_term(msg, i+3); + msg += i+3; } break; @@ -126,7 +136,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) ErlDrvBinary* bin; int i; - bin = driver_alloc_binary(256); + bin = bins[bin_ix++] = driver_alloc_binary(256); for (i = 0; i < 256; i++) { bin->orig_bytes[i] = i; } @@ -140,8 +150,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) msg[7] = (ErlDrvTermData) 23; msg[8] = ERL_DRV_TUPLE; msg[9] = (ErlDrvTermData) 2; - output_term(msg, 10); - driver_free_binary(bin); + msg += 10; } break; @@ -152,11 +161,11 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) msg[3] = driver_caller(erlang_port); msg[4] = ERL_DRV_TUPLE; msg[5] = (ErlDrvTermData) 2; - output_term(msg, 6); + msg += 6; break; case 5: - output_term(msg, make_ext_term_list(msg, 0)); + msg += make_ext_term_list(msg, 0); break; case 6: @@ -166,94 +175,91 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) msg[3] = ~((ErlDrvTermData) 0); msg[4] = ERL_DRV_TUPLE; msg[5] = (ErlDrvTermData) 2; - output_term(msg, 6); + msg += 6; break; case 7: { int len = 0; - char buf[1024]; - memset(buf, 17, sizeof(buf)); + memset(buf7, 17, sizeof(buf7)); /* empty heap binary */ msg[len++] = ERL_DRV_BUF2BINARY; msg[len++] = (ErlDrvTermData) NULL; /* NULL is ok if size == 0 */ msg[len++] = (ErlDrvTermData) 0; /* empty heap binary again */ msg[len++] = ERL_DRV_BUF2BINARY; - msg[len++] = (ErlDrvTermData) &buf[0]; /* ptr is ok if size == 0 */ + msg[len++] = (ErlDrvTermData) buf7; /* ptr is ok if size == 0 */ msg[len++] = (ErlDrvTermData) 0; /* heap binary */ msg[len++] = ERL_DRV_BUF2BINARY; - msg[len++] = (ErlDrvTermData) &buf[0]; + msg[len++] = (ErlDrvTermData) buf7; msg[len++] = (ErlDrvTermData) 17; /* off heap binary */ msg[len++] = ERL_DRV_BUF2BINARY; - msg[len++] = (ErlDrvTermData) &buf[0]; - msg[len++] = (ErlDrvTermData) sizeof(buf); + msg[len++] = (ErlDrvTermData) buf7; + msg[len++] = (ErlDrvTermData) sizeof(buf7); msg[len++] = ERL_DRV_TUPLE; msg[len++] = (ErlDrvTermData) 4; - output_term(msg, len); + msg += len; break; } case 8: msg[0] = ERL_DRV_NIL; - output_term(msg, 1); + msg += 1; break; case 9: msg[0] = ERL_DRV_ATOM; msg[1] = (ErlDrvTermData) driver_mk_atom(""); - output_term(msg, 2); + msg += 2; break; case 10: msg[0] = ERL_DRV_ATOM; msg[1] = (ErlDrvTermData) driver_mk_atom("an_atom"); - output_term(msg, 2); + msg += 2; break; case 11: msg[0] = ERL_DRV_INT; msg[1] = (ErlDrvTermData) -4711; - output_term(msg, 2); + msg += 2; break; case 12: msg[0] = ERL_DRV_UINT; msg[1] = (ErlDrvTermData) 4711; - output_term(msg, 2); + msg += 2; break; case 13: msg[0] = ERL_DRV_PORT; msg[1] = driver_mk_port(erlang_port); - output_term(msg, 2); + msg += 2; break; case 14: { - ErlDrvBinary *dbin = driver_alloc_binary(0); + ErlDrvBinary *dbin = bins[bin_ix++] = driver_alloc_binary(0); msg[0] = ERL_DRV_BINARY; msg[1] = (ErlDrvTermData) dbin; msg[2] = (ErlDrvTermData) 0; msg[3] = (ErlDrvTermData) 0; - output_term(msg, 4); - driver_free_binary(dbin); + msg += 4; break; } case 15: { - char buf[] = "hejsan"; - ErlDrvBinary *dbin = driver_alloc_binary(sizeof(buf)-1); + static const char buf[] = "hejsan"; + ErlDrvBinary *dbin = bins[bin_ix++] = driver_alloc_binary(sizeof(buf)-1); if (dbin) memcpy((void *) dbin->orig_bytes, (void *) buf, sizeof(buf)-1); msg[0] = ERL_DRV_BINARY; msg[1] = (ErlDrvTermData) dbin; msg[2] = (ErlDrvTermData) (dbin ? sizeof(buf)-1 : 0); msg[3] = (ErlDrvTermData) 0; - output_term(msg, 4); - driver_free_binary(dbin); + msg += 4; break; } @@ -261,24 +267,24 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) msg[0] = ERL_DRV_BUF2BINARY; msg[1] = (ErlDrvTermData) NULL; msg[2] = (ErlDrvTermData) 0; - output_term(msg, 3); + msg += 3; break; case 17: { - char buf[] = ""; + static const char buf[] = ""; msg[0] = ERL_DRV_BUF2BINARY; msg[1] = (ErlDrvTermData) buf; msg[2] = (ErlDrvTermData) sizeof(buf)-1; - output_term(msg, 3); + msg += 3; break; } case 18: { - char buf[] = "hoppsan"; + static const char buf[] = "hoppsan"; msg[0] = ERL_DRV_BUF2BINARY; msg[1] = (ErlDrvTermData) buf; msg[2] = (ErlDrvTermData) sizeof(buf)-1; - output_term(msg, 3); + msg += 3; break; } @@ -286,44 +292,44 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) msg[0] = ERL_DRV_STRING; msg[1] = (ErlDrvTermData) buf; msg[2] = (ErlDrvTermData) 0; - output_term(msg, 3); + msg += 3; break; case 20: { - char buf[] = ""; + static const char buf[] = ""; msg[0] = ERL_DRV_STRING; msg[1] = (ErlDrvTermData) buf; msg[2] = (ErlDrvTermData) sizeof(buf)-1; - output_term(msg, 3); + msg += 3; break; } case 21: { - char buf[] = "hippsan"; + static const char buf[] = "hippsan"; msg[0] = ERL_DRV_STRING; msg[1] = (ErlDrvTermData) buf; msg[2] = (ErlDrvTermData) sizeof(buf)-1; - output_term(msg, 3); + msg += 3; break; } case 22: msg[0] = ERL_DRV_TUPLE; msg[1] = (ErlDrvTermData) 0; - output_term(msg, 2); + msg += 2; break; case 23: msg[0] = ERL_DRV_NIL; msg[1] = ERL_DRV_LIST; msg[2] = (ErlDrvTermData) 1; - output_term(msg, 3); + msg += 3; break; case 24: msg[0] = ERL_DRV_PID; msg[1] = driver_connected(erlang_port); - output_term(msg, 2); + msg += 2; break; case 25: @@ -331,132 +337,131 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) msg[1] = ERL_DRV_STRING_CONS; msg[2] = (ErlDrvTermData) ""; msg[3] = (ErlDrvTermData) 0; - output_term(msg, 4); + msg += 4; break; case 26: { - double my_float = 0.0; + static double my_float = 0.0; msg[0] = ERL_DRV_FLOAT; msg[1] = (ErlDrvTermData) &my_float; - output_term(msg, 2); + msg += 2; break; } case 27: { - char buf[] = {131, 106}; /* [] */ + static char buf[] = {131, 106}; /* [] */ msg[0] = ERL_DRV_EXT2TERM; msg[1] = (ErlDrvTermData) buf; msg[2] = (ErlDrvTermData) sizeof(buf); - output_term(msg, 3); + msg += 3; break; } case 28: { - ErlDrvUInt64 x = ~((ErlDrvUInt64) 0); + ErlDrvUInt64* x = &u64[u64_ix++]; + *x = ~((ErlDrvUInt64) 0); msg[0] = ERL_DRV_UINT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 29: { - ErlDrvUInt64 x = ((ErlDrvUInt64) 4711) << 32; + ErlDrvUInt64* x = &u64[u64_ix++]; + *x = ((ErlDrvUInt64) 4711) << 32; msg[0] = ERL_DRV_UINT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 30: { - ErlDrvUInt64 x = 4711; + ErlDrvUInt64* x = &u64[u64_ix++]; + *x = 4711; msg[0] = ERL_DRV_UINT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 31: { - ErlDrvUInt64 x = 0; + ErlDrvUInt64* x = &u64[u64_ix++]; + *x = 0; msg[0] = ERL_DRV_UINT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 32: { - ErlDrvSInt64 x = ((((ErlDrvUInt64) 0x7fffffff) << 32) - | ((ErlDrvUInt64) 0xffffffff)); + ErlDrvSInt64* x = &s64[s64_ix++]; + *x = ((((ErlDrvUInt64) 0x7fffffff) << 32) | ((ErlDrvUInt64) 0xffffffff)); msg[0] = ERL_DRV_INT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 33: { - ErlDrvSInt64 x = (ErlDrvSInt64) (((ErlDrvUInt64) 4711) << 32); + ErlDrvSInt64* x = &s64[s64_ix++]; + *x = (ErlDrvSInt64) (((ErlDrvUInt64) 4711) << 32); msg[0] = ERL_DRV_INT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 34: { - ErlDrvSInt64 x = 4711; + ErlDrvSInt64* x = &s64[s64_ix++]; + *x = 4711; msg[0] = ERL_DRV_INT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 35: { - ErlDrvSInt64 x = 0; + ErlDrvSInt64* x = &s64[s64_ix++]; + *x = 0; msg[0] = ERL_DRV_INT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 36: { - ErlDrvSInt64 x = -1; + ErlDrvSInt64* x = &s64[s64_ix++]; + *x = -1; msg[0] = ERL_DRV_INT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 37: { - ErlDrvSInt64 x = -4711; + ErlDrvSInt64* x = &s64[s64_ix++]; + *x = -4711; msg[0] = ERL_DRV_INT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 38: { - ErlDrvSInt64 x = ((ErlDrvSInt64) ((ErlDrvUInt64) 4711) << 32)*-1; + ErlDrvSInt64* x = &s64[s64_ix++]; + *x = ((ErlDrvSInt64) ((ErlDrvUInt64) 4711) << 32)*-1; msg[0] = ERL_DRV_INT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 39: { - ErlDrvSInt64 x = ((ErlDrvSInt64) 1) << 63; + ErlDrvSInt64* x = &s64[s64_ix++]; + *x = ((ErlDrvSInt64) 1) << 63; msg[0] = ERL_DRV_INT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } @@ -464,7 +469,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) case 127: /* Error cases */ { long refc; - ErlDrvBinary* bin = driver_alloc_binary(256); + ErlDrvBinary* bin = bins[bin_ix++] = driver_alloc_binary(256); FAIL_TERM(msg, 0); @@ -537,7 +542,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) refc = driver_binary_get_refc(bin); if (refc > 3) { char sbuf[128]; - sprintf(sbuf, "bad_refc:%d", refc); + sprintf(sbuf, "bad_refc:%ld", refc); driver_failure_atom(erlang_port, sbuf); } driver_free_binary(bin); @@ -644,6 +649,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) /* Signal end of test case */ msg[0] = ERL_DRV_NIL; driver_output_term(erlang_port, msg, 1); + return; } break; @@ -651,6 +657,16 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) driver_failure_atom(erlang_port, "bad_request"); break; } + if (count > 1) { + *msg++ = ERL_DRV_NIL; + *msg++ = ERL_DRV_LIST; + *msg++ = count + 1; + } + output_term(spec, msg-spec); + if ((bin_ix|s64_ix|u64_ix) > 15) abort(); + while (bin_ix) { + driver_free_binary(bins[--bin_ix]); + } } static void output_term(ErlDrvTermData* msg, int len) diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl index e782d2f293..ba433d4e11 100644 --- a/erts/emulator/test/system_info_SUITE.erl +++ b/erts/emulator/test/system_info_SUITE.erl @@ -35,12 +35,12 @@ %-compile(export_all). -export([all/1, init_per_testcase/2, fin_per_testcase/2]). --export([process_count/1, system_version/1, misc_smoke_tests/1, heap_size/1]). +-export([process_count/1, system_version/1, misc_smoke_tests/1, heap_size/1, wordsize/1]). -define(DEFAULT_TIMEOUT, ?t:minutes(2)). all(doc) -> []; -all(suite) -> [process_count, system_version, misc_smoke_tests, heap_size]. +all(suite) -> [process_count, system_version, misc_smoke_tests, heap_size, wordsize]. init_per_testcase(_Case, Config) when is_list(Config) -> Dog = ?t:timetrap(?DEFAULT_TIMEOUT), @@ -145,3 +145,23 @@ heap_size(Config) when is_list(Config) -> ?line Hmin = proplists:get_value(min_heap_size, GCinf), ok. +wordsize(suite) -> + []; +wordsize(doc) -> + ["Tests the various wordsize variants"]; +wordsize(Config) when is_list(Config) -> + ?line A = erlang:system_info(wordsize), + ?line true = is_integer(A), + ?line A = erlang:system_info({wordsize,internal}), + ?line B = erlang:system_info({wordsize,external}), + ?line true = A =< B, + case {B,A} of + {4,4} -> + {comment, "True 32-bit emulator"}; + {8,8} -> + {comment, "True 64-bit emulator"}; + {8,4} -> + {comment, "Halfword 64-bit emulator"}; + Other -> + exit({unexpected_wordsizes,Other}) + end. diff --git a/erts/emulator/test/time_SUITE.erl b/erts/emulator/test/time_SUITE.erl index 2ad1f0d201..095e9dd1af 100644 --- a/erts/emulator/test/time_SUITE.erl +++ b/erts/emulator/test/time_SUITE.erl @@ -34,6 +34,8 @@ consistency/1, now/1, now_unique/1, now_update/1, timestamp/1]). +-export([local_to_univ_utc/1]). + -include("test_server.hrl"). -export([linear_time/1]). @@ -53,7 +55,40 @@ -define(dst_timezone, 2). all(suite) -> [univ_to_local, local_to_univ, - bad_univ_to_local, bad_local_to_univ, consistency, now, timestamp]. + local_to_univ_utc, + bad_univ_to_local, bad_local_to_univ, + consistency, now, timestamp]. + +local_to_univ_utc(suite) -> + []; +local_to_univ_utc(doc) -> + ["Test that DST = true on timezones without DST is ignored"]; +local_to_univ_utc(Config) when is_list(Config) -> + case os:type() of + {unix,_} -> + %% TZ variable has a meaning + ?line {ok, Node} = + test_server:start_node(local_univ_utc,peer, + [{args, "-env TZ UTC"}]), + ?line {{2008,8,1},{0,0,0}} = + rpc:call(Node, + erlang,localtime_to_universaltime, + [{{2008, 8, 1}, {0, 0, 0}}, + false]), + ?line {{2008,8,1},{0,0,0}} = + rpc:call(Node, + erlang,localtime_to_universaltime, + [{{2008, 8, 1}, {0, 0, 0}}, + true]), + ?line [{{2008,8,1},{0,0,0}}] = + rpc:call(Node, + calendar,local_time_to_universal_time_dst, + [{{2008, 8, 1}, {0, 0, 0}}]), + ?line test_server:stop_node(Node), + ok; + _ -> + {skip,"Only valid on Unix"} + end. %% Tests conversion from univeral to local time. diff --git a/erts/emulator/test/trace_call_time_SUITE.erl b/erts/emulator/test/trace_call_time_SUITE.erl new file mode 100644 index 0000000000..7bc91addde --- /dev/null +++ b/erts/emulator/test/trace_call_time_SUITE.erl @@ -0,0 +1,614 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% +%%% Define to run outside of test server +%%% +%%% -define(STANDALONE,1). +%%% +%%% +%%% Define for debug output +%%% +%%% -define(debug,1). + +-module(trace_call_time_SUITE). + +%% Exported end user tests + +-export([seq/3, seq_r/3]). +-export([loaded/1, a_function/1, a_called_function/1, dec/1, nif_dec/1]). + +-define(US_ERROR, 10000). +-define(R_ERROR, 0.8). +-define(SINGLE_CALL_US_TIME, 10). + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Result examination macros + +-define(CT(P,MFA),{trace,P,call,MFA}). +-define(CTT(P, MFA),{trace_ts,P,call,MFA,{_,_,_}}). +-define(RF(P,MFA,V),{trace,P,return_from,MFA,V}). +-define(RFT(P,MFA,V),{trace_ts,P,return_from,MFA,V,{_,_,_}}). +-define(RT(P,MFA),{trace,P,return_to,MFA}). +-define(RTT(P,MFA),{trace_ts,P,return_to,MFA,{_,_,_}}). + +-ifdef(debug). +-define(dbgformat(A,B),io:format(A,B)). +-else. +-define(dbgformat(A,B),noop). +-endif. + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +-include("test_server.hrl"). + +%% When run in test server. +-export([all/1, init_per_testcase/2, fin_per_testcase/2, not_run/1]). +-export([basic/1, on_and_off/1, info/1, + pause_and_restart/1, scheduling/1, called_function/1, combo/1, bif/1, nif/1]). + +init_per_testcase(_Case, Config) -> + ?line Dog=test_server:timetrap(test_server:seconds(400)), + erlang:trace_pattern({'_','_','_'}, false, [local,meta,call_time,call_count]), + erlang:trace_pattern(on_load, false, [local,meta,call_time,call_count]), + timer:now_diff(now(),now()), + [{watchdog, Dog}|Config]. + +fin_per_testcase(_Case, Config) -> + erlang:trace_pattern({'_','_','_'}, false, [local,meta,call_time,call_count]), + erlang:trace_pattern(on_load, false, [local,meta,call_time,call_count]), + erlang:trace(all, false, [all]), + Dog=?config(watchdog, Config), + test_server:timetrap_cancel(Dog), + ok. + +all(doc) -> + ["Test call count tracing of local function calls."]; +all(suite) -> + case test_server:is_native(?MODULE) of + true -> [not_run]; + false -> [basic, on_and_off, info, + pause_and_restart, scheduling, combo, bif, nif, called_function] + end. + +not_run(Config) when is_list(Config) -> + {skipped,"Native code"}. + +basic(suite) -> + []; +basic(doc) -> + ["Tests basic call count trace"]; +basic(Config) when is_list(Config) -> + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ?line M = 1000, + %% + ?line 1 = erlang:trace_pattern({?MODULE,seq, '_'}, true, [call_time]), + ?line 2 = erlang:trace_pattern({?MODULE,seq_r,'_'}, true, [call_time]), + ?line Pid = setup(), + ?line {L, T1} = execute(Pid, fun() -> seq(1, M, fun(X) -> (X+1) end) end), + ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid, M, 0, 0}], T1), + ?line ok = check_trace_info({?MODULE, seq_r, 3}, [], none), + + ?line {Lr, T2} = execute(Pid, fun() -> seq_r(1, M, fun(X) -> (X+1) end) end), + ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid, M, 0, 0}], T1), + ?line ok = check_trace_info({?MODULE, seq_r, 3}, [{Pid, 1, 0, 0}], T2/M), + ?line ok = check_trace_info({?MODULE, seq_r, 4}, [{Pid, M, 0, 0}], T2), + ?line L = lists:reverse(Lr), + + %% + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ?line Pid ! quit, + ok. + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + +on_and_off(suite) -> + []; +on_and_off(doc) -> + ["Tests turning trace parameters on and off"]; +on_and_off(Config) when is_list(Config) -> + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ?line M = 100, + %% + ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, true, [call_time]), + ?line Pid = setup(), + ?line {L, T1} = execute(Pid, {?MODULE, seq, [1, M, fun(X) -> X+1 end]}), + ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid, M, 0, 0}], T1), + + ?line N = erlang:trace_pattern({?MODULE,'_','_'}, true, [call_time]), + ?line {L, T2} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end), + ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid, M, 0, 0}], T2), + + ?line P = erlang:trace_pattern({'_','_','_'}, true, [call_time]), + ?line {L, T3} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end), + ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid, M, 0, 0}], T3), + + ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, false, [call_time]), + ?line ok = check_trace_info({?MODULE, seq, 3}, false, none), + ?line {L, _T4} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end), + ?line ok = check_trace_info({?MODULE, seq, 3}, false, none), + ?line ok = check_trace_info({?MODULE, seq_r, 4}, [], none), + ?line {Lr, T5} = execute(Pid, fun() -> seq_r(1, M, fun(X) -> X+1 end) end), + ?line ok = check_trace_info({?MODULE, seq_r, 4}, [{Pid,M,0,0}], T5), + + ?line N = erlang:trace_pattern({?MODULE,'_','_'}, false, [call_time]), + ?line ok = check_trace_info({?MODULE, seq_r, 4}, false, none), + ?line {Lr, _T6} = execute(Pid, fun() -> seq_r(1, M, fun(X) -> X+1 end) end), + ?line ok = check_trace_info({?MODULE, seq_r, 4}, false, none), + ?line L = lists:reverse(Lr), + %% + ?line Pid ! quit, + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ok. + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +info(suite) -> + []; +info(doc) -> + ["Tests the trace_info BIF"]; +info(Config) when is_list(Config) -> + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + %% + ?line 1 = erlang:trace_pattern({?MODULE,seq,3}, true, [call_time]), + ?line {call_time,[]} = erlang:trace_info({?MODULE,seq,3}, call_time), + ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, pause, [call_time]), + ?line {call_time,[]} = erlang:trace_info({?MODULE,seq,3}, call_time), + ?line {all,[_|_]=L} = erlang:trace_info({?MODULE,seq,3}, all), + ?line {value,{call_time,[]}} = lists:keysearch(call_time, 1, L), + ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, restart, [call_time]), + ?line {call_time,[]} = erlang:trace_info({?MODULE,seq,3}, call_time), + ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, false, [call_time]), + ?line {call_time,false} = erlang:trace_info({?MODULE,seq,3}, call_time), + ?line {all,false} = erlang:trace_info({?MODULE,seq,3}, all), + %% + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ok. + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +pause_and_restart(suite) -> + []; +pause_and_restart(doc) -> + ["Tests pausing and restarting call time counters"]; +pause_and_restart(Config) when is_list(Config) -> + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ?line M = 100, + ?line Pid = setup(), + %% + ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, true, [call_time]), + ?line ok = check_trace_info({?MODULE, seq, 3}, [], none), + ?line {L, T1} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end), + ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid,M,0,0}], T1), + ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, pause, [call_time]), + ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid,M,0,0}], T1), + ?line {L, T2} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end), + ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid,M,0,0}], T2), + ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, restart, [call_time]), + ?line ok = check_trace_info({?MODULE, seq, 3}, [], none), + ?line {L, T3} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end), + ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid,M,0,0}], T3), + %% + ?line Pid ! quit, + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ok. + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +scheduling(suite) -> + []; +scheduling(doc) -> + ["Tests in/out scheduling of call time counters"]; +scheduling(Config) when is_list(Config) -> + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ?line M = 1000000, + ?line Np = erlang:system_info(schedulers_online), + ?line F = 12, + + %% setup load processes + %% (single, no internal calls) + + ?line erlang:trace_pattern({?MODULE,loaded,1}, true, [call_time]), + + ?line Pids = [setup() || _ <- lists:seq(1, F*Np)], + ?line {_Ls,T1} = execute(Pids, {?MODULE,loaded,[M]}), + ?line [Pid ! quit || Pid <- Pids], + + %% logic dictates that each process will get ~ 1/F of the schedulers time + + ?line {call_time, CT} = erlang:trace_info({?MODULE,loaded,1}, call_time), + + ?line lists:foreach(fun (Pid) -> + ?line ok = case check_process_time(lists:keysearch(Pid, 1, CT), M, F, T1) of + schedule_time_error -> + test_server:comment("Warning: Failed time ratio"), + ok; + Other -> Other + end + end, Pids), + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ok. + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +combo(suite) -> + []; +combo(doc) -> + ["Tests combining local call trace and meta trace with call time trace"]; +combo(Config) when is_list(Config) -> + ?line Self = self(), + ?line Nbc = 3, + ?line MetaMs = [{'_',[],[{return_trace}]}], + ?line Flags = lists:sort([call, return_to]), + ?line LocalTracer = spawn_link(fun () -> relay_n(5 + Nbc + 3, Self) end), + ?line MetaTracer = spawn_link(fun () -> relay_n(9 + Nbc + 3, Self) end), + ?line 2 = erlang:trace_pattern({?MODULE,seq_r,'_'}, [], [local]), + ?line 2 = erlang:trace_pattern({?MODULE,seq_r,'_'}, true, [call_time]), + ?line 2 = erlang:trace_pattern({?MODULE,seq_r,'_'}, MetaMs, [{meta,MetaTracer}]), + ?line 2 = erlang:trace_pattern({?MODULE,seq_r,'_'}, true, [call_count]), + + % bifs + ?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, [], [local]), + ?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, true, [call_time]), + ?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, MetaMs, [{meta,MetaTracer}]), + %% not implemented + %?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, true, [call_count]), + + ?line 1 = erlang:trace(Self, true, [{tracer,LocalTracer} | Flags]), + %% + ?line {traced,local} = + erlang:trace_info({?MODULE,seq_r,3}, traced), + ?line {match_spec,[]} = + erlang:trace_info({?MODULE,seq_r,3}, match_spec), + ?line {meta,MetaTracer} = + erlang:trace_info({?MODULE,seq_r,3}, meta), + ?line {meta_match_spec,MetaMs} = + erlang:trace_info({?MODULE,seq_r,3}, meta_match_spec), + ?line ok = check_trace_info({?MODULE, seq_r, 3}, [], none), + + %% check empty trace_info for ?MODULE:seq_r/3 + ?line {all,[_|_]=TraceInfo} = erlang:trace_info({?MODULE,seq_r,3}, all), + ?line {value,{traced,local}} = lists:keysearch(traced, 1, TraceInfo), + ?line {value,{match_spec,[]}} = lists:keysearch(match_spec, 1, TraceInfo), + ?line {value,{meta,MetaTracer}} = lists:keysearch(meta, 1, TraceInfo), + ?line {value,{meta_match_spec,MetaMs}} = lists:keysearch(meta_match_spec, 1, TraceInfo), + ?line {value,{call_count,0}} = lists:keysearch(call_count, 1, TraceInfo), + ?line {value,{call_time,[]}} = lists:keysearch(call_time, 1, TraceInfo), + + %% check empty trace_info for erlang:term_to_binary/1 + ?line {all, [_|_] = TraceInfoBif} = erlang:trace_info({erlang, term_to_binary, 1}, all), + ?line {value,{traced,local}} = lists:keysearch(traced, 1, TraceInfoBif), + ?line {value,{match_spec,[]}} = lists:keysearch(match_spec, 1, TraceInfoBif), + ?line {value,{meta, MetaTracer}} = lists:keysearch(meta, 1, TraceInfoBif), + ?line {value,{meta_match_spec,MetaMs}} = lists:keysearch(meta_match_spec, 1, TraceInfoBif), + %% not implemented + ?line {value,{call_count,false}} = lists:keysearch(call_count, 1, TraceInfoBif), + %?line {value,{call_count,0}} = lists:keysearch(call_count, 1, TraceInfoBif), + ?line {value,{call_time,[]}} = lists:keysearch(call_time, 1, TraceInfoBif), + + %% + ?line [3,2,1] = seq_r(1, 3, fun(X) -> X+1 end), + ?line T0 = now(), + ?line with_bif(Nbc), + ?line T1 = now(), + ?line TimeB = timer:now_diff(T1,T0), + %% + + ?line List = collect(100), + ?line {MetaR, LocalR} = + lists:foldl( + fun ({P,X}, {M,L}) when P == MetaTracer -> + {[X|M],L}; + ({P,X}, {M,L}) when P == LocalTracer -> + {M,[X|L]} + end, + {[],[]}, + List), + ?line Meta = lists:reverse(MetaR), + ?line Local = lists:reverse(LocalR), + + ?line [?CTT(Self,{?MODULE,seq_r,[1,3,_]}), + ?CTT(Self,{?MODULE,seq_r,[1,3,_,[]]}), + ?CTT(Self,{?MODULE,seq_r,[2,3,_,[1]]}), + ?CTT(Self,{?MODULE,seq_r,[3,3,_,[2,1]]}), + ?RFT(Self,{?MODULE,seq_r,4},[3,2,1]), + ?RFT(Self,{?MODULE,seq_r,4},[3,2,1]), + ?RFT(Self,{?MODULE,seq_r,4},[3,2,1]), + ?RFT(Self,{?MODULE,seq_r,3},[3,2,1]), + ?CTT(Self,{erlang,term_to_binary,[3]}), % bif + ?RFT(Self,{erlang,term_to_binary,1},<<131,97,3>>), + ?CTT(Self,{erlang,term_to_binary,[2]}), + ?RFT(Self,{erlang,term_to_binary,1},<<131,97,2>>) + ] = Meta, + + ?line [?CT(Self,{?MODULE,seq_r,[1,3,_]}), + ?CT(Self,{?MODULE,seq_r,[1,3,_,[]]}), + ?CT(Self,{?MODULE,seq_r,[2,3,_,[1]]}), + ?CT(Self,{?MODULE,seq_r,[3,3,_,[2,1]]}), + ?RT(Self,{?MODULE,combo,1}), + ?CT(Self,{erlang,term_to_binary,[3]}), % bif + ?RT(Self,{?MODULE,with_bif,1}), + ?CT(Self,{erlang,term_to_binary,[2]}), + ?RT(Self,{?MODULE,with_bif,1}) + ] = Local, + + ?line ok = check_trace_info({?MODULE, seq_r, 3}, [{Self,1,0,0}], 1), + ?line ok = check_trace_info({?MODULE, seq_r, 4}, [{Self,3,0,0}], 1), + ?line ok = check_trace_info({?MODULE, seq_r, 3}, [{Self,1,0,0}], 1), + ?line ok = check_trace_info({?MODULE, seq_r, 4}, [{Self,3,0,0}], 1), + ?line ok = check_trace_info({erlang, term_to_binary, 1}, [{self(), Nbc - 1, 0, 0}], TimeB), + %% + ?line erlang:trace_pattern({'_','_','_'}, false, [local,meta,call_time]), + ?line erlang:trace_pattern(on_load, false, [local,meta,call_time]), + ?line erlang:trace(all, false, [all]), + ok. + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +bif(suite) -> + []; +bif(doc) -> + ["Tests tracing of bifs"]; +bif(Config) when is_list(Config) -> + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ?line M = 1000000, + %% + ?line 2 = erlang:trace_pattern({erlang, binary_to_term, '_'}, true, [call_time]), + ?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, true, [call_time]), + ?line Pid = setup(), + ?line {L, T1} = execute(Pid, fun() -> with_bif(M) end), + + ?line ok = check_trace_info({erlang, binary_to_term, 1}, [{Pid, M - 1, 0, 0}], T1/2), + ?line ok = check_trace_info({erlang, term_to_binary, 1}, [{Pid, M - 1, 0, 0}], T1/2), + + % disable term2binary + + ?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, false, [call_time]), + + ?line {L, T2} = execute(Pid, fun() -> with_bif(M) end), + + ?line ok = check_trace_info({erlang, binary_to_term, 1}, [{Pid, M*2 - 2, 0, 0}], T1/2 + T2), + ?line ok = check_trace_info({erlang, term_to_binary, 1}, false, none), + + %% + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ?line Pid ! quit, + ok. + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +nif(suite) -> + []; +nif(doc) -> + ["Tests tracing of nifs"]; +nif(Config) when is_list(Config) -> + load_nif(Config), + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ?line M = 1000000, + %% + ?line 1 = erlang:trace_pattern({?MODULE, nif_dec, '_'}, true, [call_time]), + ?line 1 = erlang:trace_pattern({?MODULE, with_nif, '_'}, true, [call_time]), + ?line Pid = setup(), + ?line {L, T1} = execute(Pid, fun() -> with_nif(M) end), + + % the nif is called M - 1 times, the last time the function with 'with_nif' + % returns ok and does not call the nif. + ?line ok = check_trace_info({?MODULE, nif_dec, 1}, [{Pid, M-1, 0, 0}], T1/5*4), + ?line ok = check_trace_info({?MODULE, with_nif, 1}, [{Pid, M, 0, 0}], T1/5), + + %% + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ?line Pid ! quit, + ok. + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +called_function(suite) -> + []; +called_function(doc) -> + ["Tests combining nested function calls and that the time accumulates to the right function"]; +called_function(Config) when is_list(Config) -> + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ?line M = 2100, + ?line Pid = setup(), + %% + ?line 1 = erlang:trace_pattern({?MODULE,a_function,'_'}, true, [call_time]), + ?line {L, T1} = execute(Pid, {?MODULE, a_function, [M]}), + ?line ok = check_trace_info({?MODULE, a_function, 1}, [{Pid, M, 0, 0}], T1), + + ?line 1 = erlang:trace_pattern({?MODULE,a_called_function,'_'}, true, [call_time]), + ?line {L, T2} = execute(Pid, {?MODULE, a_function, [M]}), + ?line ok = check_trace_info({?MODULE, a_function, 1}, [{Pid, M+M, 0, 0}], T1 + M*?SINGLE_CALL_US_TIME), + ?line ok = check_trace_info({?MODULE, a_called_function, 1}, [{Pid, M, 0, 0}], T2), + + + ?line 1 = erlang:trace_pattern({?MODULE,dec,'_'}, true, [call_time]), + ?line {L, T3} = execute(Pid, {?MODULE, a_function, [M]}), + ?line ok = check_trace_info({?MODULE, a_function, 1}, [{Pid, M+M+M, 0, 0}], T1 + (M+M)*?SINGLE_CALL_US_TIME), + ?line ok = check_trace_info({?MODULE, a_called_function, 1}, [{Pid, M+M, 0, 0}], T2 + M*?SINGLE_CALL_US_TIME ), + ?line ok = check_trace_info({?MODULE, dec, 1}, [{Pid, M, 0, 0}], T3), + + ?line Pid ! quit, + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ok. + +%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% The Tests +%%% + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Local helpers + + +load_nif(Config) -> + ?line Path = ?config(data_dir, Config), + ?line ok = erlang:load_nif(filename:join(Path,"trace_nif"), 0). + + +%% Stack recursive seq +seq(Stop, Stop, Succ) when is_function(Succ) -> + [Stop]; +seq(Start, Stop, Succ) when is_function(Succ) -> + [Start | seq(Succ(Start), Stop, Succ)]. + + +a_function(1) -> a_called_function(1); +a_function(N) when N > 1 -> a_function(a_called_function(N)). + +a_called_function(N) -> dec(N). + +with_bif(1) -> ok; +with_bif(N) -> + with_bif(erlang:binary_to_term(erlang:term_to_binary(N)) - 1). + +with_nif(0) -> error; +with_nif(1) -> ok; +with_nif(N) -> + with_nif(?MODULE:nif_dec(N)). + + +nif_dec(N) -> 0. + +dec(N) -> + loaded(10000), + N - 1. + +loaded(N) when N > 1 -> loaded(N - 1); +loaded(_) -> 5. + + +%% Tail recursive seq, result list is reversed +seq_r(Start, Stop, Succ) when is_function(Succ) -> + seq_r(Start, Stop, Succ, []). + +seq_r(Stop, Stop, _, R) -> + [Stop | R]; +seq_r(Start, Stop, Succ, R) -> + seq_r(Succ(Start), Stop, Succ, [Start | R]). + +% Check call time tracing data and print mismatches +check_trace_info(Mfa, [{Pid, C,_,_}] = Expect, Time) -> + case erlang:trace_info(Mfa, call_time) of + % Time tests are somewhat problematic. We want to know if Time (EXPECTED_TIME) and S*1000000 + Us (ACTUAL_TIME) + % is the same. + % If the ratio EXPECTED_TIME/ACTUAL_TIME is ~ 1 or if EXPECTED_TIME - ACTUAL_TIME is near zero, the test is ok. + {call_time,[{Pid,C,S,Us}]} when S >= 0, Us >= 0, abs(1 - Time/(S*1000000 + Us)) < ?R_ERROR; abs(Time - S*1000000 - Us) < ?US_ERROR -> + ok; + {call_time,[{Pid,C,S,Us}]} -> + Sum = S*1000000 + Us, + io:format("Expected ~p -> {call_time, ~p (Time ~p us)}~n - got ~w s. ~w us. = ~w us. - ~w -> delta ~w (ratio ~.2f, should be 1.0)~n", + [Mfa, Expect, Time, S, Us, Sum, Time, Sum - Time, Time/Sum]), + time_error; + Other -> + io:format("Expected ~p -> {call_time, ~p (Time ~p us)}~n - got ~p~n", [ Mfa, Expect, Time, Other]), + time_count_error + end; +check_trace_info(Mfa, Expect, _) -> + case erlang:trace_info(Mfa, call_time) of + {call_time, Expect} -> + ok; + Other -> + io:format("Expected ~p -> {call_time, ~p}~n - got ~p~n", [Mfa, Expect, Other]), + result_not_expected_error + end. + + +%check process time +check_process_time({value,{Pid, M, S, Us}}, M, F, Time) -> + ?line Sum = S*1000000 + Us, + if + abs(1 - (F/(Time/Sum))) < ?R_ERROR -> + ok; + true -> + io:format("- Pid ~p, Got ratio ~.2f, expected ratio ~w~n", [Pid, Time/Sum,F]), + schedule_time_error + end; +check_process_time(Other, M, _, _) -> + io:format(" - Got ~p, expected count ~w~n", [Other, M]), + error. + + + +%% Message relay process +relay_n(0, _) -> + ok; +relay_n(N, Dest) -> + receive Msg -> + Dest ! {self(), Msg}, + relay_n(N-1, Dest) + end. + + + +%% Collect received messages +collect(Time) -> + Ref = erlang:start_timer(Time, self(), done), + L = lists:reverse(collect([], Ref)), + ?dbgformat("Got: ~p~n",[L]), + L. + +collect(A, 0) -> + receive + Mess -> + collect([Mess | A], 0) + after 0 -> + A + end; +collect(A, Ref) -> + receive + {timeout, Ref, done} -> + collect(A, 0); + Mess -> + collect([Mess | A], Ref) + end. + +setup() -> + Pid = spawn_link(fun() -> loop() end), + ?line 1 = erlang:trace(Pid, true, [call]), + Pid. + +execute(Pids, Mfa) when is_list(Pids) -> + T0 = now(), + [P ! {self(), execute, Mfa} || P <- Pids], + As = [receive {P, answer, Answer} -> Answer end || P <- Pids], + T1 = now(), + {As, timer:now_diff(T1,T0)}; +execute(P, Mfa) -> + T0 = now(), + P ! {self(), execute, Mfa}, + A = receive {P, answer, Answer} -> Answer end, + T1 = now(), + {A, timer:now_diff(T1,T0)}. + + + +loop() -> + receive + quit -> + ok; + {Pid, execute, Fun } when is_function(Fun) -> + Pid ! {self(), answer, erlang:apply(Fun, [])}, + loop(); + {Pid, execute, {M, F, A}} -> + Pid ! {self(), answer, erlang:apply(M, F, A)}, + loop() + end. diff --git a/erts/emulator/test/trace_call_time_SUITE_data/Makefile.src b/erts/emulator/test/trace_call_time_SUITE_data/Makefile.src new file mode 100644 index 0000000000..2b2a35bd2c --- /dev/null +++ b/erts/emulator/test/trace_call_time_SUITE_data/Makefile.src @@ -0,0 +1,6 @@ + +NIF_LIBS = trace_nif@dll@ + +all: $(NIF_LIBS) + +@SHLIB_RULES@ diff --git a/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c b/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c new file mode 100644 index 0000000000..33b346aab7 --- /dev/null +++ b/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c @@ -0,0 +1,37 @@ +#include "erl_nif.h" + + +static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +{ + return 0; +} + +static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +{ + return 0; +} + +static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) +{ + return 0; +} + +static void unload(ErlNifEnv* env, void* priv_data) +{ +} + +static ERL_NIF_TERM nif_dec_1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + int x = 0; + enif_get_uint(env, argv[0], &x); + return enif_make_int(env, x - 1); +} + + + +static ErlNifFunc nif_funcs[] = +{ + {"nif_dec", 1, nif_dec_1} +}; + +ERL_NIF_INIT(trace_call_time_SUITE,nif_funcs,load,reload,upgrade,unload) diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops index 4a859c3094..de19a2e35b 100755 --- a/erts/emulator/utils/beam_makeops +++ b/erts/emulator/utils/beam_makeops @@ -1,20 +1,20 @@ #!/usr/bin/env perl # # %CopyrightBegin% -# -# Copyright Ericsson AB 1998-2009. All Rights Reserved. -# +# +# Copyright Ericsson AB 1998-2010. All Rights Reserved. +# # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in # compliance with the License. You should have received a copy of the # Erlang Public License along with this software. If not, it can be # retrieved online at http://www.erlang.org/. -# +# # Software distributed under the License is distributed on an "AS IS" # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See # the License for the specific language governing rights and limitations # under the License. -# +# # %CopyrightEnd% # use strict; @@ -499,7 +499,11 @@ sub emulator_output { print "\n"; print "#ifdef ARCH_64\n"; print "# define BEAM_LOOSE_MASK 0x1FFFUL\n"; + print "#if HALFWORD_HEAP\n"; + print "# define BEAM_TIGHT_MASK 0x1FFCUL\n"; + print "#else\n"; print "# define BEAM_TIGHT_MASK 0x1FF8UL\n"; + print "#endif\n"; print "# define BEAM_LOOSE_SHIFT 16\n"; print "# define BEAM_TIGHT_SHIFT 16\n"; print "#else\n"; @@ -813,7 +817,7 @@ sub basic_generator { # if ($flags =~ /-pack/ && $hot) { - ($prefix, $pack_spec, @args) = &do_pack(@args); + ($prefix, $pack_spec, @args) = &do_pack(@args); } # @@ -907,16 +911,16 @@ sub basic_generator { my($code); if (defined $macro{$name}) { my($macro_code) = "$prefix$macro(" . join(', ', @f) . ");"; - $var_decls .= "Uint tmp_packed1;" + $var_decls .= "BeamInstr tmp_packed1;" if $macro_code =~ /tmp_packed1/; - $var_decls .= "Uint tmp_packed2;" + $var_decls .= "BeamInstr tmp_packed2;" if $macro_code =~ /tmp_packed2/; if ($flags =~ /-nonext/) { $code = "$macro_code\n"; } else { $code = join("\n", "{ $var_decls", - "Eterm* next;", + "BeamInstr* next;", "PreFetch($size, next);", "$macro_code", "NextPF($size, next);", diff --git a/erts/emulator/utils/make_tables b/erts/emulator/utils/make_tables index b5391234cf..918ef62094 100755 --- a/erts/emulator/utils/make_tables +++ b/erts/emulator/utils/make_tables @@ -1,20 +1,20 @@ #!/usr/bin/env perl # # %CopyrightBegin% -# -# Copyright Ericsson AB 1999-2009. All Rights Reserved. -# +# +# Copyright Ericsson AB 1999-2010. All Rights Reserved. +# # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in # compliance with the License. You should have received a copy of the # Erlang Public License along with this software. If not, it can be # retrieved online at http://www.erlang.org/. -# +# # Software distributed under the License is distributed on an "AS IS" # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See # the License for the specific language governing rights and limitations # under the License. -# +# # %CopyrightEnd% # use strict; @@ -184,7 +184,7 @@ for ($i = 0; $i < @bif; $i++) { my $arity = $bif[$i]->[2]; my $args = join(', ', 'Process*', ('Eterm') x $arity); print "Eterm $bif[$i]->[3]($args);\n"; - print "Eterm wrap_$bif[$i]->[3]($args, Uint *I);\n"; + print "Eterm wrap_$bif[$i]->[3]($args, UWord *I);\n"; } print "#endif\n"; @@ -225,15 +225,15 @@ for ($i = 0; $i < @bif; $i++) { for ($arg = 1; $arg <= $arity; $arg++) { print ", Eterm arg$arg"; } - print ", Uint *I)\n"; + print ", UWord *I)\n"; print "{\n"; print " return erts_bif_trace($i, p"; for ($arg = 1; $arg <= 3; $arg++) { if ($arg <= $arity) { print ", arg$arg"; - } elsif ($arg == ($arity + 1)) { - # Place I in correct position - print ", (Eterm) I"; + #} elsif ($arg == ($arity + 1)) { + # # Place I in correct position + # print ", (Eterm) I"; } else { print ", 0"; } |